from ..base import LoadClassStep
from ...loaders import Loader
[docs]class ImportStep(LoadClassStep):
"""
Load raw data as a known format.
After reading in raw data with ReadStep,
this data should be prepared and parsed as
one of the supported formats.
"""
base_class = Loader
import_path = 'xmm.pipeline.loaders.'
load_name = 'loader'
def configure_options(self, options):
self.count = options.get('count')
if self.count:
del options['count']
return options
def process_step(self, state, context):
context['load'] = {}
self.loader.begin(context)
stream = next(state) if state is not None else None
if self.count:
try:
context['total_count'] = self.loader.get_count(stream)
except NotImplementedError:
pass
state = self.loader.load(stream)
return state, context
def cleanup(self, context, exception=None):
self.loader.end(context)
return context