Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def assert_all_results_equivalent(expected_results, result_results):
check.list_param(expected_results, 'expected_results', of_type=SolidExecutionResult)
check.list_param(result_results, 'result_results', of_type=SolidExecutionResult)
assert len(expected_results) == len(result_results)
for expected, result in zip(expected_results, result_results):
assert_equivalent_results(expected, result)
def interrupted(steps_interrupted):
check.list_param(steps_interrupted, 'steps_interrupted', str)
return EngineEventData(
metadata_entries=[EventMetadataEntry.text(str(steps_interrupted), 'steps_interrupted')]
)
self.name = check.opt_str_param(name, 'name', DEFAULT_MODE_NAME)
self.resource_defs = check.opt_dict_param(
resource_defs, 'resource_defs', key_type=str, value_type=ResourceDefinition
)
self.loggers = (
check.opt_dict_param(
logger_defs, 'logger_defs', key_type=str, value_type=LoggerDefinition
)
or default_loggers()
)
self.system_storage_defs = check.list_param(
system_storage_defs if system_storage_defs else default_system_storage_defs,
'system_storage_defs',
of_type=SystemStorageDefinition,
)
self.executor_defs = check.list_param(
executor_defs if executor_defs else default_executors,
'executor_defs',
of_type=ExecutorDefinition,
)
self.description = check.opt_str_param(description, 'description')
def _build_sub_pipeline(pipeline_def, solid_names):
'''
Build a pipeline which is a subset of another pipeline.
Only includes the solids which are in solid_names.
'''
from dagster.core.definitions.handle import ExecutionTargetHandle
check.inst_param(pipeline_def, 'pipeline_def', PipelineDefinition)
check.list_param(solid_names, 'solid_names', of_type=str)
solid_name_set = set(solid_names)
solids = list(map(pipeline_def.solid_named, solid_names))
deps = {_dep_key_of(solid): {} for solid in solids}
for solid in solids:
for input_handle in solid.input_handles():
if pipeline_def.dependency_structure.has_singular_dep(input_handle):
output_handle = pipeline_def.dependency_structure.get_singular_dep(input_handle)
if output_handle.solid.name in solid_name_set:
deps[_dep_key_of(solid)][input_handle.input_def.name] = DependencyDefinition(
solid=output_handle.solid.name, output=output_handle.output_def.name
)
elif pipeline_def.dependency_structure.has_multi_deps(input_handle):
output_handles = pipeline_def.dependency_structure.get_multi_deps(input_handle)
deps[_dep_key_of(solid)][input_handle.input_def.name] = MultiDependencyDefinition(
def __new__(cls, run_id, logging_tags, loggers):
return super(DagsterLogManager, cls).__new__(
cls,
run_id=check.str_param(run_id, 'run_id'),
logging_tags=check.dict_param(logging_tags, 'logging_tags'),
loggers=check.list_param(loggers, 'loggers', of_type=logging.Logger),
)
def __init__(self, *args, **kwargs):
self.step_keys = check.list_param(kwargs.pop('step_keys'), 'step_keys', str)
super(DagsterExecutionStepNotFoundError, self).__init__(*args, **kwargs)
def __new__(cls, step_key, marshalled_inputs, marshalled_outputs):
return super(StepExecution, cls).__new__(
cls,
check.str_param(step_key, 'step_key'),
check.list_param(marshalled_inputs, 'marshalled_inputs', of_type=MarshalledInput),
check.list_param(marshalled_outputs, 'marshalled_outputs', of_type=MarshalledOutput),
)
def __new__(cls, field_names, field_defs):
return super(MissingFieldsErrorData, cls).__new__(
cls,
check.list_param(field_names, 'field_names', of_type=str),
[check_field_param(field_def, 'field_defs') for field_def in field_defs],
)
context_definitions=None,
dependencies=None,
):
'''
Args:
solids (List[SolidDefinition]): Solids in the pipeline
name (str): Name. This is optional, mostly for situations that require ephemeral
pipeline definitions for fast scaffolding or testing.
description (str): Description of the pipline.
context_definitions (Dict[str, PipelineContextDefinition]): See class description.
dependencies: (Dict[str, Dict[str, DependencyDefinition]]): See class description.
'''
self.name = check.opt_str_param(name, 'name', '<>')
self.description = check.opt_str_param(description, 'description')
check.list_param(solids, 'solids')
if context_definitions is None:
context_definitions = _default_pipeline_context_definitions()
self.context_definitions = check.dict_param(
context_definitions,
'context_definitions',
key_type=str,
value_type=PipelineContextDefinition,
)
dependencies = check_opt_two_dim_dict(
dependencies,
'dependencies',
value_type=DependencyDefinition,
)