Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def load_target_info_from_cli_args(cli_args):
check.dict_param(cli_args, 'cli_args')
if all_none(cli_args):
cli_args['repository_yaml'] = 'repository.yml'
return RepositoryTargetInfo(**cli_args)
def _log(self, method, orig_message, message_props):
check.str_param(method, 'method')
check.str_param(orig_message, 'orig_message')
check.dict_param(message_props, 'message_props')
check.invariant(
'extra' not in message_props, 'do not allow until explicit support is handled'
)
check.invariant(
'exc_info' not in message_props, 'do not allow until explicit support is handled'
)
check.invariant('orig_message' not in message_props, 'orig_message reserved value')
check.invariant('message' not in message_props, 'message reserved value')
check.invariant('log_message_id' not in message_props, 'log_message_id reserved value')
check.invariant('log_timestamp' not in message_props, 'log_timestamp reserved value')
log_message_id = str(uuid.uuid4())
log_timestamp = datetime.datetime.utcnow().isoformat()
def __new__(
cls, pipeline_def, mode_def, executor_def, pipeline_run, environment_config, executor_config
):
return super(InitExecutorContext, cls).__new__(
cls,
pipeline_def=check.inst_param(pipeline_def, 'pipeline_def', PipelineDefinition),
mode_def=check.inst_param(mode_def, 'mode_def', ModeDefinition),
executor_def=check.inst_param(executor_def, 'executor_def', ExecutorDefinition),
pipeline_run=check.inst_param(pipeline_run, 'pipeline_run', PipelineRun),
environment_config=check.inst_param(
environment_config, 'environment_config', EnvironmentConfig
),
executor_config=check.dict_param(executor_config, executor_config, key_type=str),
)
def dagster_event_from_dict(event_dict, pipeline_name):
check.dict_param(event_dict, 'event_dict', key_type=str)
check.str_param(pipeline_name, 'pipeline_name')
# Get event_type
event_type = _handled_events().get(event_dict['__typename'])
if not event_type:
raise Exception('unhandled event type %s' % event_dict['__typename'])
# Get event_specific_data
event_specific_data = None
if event_type == DagsterEventType.STEP_OUTPUT:
event_specific_data = StepOutputData(
step_output_handle=StepOutputHandle(
event_dict['step']['key'], event_dict['outputName']
),
type_check_data=TypeCheckData(
success=event_dict['typeCheck']['success'],
previous_run_id,
step_keys_to_execute,
):
missing_steps = [step_key for step_key in step_keys_to_execute if step_key not in step_dict]
if missing_steps:
raise DagsterExecutionStepNotFoundError(
'Execution plan does not contain step{plural}: {steps}'.format(
plural='s' if len(missing_steps) > 1 else '', steps=', '.join(missing_steps)
),
step_keys=missing_steps,
)
return super(ExecutionPlan, cls).__new__(
cls,
pipeline_def=check.inst_param(pipeline_def, 'pipeline_def', PipelineDefinition),
step_dict=check.dict_param(
step_dict, 'step_dict', key_type=str, value_type=ExecutionStep
),
deps=check.dict_param(deps, 'deps', key_type=str, value_type=set),
steps=list(step_dict.values()),
artifacts_persisted=check.bool_param(artifacts_persisted, 'artifacts_persisted'),
previous_run_id=check.opt_str_param(previous_run_id, 'previous_run_id'),
step_keys_to_execute=check.list_param(
step_keys_to_execute, 'step_keys_to_execute', of_type=str
),
def check_opt_two_dim_dict(ddict, param_name, key_type=None, value_type=None):
ddict = check.opt_dict_param(ddict, param_name, key_type=key_type, value_type=dict)
for sub_dict in ddict.values():
check.dict_param(sub_dict, 'sub_dict', key_type=key_type, value_type=value_type)
return ddict
def _gather_all_types(solids, context_definitions, environment_type):
check.list_param(solids, 'solids', SolidDefinition)
check.dict_param(
context_definitions,
'context_definitions',
key_type=str,
value_type=PipelineContextDefinition,
)
check.inst_param(environment_type, 'environment_type', types.DagsterType)
for solid in solids:
for dagster_type in solid.iterate_types():
yield dagster_type
for context_definition in context_definitions.values():
if context_definition.config_field:
for dagster_type in context_definition.config_field.dagster_type.iterate_types():
yield dagster_type
def __init__(self, run_id, logging_tags, loggers):
self.run_id = check.str_param(run_id, 'run_id')
self.logging_tags = check.dict_param(logging_tags, 'logging_tags')
self.loggers = check.list_param(loggers, 'loggers', of_type=logging.Logger)
def materialize_runtime_value(self, config_spec, runtime_value):
check.dict_param(config_spec, 'config_spec')
selector_key, selector_value = list(config_spec.items())[0]
if selector_key == 'json':
json_file_path = selector_value['path']
json_value = json.dumps({'value': runtime_value})
with open(json_file_path, 'w') as ff:
ff.write(json_value)
else:
check.failed(
'Unsupported selector key: {selector_key}'.format(selector_key=selector_key)
)
def define_resource_dictionary_cls(name, resources):
check.str_param(name, 'name')
check.dict_param(resources, 'resources', key_type=str, value_type=ResourceDefinition)
fields = {}
for resource_name, resource in resources.items():
if resource.config_field:
fields[resource_name] = Field(
SystemNamedDict(name + '.' + resource_name, {'config': resource.config_field})
)
return SystemNamedDict(name=name, fields=fields)