Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def get_named_thing(llist, name):
for cn in llist:
if cn['name'] == name:
return cn
check.failed('not found')
def copy_intermediate_from_prev_run(self, context, previous_run_id, step_output_handle):
check.failed('not implemented in in memory')
def _format_config_item(config, current_indent=0):
printer = IndentingBlockPrinter(indent_level=2, current_indent=current_indent)
if isinstance(config, dict):
return _format_config_subdict(config, printer.current_indent)
elif isinstance(config, list):
return _format_config_sublist(config, printer.current_indent)
elif isinstance(config, bool):
return repr(config).lower()
else:
return repr(config).replace('\'', '"')
check.dict_param(config, 'config', key_type=str)
if not isinstance(config, dict):
check.failed('Expected a dict to format as config, got: {item}'.format(item=repr(config)))
return _format_config_subdict(config)
elif config_type.is_selector:
errors = validate_selector_config_value(config_type, config_value, stack)
elif config_type.is_composite:
errors = validate_composite_config_value(config_type, config_value, stack)
elif config_type.is_list:
errors = validate_list_value(config_type, config_value, stack)
elif config_type.is_nullable:
errors = (
[]
if config_value is None
else _validate_config(config_type.inner_type, config_value, stack)
)
elif config_type.is_enum:
errors = validate_enum_value(config_type, config_value, stack)
else:
check.failed('Unsupported type {name}'.format(name=config_type.name))
for error in errors:
yield error
def dataframe_output_schema(_context, file_type, file_options, pandas_df):
check.str_param(file_type, 'file_type')
check.dict_param(file_options, 'file_options')
check.inst_param(pandas_df, 'pandas_df', DataFrame)
if file_type == 'csv':
path = file_options['path']
pandas_df.to_csv(path, index=False, **dict_without_keys(file_options, 'path'))
elif file_type == 'parquet':
pandas_df.to_parquet(file_options['path'])
elif file_type == 'table':
pandas_df.to_csv(file_options['path'], sep='\t', index=False)
else:
check.failed('Unsupported file_type {file_type}'.format(file_type=file_type))
return Materialization.file(file_options['path'])
def construct_arg_dicts(input_list):
structured_flags = structure_flags(input_list)
if structured_flags is None:
return {}
if structured_flags.single_argument or structured_flags.named_arguments:
check.failed('only supporting named key arguments right now')
input_arg_dicts = defaultdict(lambda: {})
for nka in structured_flags.named_key_arguments:
input_arg_dicts[nka.name][nka.key] = nka.value
return input_arg_dicts
def execute_plan_iterator(execution_plan, pipeline_run, environment_dict=None, instance=None):
check.inst_param(execution_plan, 'execution_plan', ExecutionPlan)
check.inst_param(pipeline_run, 'pipeline_run', PipelineRun)
environment_dict = check.opt_dict_param(environment_dict, 'environment_dict')
instance = check.inst_param(instance, 'instance', DagsterInstance)
with scoped_pipeline_context(
execution_plan.pipeline_def, environment_dict, pipeline_run, instance
) as pipeline_context:
return _steps_execution_iterator(
pipeline_context, execution_plan=execution_plan, pipeline_run=pipeline_run
)
check.failed('Unexpected state, should be unreachable')
def _resolve_config_schema_type(dagster_type):
# This replicates a subset of resolve_to_config_type
# Including resolve_to_config_type directly has a nasty circular
# dependency.
if isinstance(dagster_type, ConfigType):
return dagster_type
if BuiltinEnum.contains(dagster_type):
return ConfigType.from_builtin_enum(dagster_type)
elif isinstance(dagster_type, WrappingListType):
return List(dagster_type.inner_type)
elif isinstance(dagster_type, WrappingNullableType):
return Nullable(dagster_type.inner_type)
check.failed('should not reach. got {dagster_type}'.format(dagster_type=dagster_type))
def executor_def_from_config(mode_definition, environment_config):
for executor_def in mode_definition.executor_defs:
if executor_def.name == environment_config.execution.execution_engine_name:
return executor_def
check.failed(
'Could not find executor {}. Should have be caught by config system'.format(
environment_config.execution.execution_engine_name
)