Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
@solid(input_defs=[InputDefinition('df', DataFrame)])
def df_as_config(_context, df):
assert df.to_dict('list') == {'num1': [1, 3], 'num2': [2, 4]}
called['yup'] = True
@pipeline
def test_pipeline():
return df_as_config()
result = execute_pipeline(
test_pipeline,
{
'solids': {
'df_as_config': {
'inputs': {
'df': {'csv': {'path': script_relative_path('num_pipes.csv'), 'sep': '|'}}
}
}
}
},
)
assert result.success
assert called['yup']
def path_to_tutorial_file(path):
return script_relative_path(os.path.join('../../dagster_examples/intro_tutorial/', path))
def test_loader_from_default_repository_module_yaml():
pipeline = load_pipeline_from_target_info(
PipelineTargetInfo(
module_name=None,
pipeline_name='repo_demo_pipeline',
python_file=None,
fn_name=None,
repository_yaml=script_relative_path('repository_module.yml'),
)
)
assert isinstance(pipeline, PipelineDefinition)
assert pipeline.name == 'repo_demo_pipeline'
def test_tutorial_script_part_four():
check_script(script_relative_path('../../dagster_examples/intro_tutorial/config.py'))
def test_repository_python_file():
python_file = script_relative_path('bar_repo.py')
module = imp.load_source('bar_repo', python_file)
mode_data = create_pipeline_loading_mode_data(
PipelineTargetInfo(
module_name=None,
pipeline_name='foo',
python_file=python_file,
fn_name='define_bar_repo',
repository_yaml=None,
)
)
assert mode_data == PipelineLoadingModeData(
mode=PipelineTargetMode.REPOSITORY,
data=RepositoryData(
entrypoint=LoaderEntrypoint(module, 'bar_repo', 'define_bar_repo', {}),
def test_pipeline_download():
config_object = load_yaml_from_glob_list(
[
script_relative_path('../environments/local_base.yml'),
script_relative_path('../environments/local_fast_download.yml'),
]
)
result = execute_pipeline(define_airline_demo_download_pipeline(), config_object)
assert result.success
'pipeline_name': ('hello_cereal_pipeline',),
'python_file': None,
'module_name': 'dagster_examples.intro_tutorial.repos',
'fn_name': 'define_repo',
},
{
'repository_yaml': None,
'pipeline_name': (),
'python_file': None,
'module_name': 'dagster_examples.intro_tutorial.repos',
'fn_name': 'hello_cereal_pipeline',
},
{
'repository_yaml': None,
'pipeline_name': (),
'python_file': script_relative_path('test_cli_commands.py'),
'module_name': None,
'fn_name': 'define_foo_pipeline',
},
{
'repository_yaml': None,
'pipeline_name': (),
'python_file': script_relative_path('test_cli_commands.py'),
'module_name': None,
'fn_name': 'foo_pipeline',
},
import pytest
from airflow.exceptions import AirflowException
from airflow.utils import timezone
from dagster_airflow.factory import (
make_airflow_dag_containerized_for_handle,
make_airflow_dag_for_handle,
make_airflow_dag_kubernetized_for_handle,
)
from dagster_airflow.test_fixtures import execute_tasks_in_dag
from dagster_airflow_tests.conftest import IMAGE
from dagster import ExecutionTargetHandle
from dagster.utils import load_yaml_from_glob_list, script_relative_path
ENVIRONMENTS_PATH = script_relative_path(
os.path.join(
'..',
'..',
'..',
'.buildkite',
'images',
'docker',
'test_project',
'test_pipelines',
'environments',
)
)
def test_error_dag_python():
pipeline_name = 'demo_error_pipeline'
def test_tutorial_intro_tutorial_hello_world_script():
check_script(script_relative_path('../../dagster_examples/intro_tutorial/hello_world.py'))
solid_subset=['can_fail'],
),
PresetDefinition.from_files(
'failing_2', environment_files=[script_relative_path('pass_env.yaml')]
),
],
)
with pytest.raises(DagsterInvalidDefinitionError):
PresetDefinition.from_files(
'invalid_1', environment_files=[script_relative_path('not_a_file.yaml')]
)
with pytest.raises(DagsterInvariantViolationError):
PresetDefinition.from_files(
'invalid_2', environment_files=[script_relative_path('test_repository_definition.py')]
)
assert execute_pipeline_with_preset(pipeline, 'passing').success
assert execute_pipeline_with_preset(pipeline, 'passing_direct_dict').success
with pytest.raises(DagsterExecutionStepExecutionError):
execute_pipeline_with_preset(pipeline, 'failing_1')
with pytest.raises(DagsterExecutionStepExecutionError):
execute_pipeline_with_preset(pipeline, 'failing_2')
with pytest.raises(DagsterInvariantViolationError, match="Could not find preset"):
execute_pipeline_with_preset(pipeline, 'not_failing')