Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
'SystemComputeExecutionContext must have valid environment_dict',
)
system_compute_context = compute_context.get_system_context()
base_dir = '/tmp/dagstermill/{run_id}/'.format(run_id=compute_context.run_id)
output_notebook_dir = os.path.join(base_dir, 'output_notebooks/')
mkdir_p(output_notebook_dir)
temp_path = os.path.join(
output_notebook_dir, '{prefix}-out.ipynb'.format(prefix=str(uuid.uuid4()))
)
with safe_tempfile_path() as output_log_path:
# Scaffold the registration here
nb = load_notebook_node(notebook_path)
nb_no_parameters = replace_parameters(
system_compute_context,
nb,
get_papermill_parameters(system_compute_context, inputs, output_log_path),
)
intermediate_path = os.path.join(
output_notebook_dir, '{prefix}-inter.ipynb'.format(prefix=str(uuid.uuid4()))
)
write_ipynb(nb_no_parameters, intermediate_path)
with user_code_error_boundary(
DagstermillExecutionError,
lambda: (
'Error occurred during the execution of Dagstermill solid '
'{solid_name}: {notebook_path}'.format(
solid_name=name, notebook_path=notebook_path
def execute_retroactive_scaffold(notebook_path):
nb = load_notebook_node(notebook_path)
new_nb = copy.deepcopy(nb)
import_cell_source = 'import dagstermill'
import_cell = nbformat.v4.new_code_cell(source=import_cell_source)
parameters_cell_source = 'context = dagstermill.get_context()'
parameters_cell = nbformat.v4.new_code_cell(source=parameters_cell_source)
parameters_cell.metadata['tags'] = ['parameters']
new_nb.cells = [import_cell, parameters_cell] + nb.cells
write_ipynb(new_nb, notebook_path)
"""
Returns a Notebook object loaded from the location specified at 'path'.
Args:
path (str): Path to notebook ".ipynb" file.
Returns:
A Notebook object.
"""
if not path.endswith(".ipynb"):
raise PapermillException(
"Notebooks should have an '.ipynb' file extension. Provided path: '%s'", path)
nb = Notebook()
nb.path = path
nb.node = load_notebook_node(path)
return nb
"""
Generate a set of notebooks using the Papermill APIs. The input notebook
must contain a cell with a `parameters` tag. Papermill will generate a set of
notebooks based on the parameters defined in the input yaml
Args:
input_nb_path: string
Path to the source template notebook
yml_parameters_path: string
Path to the yaml spec with parameters
Returns: list
A list of paths to the generated notebooks
"""
y = read_yaml_file(yml_parameters_path)
input_nb = load_notebook_node(input_nb_path)
# Create the cartesian product of the parameters
hp_values = list(product(*y.values()))
# Now recreate a dictionary with the correct keys
hp_dicts = [dict(zip(y.keys(), x)) for x in hp_values]
# For each combination of parameters generate a notebook from the template
output_paths = list()
for params in hp_dicts:
params_str = print_dict_parametes(params)
output_path = input_nb_path.replace(".ipynb", "") + params_str + ".ipynb"
output_nb = parameterize_notebook(
input_nb,
parameters=params
)
Returns
-------
nb : NotebookNode
Executed notebook object
"""
path_parameters = add_builtin_parameters(parameters)
input_path = parameterize_path(input_path, path_parameters)
output_path = parameterize_path(output_path, path_parameters)
logger.info("Input Notebook: %s" % get_pretty_path(input_path))
logger.info("Output Notebook: %s" % get_pretty_path(output_path))
with local_file_io_cwd():
if cwd is not None:
logger.info("Working directory: {}".format(get_pretty_path(cwd)))
nb = load_notebook_node(input_path)
# Parameterize the Notebook.
if parameters:
nb = parameterize_notebook(nb, parameters, report_mode)
nb = prepare_notebook_metadata(nb, input_path, output_path, report_mode)
if not prepare_only:
# Fetch the kernel name if it's not supplied
kernel_name = kernel_name or nb.metadata.kernelspec.name
# Execute the Notebook in `cwd` if it is set
with chdir(cwd):
nb = papermill_engines.execute_notebook_with_engine(
engine_name,
nb,