How to use the papermill.iorw.load_notebook_node function in papermill

To help you get started, we’ve selected a few papermill examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github dagster-io / dagster / python_modules / dagstermill / dagstermill / solids.py View on Github external
'SystemComputeExecutionContext must have valid environment_dict',
        )

        system_compute_context = compute_context.get_system_context()

        base_dir = '/tmp/dagstermill/{run_id}/'.format(run_id=compute_context.run_id)
        output_notebook_dir = os.path.join(base_dir, 'output_notebooks/')
        mkdir_p(output_notebook_dir)

        temp_path = os.path.join(
            output_notebook_dir, '{prefix}-out.ipynb'.format(prefix=str(uuid.uuid4()))
        )

        with safe_tempfile_path() as output_log_path:
            # Scaffold the registration here
            nb = load_notebook_node(notebook_path)
            nb_no_parameters = replace_parameters(
                system_compute_context,
                nb,
                get_papermill_parameters(system_compute_context, inputs, output_log_path),
            )
            intermediate_path = os.path.join(
                output_notebook_dir, '{prefix}-inter.ipynb'.format(prefix=str(uuid.uuid4()))
            )
            write_ipynb(nb_no_parameters, intermediate_path)

            with user_code_error_boundary(
                DagstermillExecutionError,
                lambda: (
                    'Error occurred during the execution of Dagstermill solid '
                    '{solid_name}: {notebook_path}'.format(
                        solid_name=name, notebook_path=notebook_path
github dagster-io / dagster / python_modules / dagstermill / dagstermill / cli.py View on Github external
def execute_retroactive_scaffold(notebook_path):
    nb = load_notebook_node(notebook_path)
    new_nb = copy.deepcopy(nb)

    import_cell_source = 'import dagstermill'
    import_cell = nbformat.v4.new_code_cell(source=import_cell_source)

    parameters_cell_source = 'context = dagstermill.get_context()'
    parameters_cell = nbformat.v4.new_code_cell(source=parameters_cell_source)
    parameters_cell.metadata['tags'] = ['parameters']

    new_nb.cells = [import_cell, parameters_cell] + nb.cells
    write_ipynb(new_nb, notebook_path)
github nteract / papermill / papermill / api.py View on Github external
"""
    Returns a Notebook object loaded from the location specified at 'path'.

    Args:
        path (str): Path to notebook ".ipynb" file.

    Returns:
        A Notebook object.
    """
    if not path.endswith(".ipynb"):
        raise PapermillException(
            "Notebooks should have an '.ipynb' file extension. Provided path: '%s'", path)

    nb = Notebook()
    nb.path = path
    nb.node = load_notebook_node(path)
    return nb
github kubeflow-kale / kale / backend / kale / notebook_gen.py View on Github external
"""
    Generate a set of notebooks using the Papermill APIs. The input notebook
    must contain a cell with a `parameters` tag. Papermill will generate a set of
    notebooks based on the parameters defined in the input yaml
    Args:
        input_nb_path: string
                        Path to the source template notebook
        yml_parameters_path: string
                                Path to the yaml spec with parameters

    Returns: list
                A list of paths to the generated notebooks

    """
    y = read_yaml_file(yml_parameters_path)
    input_nb = load_notebook_node(input_nb_path)

    # Create the cartesian product of the parameters
    hp_values = list(product(*y.values()))

    # Now recreate a dictionary with the correct keys
    hp_dicts = [dict(zip(y.keys(), x)) for x in hp_values]

    # For each combination of parameters generate a notebook from the template
    output_paths = list()
    for params in hp_dicts:
        params_str = print_dict_parametes(params)
        output_path = input_nb_path.replace(".ipynb", "") + params_str + ".ipynb"
        output_nb = parameterize_notebook(
            input_nb,
            parameters=params
        )
github nteract / papermill / papermill / execute.py View on Github external
Returns
    -------
    nb : NotebookNode
       Executed notebook object
    """
    path_parameters = add_builtin_parameters(parameters)
    input_path = parameterize_path(input_path, path_parameters)
    output_path = parameterize_path(output_path, path_parameters)

    logger.info("Input Notebook:  %s" % get_pretty_path(input_path))
    logger.info("Output Notebook: %s" % get_pretty_path(output_path))
    with local_file_io_cwd():
        if cwd is not None:
            logger.info("Working directory: {}".format(get_pretty_path(cwd)))

        nb = load_notebook_node(input_path)

        # Parameterize the Notebook.
        if parameters:
            nb = parameterize_notebook(nb, parameters, report_mode)

        nb = prepare_notebook_metadata(nb, input_path, output_path, report_mode)

        if not prepare_only:
            # Fetch the kernel name if it's not supplied
            kernel_name = kernel_name or nb.metadata.kernelspec.name

            # Execute the Notebook in `cwd` if it is set
            with chdir(cwd):
                nb = papermill_engines.execute_notebook_with_engine(
                    engine_name,
                    nb,