How to use the datalab.utils.commands.notebook_environment function in datalab

To help you get started, we’ve selected a few datalab examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github googledatalab / pydatalab / datalab / bigquery / commands / _bigquery.py View on Github external
Args:
    args: the argument following '%bigquery create '.
  """
  if args['command'] == 'dataset':
    try:
      datalab.bigquery.Dataset(args['name']).create(friendly_name=args['friendly'],
                                                    description=cell_body)
    except Exception as e:
      print('Failed to create dataset %s: %s' % (args['name'], e))
  else:
    if cell_body is None:
      print('Failed to create %s: no schema specified' % args['name'])
    else:
      try:
        record = datalab.utils.commands.parse_config(cell_body,
                                                     datalab.utils.commands.notebook_environment(),
                                                     as_dict=False)
        schema = datalab.bigquery.Schema(record)
        datalab.bigquery.Table(args['name']).create(schema=schema, overwrite=args['overwrite'])
      except Exception as e:
        print('Failed to create table %s: %s' % (args['name'], e))
github googledatalab / pydatalab / datalab / mlalpha / commands / _mlalpha.py View on Github external
dataset_parser.add_argument('--name',
                              help='The name of the dataset to define.', required=True)
  dataset_parser.set_defaults(func=_dataset)
  module_parser = parser.subcommand('module', 'Define a trainer module.')
  module_parser.add_argument('--name', help='The name of the module.', required=True)
  module_parser.add_argument('--main',
                             help='Whether this module is has main function in the trainer ' +
                                  'package.',
                             action='store_true', default=False)
  module_parser.set_defaults(func=_module)
  package_parser = parser.subcommand('package','Create a trainer package from all modules ' +
                                     'defined with %%mlalpha module.')
  package_parser.add_argument('--name', help='The name of the package.', required=True)
  package_parser.add_argument('--output', help='the output dir of the package.', required=True)
  package_parser.set_defaults(func=_package)
  namespace = datalab.utils.commands.notebook_environment()
  return datalab.utils.commands.handle_magic_line(line, cell, parser, namespace=namespace)
github googledatalab / pydatalab / datalab / mlalpha / commands / _ml.py View on Github external
%s
%s
[Description]:
%s

Cloud Run Command:

%s
%s
[Description]:
%s
""" % (command_local, args_local, docstring_local, command_cloud, args_cloud, docstring_cloud)
      return datalab.utils.commands.render_text(output, preformatted=True)

    env = datalab.utils.commands.notebook_environment()
    func_args = datalab.utils.commands.parse_config(cell, env)
    if args['cloud'] is True:
      return pr.run_func(cloud_func_name, func_args)
    else:
      return pr.run_func(local_func_name, func_args)
github googledatalab / pydatalab / datalab / bigquery / commands / _bigquery.py View on Github external
def _dryrun_cell(args, cell_body):
  """Implements the BigQuery cell magic used to dry run BQ queries.

   The supported syntax is:
   %%bigquery dryrun [-q|--sql ]
   []

  Args:
    args: the argument following '%bigquery dryrun'.
    cell_body: optional contents of the cell interpreted as YAML or JSON.
  Returns:
    The response wrapped in a DryRunStats object
  """
  query = _get_query_argument(args, cell_body, datalab.utils.commands.notebook_environment())

  if args['verbose']:
    print(query.sql)
  result = query.execute_dry_run(dialect=args['dialect'], billing_tier=args['billing'])
  return datalab.bigquery._query_stats.QueryStats(total_bytes=result['totalBytesProcessed'],
                                                  is_cached=result['cacheHit'])
github googledatalab / pydatalab / datalab / mlalpha / commands / _ml.py View on Github external
required=True)
  predict_parser.set_defaults(func=_predict)

  batch_predict_parser = parser.subcommand('batch_predict', 'Batch predict with an ML model.')
  batch_predict_parser.add_argument('--usage',
                                    help='Show usage from the specified prediction package',
                                    action='store_true', default=False)
  batch_predict_parser.add_argument('--cloud',
                                    help='Whether to run prediction in the cloud.',
                                    action='store_true', default=False)
  batch_predict_parser.add_argument('--package',
                                    help='The prediction package to use. Can be a gs or local path.',
                                    required=True)
  batch_predict_parser.set_defaults(func=_batch_predict)

  namespace = datalab.utils.commands.notebook_environment()
  return datalab.utils.commands.handle_magic_line(line, cell, parser, namespace=namespace)
github googledatalab / pydatalab / datalab / mlalpha / commands / _mlalpha.py View on Github external
def _dataset(args, cell):
  if not cell:
    _output_dataset_template(args['name'])
    return
  env = datalab.utils.commands.notebook_environment()
  config = datalab.utils.commands.parse_config(cell, env)
  datalab.utils.commands.validate_config(config, ['source', 'featureset'],
      optional_keys=['format'])
  if config['featureset'] not in env:
    raise Exception('"%s" is not defined.' % config['featureset'])
  featureset_class = env[config['featureset']]
  format = config.get('format', 'csv')
  ds = datalab.mlalpha.DataSet(featureset_class(), config['source'], format=format)
  env[args['name']] = ds
github googledatalab / pydatalab / datalab / mlalpha / commands / _mlalpha.py View on Github external
def _evaluate(args, cell):
  if not cell:
    _output_evaluate_template(args['cloud'])
    return

  env = datalab.utils.commands.notebook_environment()
  config = datalab.utils.commands.parse_config(cell, env)
  datalab.utils.commands.validate_config(config,
     ['preprocessed_eval_data_path', 'metadata_path', 'model_dir', 'output_dir'],
     optional_keys=['output_prediction_name', 'output_score_name'])
  command = '%%mlalpha evaluate'
  if args['cloud']:
    command += ' --cloud'
  command += '\n' + cell
  _output_evaluate_code_template(command, args['cloud'], config['preprocessed_eval_data_path'],
      config['metadata_path'], config['model_dir'], config['output_dir'],
      output_prediction_name=config.get('output_prediction_name', None),
      output_score_name=config.get('output_score_name', None))