Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
Args:
args: the argument following '%bigquery create '.
"""
if args['command'] == 'dataset':
try:
datalab.bigquery.Dataset(args['name']).create(friendly_name=args['friendly'],
description=cell_body)
except Exception as e:
print('Failed to create dataset %s: %s' % (args['name'], e))
else:
if cell_body is None:
print('Failed to create %s: no schema specified' % args['name'])
else:
try:
record = datalab.utils.commands.parse_config(cell_body,
datalab.utils.commands.notebook_environment(),
as_dict=False)
schema = datalab.bigquery.Schema(record)
datalab.bigquery.Table(args['name']).create(schema=schema, overwrite=args['overwrite'])
except Exception as e:
print('Failed to create table %s: %s' % (args['name'], e))
dataset_parser.add_argument('--name',
help='The name of the dataset to define.', required=True)
dataset_parser.set_defaults(func=_dataset)
module_parser = parser.subcommand('module', 'Define a trainer module.')
module_parser.add_argument('--name', help='The name of the module.', required=True)
module_parser.add_argument('--main',
help='Whether this module is has main function in the trainer ' +
'package.',
action='store_true', default=False)
module_parser.set_defaults(func=_module)
package_parser = parser.subcommand('package','Create a trainer package from all modules ' +
'defined with %%mlalpha module.')
package_parser.add_argument('--name', help='The name of the package.', required=True)
package_parser.add_argument('--output', help='the output dir of the package.', required=True)
package_parser.set_defaults(func=_package)
namespace = datalab.utils.commands.notebook_environment()
return datalab.utils.commands.handle_magic_line(line, cell, parser, namespace=namespace)
%s
%s
[Description]:
%s
Cloud Run Command:
%s
%s
[Description]:
%s
""" % (command_local, args_local, docstring_local, command_cloud, args_cloud, docstring_cloud)
return datalab.utils.commands.render_text(output, preformatted=True)
env = datalab.utils.commands.notebook_environment()
func_args = datalab.utils.commands.parse_config(cell, env)
if args['cloud'] is True:
return pr.run_func(cloud_func_name, func_args)
else:
return pr.run_func(local_func_name, func_args)
def _dryrun_cell(args, cell_body):
"""Implements the BigQuery cell magic used to dry run BQ queries.
The supported syntax is:
%%bigquery dryrun [-q|--sql ]
[]
Args:
args: the argument following '%bigquery dryrun'.
cell_body: optional contents of the cell interpreted as YAML or JSON.
Returns:
The response wrapped in a DryRunStats object
"""
query = _get_query_argument(args, cell_body, datalab.utils.commands.notebook_environment())
if args['verbose']:
print(query.sql)
result = query.execute_dry_run(dialect=args['dialect'], billing_tier=args['billing'])
return datalab.bigquery._query_stats.QueryStats(total_bytes=result['totalBytesProcessed'],
is_cached=result['cacheHit'])
required=True)
predict_parser.set_defaults(func=_predict)
batch_predict_parser = parser.subcommand('batch_predict', 'Batch predict with an ML model.')
batch_predict_parser.add_argument('--usage',
help='Show usage from the specified prediction package',
action='store_true', default=False)
batch_predict_parser.add_argument('--cloud',
help='Whether to run prediction in the cloud.',
action='store_true', default=False)
batch_predict_parser.add_argument('--package',
help='The prediction package to use. Can be a gs or local path.',
required=True)
batch_predict_parser.set_defaults(func=_batch_predict)
namespace = datalab.utils.commands.notebook_environment()
return datalab.utils.commands.handle_magic_line(line, cell, parser, namespace=namespace)
def _dataset(args, cell):
if not cell:
_output_dataset_template(args['name'])
return
env = datalab.utils.commands.notebook_environment()
config = datalab.utils.commands.parse_config(cell, env)
datalab.utils.commands.validate_config(config, ['source', 'featureset'],
optional_keys=['format'])
if config['featureset'] not in env:
raise Exception('"%s" is not defined.' % config['featureset'])
featureset_class = env[config['featureset']]
format = config.get('format', 'csv')
ds = datalab.mlalpha.DataSet(featureset_class(), config['source'], format=format)
env[args['name']] = ds
def _evaluate(args, cell):
if not cell:
_output_evaluate_template(args['cloud'])
return
env = datalab.utils.commands.notebook_environment()
config = datalab.utils.commands.parse_config(cell, env)
datalab.utils.commands.validate_config(config,
['preprocessed_eval_data_path', 'metadata_path', 'model_dir', 'output_dir'],
optional_keys=['output_prediction_name', 'output_score_name'])
command = '%%mlalpha evaluate'
if args['cloud']:
command += ' --cloud'
command += '\n' + cell
_output_evaluate_code_template(command, args['cloud'], config['preprocessed_eval_data_path'],
config['metadata_path'], config['model_dir'], config['output_dir'],
output_prediction_name=config.get('output_prediction_name', None),
output_score_name=config.get('output_score_name', None))