Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
outputs=None,
outputs_recursive=None,
wait=False):
envs = envs or {}
labels = labels or {}
inputs = inputs or {}
inputs_recursive = inputs_recursive or {}
outputs = outputs or {}
outputs_recursive = outputs_recursive or {}
labels['test-token'] = test_setup.TEST_TOKEN
labels['test-name'] = test_setup.TEST_NAME
logging = param_util.build_logging_param(test.LOGGING)
job_resources = job_model.Resources(
image='ubuntu', logging=logging, zones=['us-central1-*'])
env_data = {job_model.EnvParam(k, v) for (k, v) in envs.items()}
label_data = {job_model.LabelParam(k, v) for (k, v) in labels.items()}
input_file_param_util = param_util.InputFileParamUtil('input')
input_data = set()
for (recursive, items) in ((False, inputs.items()),
(True, inputs_recursive.items())):
for (name, value) in items:
name = input_file_param_util.get_variable_name(name)
input_data.add(input_file_param_util.make_param(name, value, recursive))
output_file_param_util = param_util.OutputFileParamUtil('output')
output_data = set()
for (recursive, items) in ((False, outputs.items()),
job_params = {
'envs': env_data,
'inputs': input_data,
'outputs': output_data,
'labels': label_data,
}
task_descriptors = [
job_model.TaskDescriptor({
'task-id': None
}, {
'envs': set(),
'labels': set(),
'inputs': set(),
'outputs': set(),
}, job_model.Resources())
]
return dsub.run(
get_dsub_provider(),
job_resources,
job_params,
task_descriptors,
name=job_name,
command=command,
wait=wait,
disable_warning=True)
job-id, task-id, user-id, and job-name.
"""
if not job_resources.logging:
return
for task_descriptor in task_descriptors:
logging_uri = provider_base.format_logging_uri(
job_resources.logging.uri, job_metadata, task_descriptor.task_metadata)
logging_path = job_model.LoggingParam(logging_uri,
job_resources.logging.file_provider)
if task_descriptor.task_resources:
task_descriptor.task_resources = task_descriptor.task_resources._replace(
logging_path=logging_path)
else:
task_descriptor.task_resources = job_model.Resources(
logging_path=logging_path)
params['envs'] = cls._env_params_from_dict(job.get('envs', {}))
params['inputs'] = cls._input_file_params_from_dict(
job.get('inputs', {}), False)
params['outputs'] = cls._output_file_params_from_dict(
job.get('outputs', {}), False)
if job.get('task-id') is None:
job_params = params
task_metadata = {'task-id': None}
task_params = {}
else:
job_params = {}
task_metadata = {'task-id': str(job.get('task-id'))}
task_params = params
task_resources = Resources(logging_path=job.get('logging'))
task_descriptors = [
TaskDescriptor.get_complete_descriptor(task_metadata, task_params,
task_resources)
]
return JobDescriptor.get_complete_descriptor(
job_metadata, job_params, job_resources, task_descriptors)
scopes=None,
keep_alive=None,
cpu_platform=None,
network=None,
subnetwork=None,
use_private_address=None,
accelerator_type=None,
accelerator_count=0,
nvidia_driver_version=None,
timeout=None,
log_interval=None,
ssh=None,
enable_stackdriver_monitoring=None,
max_retries=None,
max_preemptible_attempts=None):
return super(Resources, cls).__new__(
cls, min_cores, min_ram, machine_type, disk_size, disk_type,
boot_disk_size, preemptible, image, logging, logging_path, regions,
zones, service_account, scopes, keep_alive, cpu_platform, network,
subnetwork, use_private_address, accelerator_type, accelerator_count,
nvidia_driver_version, timeout, log_interval, ssh,
enable_stackdriver_monitoring, max_retries, max_preemptible_attempts)
args.tasks, args.retries, input_file_param_util, output_file_param_util)
# Validate job data + task data
_validate_job_and_task_arguments(job_params, task_descriptors)
else:
# Create the implicit task
task_metadata = {'task-id': None}
if args.retries:
task_metadata['task-attempt'] = 1
task_descriptors = [
job_model.TaskDescriptor(task_metadata, {
'labels': set(),
'envs': set(),
'inputs': set(),
'outputs': set()
}, job_model.Resources())
]
return run(
provider_base.get_provider(args, resources),
_get_job_resources(args),
job_params,
task_descriptors,
name=args.name,
dry_run=args.dry_run,
command=args.command,
script=args.script,
user=args.user,
user_project=args.user_project,
wait=args.wait,
retries=args.retries,
max_preemptible_attempts=args.preemptible,