Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
script_var=_SCRIPT_VARNAME,
python_decode_script=_PYTHON_DECODE_SCRIPT,
script_path=script_path,
mk_io_dirs=_MK_IO_DIRS)
prepare_env = self._get_prepare_env(script, task_view, inputs, outputs,
mounts)
localization_env = self._get_localization_env(inputs, user_project)
user_environment = self._build_user_environment(envs, inputs, outputs,
mounts)
delocalization_env = self._get_delocalization_env(outputs, user_project)
# Build the list of actions
actions = []
actions.append(
google_v2_pipelines.build_action(
name='logging',
flags='RUN_IN_BACKGROUND',
image_uri=_CLOUD_SDK_IMAGE,
environment=logging_env,
entrypoint='/bin/bash',
commands=['-c', continuous_logging_cmd]))
if job_resources.ssh:
actions.append(
google_v2_pipelines.build_action(
name='ssh',
image_uri=_SSH_IMAGE,
mounts=[mnt_datadisk],
entrypoint='ssh-server',
port_mappings={_DEFAULT_SSH_PORT: _DEFAULT_SSH_PORT},
flags='RUN_IN_BACKGROUND'))
]),
google_v2_pipelines.build_action(
name='delocalization',
image_uri=_CLOUD_SDK_IMAGE,
mounts=[mnt_datadisk],
environment=delocalization_env,
entrypoint='/bin/bash',
commands=[
'-c',
_LOCALIZATION_CMD.format(
log_msg_fn=_LOG_MSG_FN,
recursive_cp_fn=_GSUTIL_RSYNC_FN,
cp_fn=_GSUTIL_CP_FN,
cp_loop=_DELOCALIZATION_LOOP)
]),
google_v2_pipelines.build_action(
name='final_logging',
flags='ALWAYS_RUN',
image_uri=_CLOUD_SDK_IMAGE,
environment=logging_env,
entrypoint='/bin/bash',
commands=['-c', logging_cmd]),
])
assert len(actions) - 2 == user_action
assert len(actions) == final_logging_action
# Prepare the VM (resources) configuration
disks = [
google_v2_pipelines.build_disk(
_DATA_DISK_NAME,
job_resources.disk_size,
def _get_mount_actions(self, mounts, mnt_datadisk):
"""Returns a list of two actions per gcs bucket to mount."""
actions_to_add = []
for mount in mounts:
bucket = mount.value[len('gs://'):]
mount_path = mount.docker_path
actions_to_add.extend([
google_v2_pipelines.build_action(
name='mount-{}'.format(bucket),
flags=['ENABLE_FUSE', 'RUN_IN_BACKGROUND'],
image_uri=_GCSFUSE_IMAGE,
mounts=[mnt_datadisk],
commands=[
'--implicit-dirs', '--foreground', '-o ro', bucket,
os.path.join(providers_util.DATA_MOUNT_POINT, mount_path)
]),
google_v2_pipelines.build_action(
name='mount-wait-{}'.format(bucket),
flags=['ENABLE_FUSE'],
image_uri=_GCSFUSE_IMAGE,
mounts=[mnt_datadisk],
commands=[
'wait',
os.path.join(providers_util.DATA_MOUNT_POINT, mount_path)
environment=logging_env,
entrypoint='/bin/bash',
commands=['-c', continuous_logging_cmd]))
if job_resources.ssh:
actions.append(
google_v2_pipelines.build_action(
name='ssh',
image_uri=_SSH_IMAGE,
mounts=[mnt_datadisk],
entrypoint='ssh-server',
port_mappings={_DEFAULT_SSH_PORT: _DEFAULT_SSH_PORT},
flags='RUN_IN_BACKGROUND'))
actions.append(
google_v2_pipelines.build_action(
name='prepare',
image_uri=_CLOUD_SDK_IMAGE,
mounts=[mnt_datadisk],
environment=prepare_env,
entrypoint='/bin/bash',
commands=['-c', prepare_command]),)
actions.extend(mount_actions)
actions.extend([
google_v2_pipelines.build_action(
name='localization',
image_uri=_CLOUD_SDK_IMAGE,
mounts=[mnt_datadisk],
environment=localization_env,
entrypoint='/bin/bash',
delocalization_env = self._get_delocalization_env(outputs, user_project)
# Build the list of actions
actions = []
actions.append(
google_v2_pipelines.build_action(
name='logging',
flags='RUN_IN_BACKGROUND',
image_uri=_CLOUD_SDK_IMAGE,
environment=logging_env,
entrypoint='/bin/bash',
commands=['-c', continuous_logging_cmd]))
if job_resources.ssh:
actions.append(
google_v2_pipelines.build_action(
name='ssh',
image_uri=_SSH_IMAGE,
mounts=[mnt_datadisk],
entrypoint='ssh-server',
port_mappings={_DEFAULT_SSH_PORT: _DEFAULT_SSH_PORT},
flags='RUN_IN_BACKGROUND'))
actions.append(
google_v2_pipelines.build_action(
name='prepare',
image_uri=_CLOUD_SDK_IMAGE,
mounts=[mnt_datadisk],
environment=prepare_env,
entrypoint='/bin/bash',
commands=['-c', prepare_command]),)
cp_loop=_LOCALIZATION_LOOP)
]),
google_v2_pipelines.build_action(
name='user-command',
image_uri=job_resources.image,
mounts=[mnt_datadisk] + persistent_disk_mounts,
environment=user_environment,
entrypoint='/usr/bin/env',
commands=[
'bash', '-c',
_USER_CMD.format(
tmp_dir=providers_util.TMP_DIR,
working_dir=providers_util.WORKING_DIR,
user_script=script_path)
]),
google_v2_pipelines.build_action(
name='delocalization',
image_uri=_CLOUD_SDK_IMAGE,
mounts=[mnt_datadisk],
environment=delocalization_env,
entrypoint='/bin/bash',
commands=[
'-c',
_LOCALIZATION_CMD.format(
log_msg_fn=_LOG_MSG_FN,
recursive_cp_fn=_GSUTIL_RSYNC_FN,
cp_fn=_GSUTIL_CP_FN,
cp_loop=_DELOCALIZATION_LOOP)
]),
google_v2_pipelines.build_action(
name='final_logging',
flags='ALWAYS_RUN',