Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
"""
metadata = info['metadata']
lines = [] # The output that we're accumulating
# Bundle fields
for key in ('bundle_type', 'uuid', 'data_hash', 'state', 'command', 'is_anonymous'):
if not raw:
if key not in info: continue
lines.append(self.key_value_str(key, info.get(key)))
# Owner info
lines.append(self.key_value_str('owner', self.simple_user_str(info['owner'])))
# Metadata fields (standard)
cls = get_bundle_subclass(info['bundle_type'])
for key, value in worksheet_util.get_formatted_metadata(cls, metadata, raw):
lines.append(self.key_value_str(key, value))
# Metadata fields (non-standard)
standard_keys = set(spec.key for spec in cls.METADATA_SPECS)
for key, value in metadata.items():
if key in standard_keys: continue
lines.append(self.key_value_str(key, value))
# Dependencies (both hard dependencies and soft)
def display_dependencies(label, deps):
lines.append(label + ':')
for dep in deps:
child = dep['child_path']
parent = path_util.safe_join(contents_str(dep['parent_name']) + '(' + dep['parent_uuid'] + ')', dep['parent_path'])
lines.append(' %s: %s' % (child, parent))
get_children='children' in include_set,
get_permissions='group_permissions' in include_set,
get_host_worksheets='host_worksheets' in include_set,
ignore_not_found=False,
)
# Create list of bundles in original order
bundles = [bundles_dict[uuid] for uuid in bundle_uuids]
# Build response document
document = BundleSchema(many=True).dump(bundles).data
# Shim in display metadata used by the front-end application
if query_get_bool('include_display_metadata', default=False):
for bundle, data in zip(bundles, document['data']):
bundle_class = get_bundle_subclass(bundle['bundle_type'])
json_api_meta(
data,
{
'editable_metadata_keys': worksheet_util.get_editable_metadata_fields(
bundle_class
),
'metadata_type': worksheet_util.get_metadata_types(bundle_class),
},
)
if 'owner' in include_set:
owner_ids = set(b['owner_id'] for b in bundles if b['owner_id'] is not None)
json_api_include(document, UserSchema(), local.model.get_users(owner_ids))
if 'group_permissions' in include_set:
for bundle in bundles:
new_metadata = new_info['metadata']
if new_output_name:
if old_bundle_uuid == old_output:
new_metadata['name'] = new_output_name
else:
# Just make up a name heuristically
new_metadata['name'] = new_output_name + '-' + old_info['metadata']['name']
# By default, the mimic bundle uses whatever image the old bundle uses
# Preferably it uses the SHA256 image digest, but it may simply copy request_docker_image
# if it is not present
if new_info['bundle_type'] == 'run' and new_metadata.get('docker_image', ''):
# Put docker_image in requested_docker_image if it is present and this is a run bundle
new_metadata['request_docker_image'] = new_metadata['docker_image']
cls = get_bundle_subclass(new_info['bundle_type'])
for spec in cls.METADATA_SPECS:
# Remove automatically generated keys
if spec.generated and spec.key in new_metadata:
del new_metadata[spec.key]
# Override original metadata keys
if spec.key in metadata_override:
new_metadata[spec.key] = metadata_override[spec.key]
# Set up info dict
new_info['metadata'] = new_metadata
new_info['dependencies'] = new_dependencies
if dry_run:
new_info['uuid'] = None
else:
if new_info['bundle_type'] not in ('make', 'run'):
# Check for all necessary permissions
worksheet = local.model.get_worksheet(worksheet_uuid, fetch_items=False)
check_worksheet_has_all_permission(local.model, request.user, worksheet)
worksheet_util.check_worksheet_not_frozen(worksheet)
request.user.check_quota(need_time=True, need_disk=True)
created_uuids = []
for bundle in bundles:
# Prep bundle info for saving into database
# Unfortunately cannot use the `construct` methods because they don't
# provide a uniform interface for constructing bundles for all types
# Hopefully this can all be unified after REST migration is complete
bundle_uuid = bundle.setdefault('uuid', spec_util.generate_uuid())
created_uuids.append(bundle_uuid)
bundle_class = get_bundle_subclass(bundle['bundle_type'])
bundle['owner_id'] = request.user.user_id
if issubclass(bundle_class, UploadedBundle) or query_get_bool('wait_for_upload', False):
bundle['state'] = State.UPLOADING
else:
bundle['state'] = State.CREATED
bundle['is_anonymous'] = worksheet.is_anonymous # inherit worksheet anonymity
bundle.setdefault('metadata', {})['created'] = int(time.time())
for dep in bundle.setdefault('dependencies', []):
dep['child_uuid'] = bundle_uuid
# Create bundle object
bundle = bundle_class(bundle, strict=False)
# Save bundle into model
local.model.save_bundle(bundle)
lone_output = (len(old_inputs) == 0 and old_bundle_uuid == old_output)
downstream_of_inputs = any(dep['parent_uuid'] in downstream for dep in info['dependencies'])
if lone_output or downstream_of_inputs:
# Now create a new bundle that mimics the old bundle.
# Only change the name if the output name is supplied.
new_info = copy.deepcopy(info)
new_metadata = new_info['metadata']
if new_output_name:
if old_bundle_uuid == old_output:
new_metadata['name'] = new_output_name
else:
# Just make up a name heuristically
new_metadata['name'] = new_output_name + '-' + info['metadata']['name']
# Remove all the automatically generated keys
cls = get_bundle_subclass(new_info['bundle_type'])
for spec in cls.METADATA_SPECS:
if spec.generated and spec.key in new_metadata:
new_metadata.pop(spec.key)
# Set the targets
targets = [(dep['child_path'], (dep['parent_uuid'], dep['parent_path'])) for dep in new_dependencies]
if dry_run:
new_bundle_uuid = None
else:
if new_info['bundle_type'] not in ('make', 'run'):
raise UsageError('Can\'t mimic %s since it is not make or run' % old_bundle_uuid)
new_bundle_uuid = self._derive_bundle(new_info['bundle_type'], \
targets, new_info['command'], new_metadata, worksheet_uuid)
new_info['uuid'] = new_bundle_uuid
# Add metadata arguments for UploadedBundle and all of its subclasses.
metadata_keys = set()
metadata_util.add_arguments(UploadedBundle, metadata_keys, parser)
for bundle_type in UPLOADED_TYPES:
bundle_subclass = get_bundle_subclass(bundle_type)
metadata_util.add_arguments(bundle_subclass, metadata_keys, parser)
metadata_util.add_auto_argument(parser)
args = parser.parse_args(argv)
# Check that the upload path exists.
path_util.check_isvalid(path_util.normalize(args.path), 'upload')
# Pull out the upload bundle type from the arguments and validate it.
if args.bundle_type not in UPLOADED_TYPES:
raise UsageError('Invalid bundle type %s (options: [%s])' % (
args.bundle_type, '|'.join(sorted(UPLOADED_TYPES)),
))
bundle_subclass = get_bundle_subclass(args.bundle_type)
metadata = metadata_util.request_missing_data(bundle_subclass, args)
# Type-check the bundle metadata BEFORE uploading the bundle data.
# This optimization will avoid file copies on failed bundle creations.
bundle_subclass.construct(data_hash='', metadata=metadata).validate()
print self.client.upload(args.bundle_type, args.path, metadata, worksheet_uuid)
@staticmethod
def get_bundle_info(uuid):
bundle_info = get_bundle_info(uuid, True, True, True)
if bundle_info is None:
return None
# Set permissions
bundle_info['edit_permission'] = (bundle_info['permission'] == GROUP_OBJECT_PERMISSION_ALL)
# Format permissions into strings
bundle_info['permission_str'] = permission_str(bundle_info['permission'])
for group_permission in bundle_info['group_permissions']:
group_permission['permission_str'] = permission_str(group_permission['permission'])
metadata = bundle_info['metadata']
cls = get_bundle_subclass(bundle_info['bundle_type'])
for key, value in worksheet_util.get_formatted_metadata(cls, metadata):
metadata[key] = value
bundle_info['metadata'] = metadata
bundle_info['editable_metadata_fields'] = worksheet_util.get_editable_metadata_fields(cls)
return bundle_info
def do_upload_command(self, argv, parser):
worksheet_uuid = self.env_model.get_current_worksheet()
help_text = 'bundle_type: [%s]' % ('|'.join(sorted(UPLOADED_TYPES)))
parser.add_argument('bundle_type', help=help_text)
parser.add_argument('path', help='path of the directory to upload')
# Add metadata arguments for UploadedBundle and all of its subclasses.
metadata_keys = set()
metadata_util.add_arguments(UploadedBundle, metadata_keys, parser)
for bundle_type in UPLOADED_TYPES:
bundle_subclass = get_bundle_subclass(bundle_type)
metadata_util.add_arguments(bundle_subclass, metadata_keys, parser)
metadata_util.add_auto_argument(parser)
args = parser.parse_args(argv)
# Check that the upload path exists.
path_util.check_isvalid(path_util.normalize(args.path), 'upload')
# Pull out the upload bundle type from the arguments and validate it.
if args.bundle_type not in UPLOADED_TYPES:
raise UsageError('Invalid bundle type %s (options: [%s])' % (
args.bundle_type, '|'.join(sorted(UPLOADED_TYPES)),
))
bundle_subclass = get_bundle_subclass(args.bundle_type)
metadata = metadata_util.request_missing_data(bundle_subclass, args)
# Type-check the bundle metadata BEFORE uploading the bundle data.
# This optimization will avoid file copies on failed bundle creations.
bundle_subclass.construct(data_hash='', metadata=metadata).validate()
print self.client.upload(args.bundle_type, args.path, metadata, worksheet_uuid)