How to use the codalab.bundles.get_bundle_subclass function in codalab

To help you get started, we’ve selected a few codalab examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github codalab / codalab-worksheets / codalab / lib / bundle_cli.py View on Github external
"""

        metadata = info['metadata']
        lines = []  # The output that we're accumulating

        # Bundle fields
        for key in ('bundle_type', 'uuid', 'data_hash', 'state', 'command', 'is_anonymous'):
            if not raw:
                if key not in info: continue
            lines.append(self.key_value_str(key, info.get(key)))

        # Owner info
        lines.append(self.key_value_str('owner', self.simple_user_str(info['owner'])))

        # Metadata fields (standard)
        cls = get_bundle_subclass(info['bundle_type'])
        for key, value in worksheet_util.get_formatted_metadata(cls, metadata, raw):
            lines.append(self.key_value_str(key, value))

        # Metadata fields (non-standard)
        standard_keys = set(spec.key for spec in cls.METADATA_SPECS)
        for key, value in metadata.items():
            if key in standard_keys: continue
            lines.append(self.key_value_str(key, value))

        # Dependencies (both hard dependencies and soft)
        def display_dependencies(label, deps):
            lines.append(label + ':')
            for dep in deps:
                child = dep['child_path']
                parent = path_util.safe_join(contents_str(dep['parent_name']) + '(' + dep['parent_uuid'] + ')', dep['parent_path'])
                lines.append('  %s: %s' % (child, parent))
github codalab / codalab-worksheets / codalab / rest / bundles.py View on Github external
get_children='children' in include_set,
        get_permissions='group_permissions' in include_set,
        get_host_worksheets='host_worksheets' in include_set,
        ignore_not_found=False,
    )

    # Create list of bundles in original order
    bundles = [bundles_dict[uuid] for uuid in bundle_uuids]

    # Build response document
    document = BundleSchema(many=True).dump(bundles).data

    # Shim in display metadata used by the front-end application
    if query_get_bool('include_display_metadata', default=False):
        for bundle, data in zip(bundles, document['data']):
            bundle_class = get_bundle_subclass(bundle['bundle_type'])
            json_api_meta(
                data,
                {
                    'editable_metadata_keys': worksheet_util.get_editable_metadata_fields(
                        bundle_class
                    ),
                    'metadata_type': worksheet_util.get_metadata_types(bundle_class),
                },
            )

    if 'owner' in include_set:
        owner_ids = set(b['owner_id'] for b in bundles if b['owner_id'] is not None)
        json_api_include(document, UserSchema(), local.model.get_users(owner_ids))

    if 'group_permissions' in include_set:
        for bundle in bundles:
github codalab / codalab-worksheets / codalab / lib / bundle_util.py View on Github external
new_metadata = new_info['metadata']
            if new_output_name:
                if old_bundle_uuid == old_output:
                    new_metadata['name'] = new_output_name
                else:
                    # Just make up a name heuristically
                    new_metadata['name'] = new_output_name + '-' + old_info['metadata']['name']

            # By default, the mimic bundle uses whatever image the old bundle uses
            # Preferably it uses the SHA256 image digest, but it may simply copy request_docker_image
            # if it is not present
            if new_info['bundle_type'] == 'run' and new_metadata.get('docker_image', ''):
                # Put docker_image in requested_docker_image if it is present and this is a run bundle
                new_metadata['request_docker_image'] = new_metadata['docker_image']

            cls = get_bundle_subclass(new_info['bundle_type'])
            for spec in cls.METADATA_SPECS:
                # Remove automatically generated keys
                if spec.generated and spec.key in new_metadata:
                    del new_metadata[spec.key]
                # Override original metadata keys
                if spec.key in metadata_override:
                    new_metadata[spec.key] = metadata_override[spec.key]

            # Set up info dict
            new_info['metadata'] = new_metadata
            new_info['dependencies'] = new_dependencies

            if dry_run:
                new_info['uuid'] = None
            else:
                if new_info['bundle_type'] not in ('make', 'run'):
github codalab / codalab-worksheets / codalab / rest / bundles.py View on Github external
# Check for all necessary permissions
    worksheet = local.model.get_worksheet(worksheet_uuid, fetch_items=False)
    check_worksheet_has_all_permission(local.model, request.user, worksheet)
    worksheet_util.check_worksheet_not_frozen(worksheet)
    request.user.check_quota(need_time=True, need_disk=True)

    created_uuids = []
    for bundle in bundles:
        # Prep bundle info for saving into database
        # Unfortunately cannot use the `construct` methods because they don't
        # provide a uniform interface for constructing bundles for all types
        # Hopefully this can all be unified after REST migration is complete
        bundle_uuid = bundle.setdefault('uuid', spec_util.generate_uuid())
        created_uuids.append(bundle_uuid)
        bundle_class = get_bundle_subclass(bundle['bundle_type'])
        bundle['owner_id'] = request.user.user_id
        if issubclass(bundle_class, UploadedBundle) or query_get_bool('wait_for_upload', False):
            bundle['state'] = State.UPLOADING
        else:
            bundle['state'] = State.CREATED
        bundle['is_anonymous'] = worksheet.is_anonymous  # inherit worksheet anonymity
        bundle.setdefault('metadata', {})['created'] = int(time.time())
        for dep in bundle.setdefault('dependencies', []):
            dep['child_uuid'] = bundle_uuid

        # Create bundle object
        bundle = bundle_class(bundle, strict=False)

        # Save bundle into model
        local.model.save_bundle(bundle)
github codalab / codalab-worksheets / codalab / client / local_bundle_client.py View on Github external
lone_output = (len(old_inputs) == 0 and old_bundle_uuid == old_output)
            downstream_of_inputs = any(dep['parent_uuid'] in downstream for dep in info['dependencies'])
            if lone_output or downstream_of_inputs:
                # Now create a new bundle that mimics the old bundle.
                # Only change the name if the output name is supplied.
                new_info = copy.deepcopy(info)
                new_metadata = new_info['metadata']
                if new_output_name:
                    if old_bundle_uuid == old_output:
                        new_metadata['name'] = new_output_name
                    else:
                        # Just make up a name heuristically
                        new_metadata['name'] = new_output_name + '-' + info['metadata']['name']

                # Remove all the automatically generated keys
                cls = get_bundle_subclass(new_info['bundle_type'])
                for spec in cls.METADATA_SPECS:
                    if spec.generated and spec.key in new_metadata:
                        new_metadata.pop(spec.key)

                # Set the targets
                targets = [(dep['child_path'], (dep['parent_uuid'], dep['parent_path'])) for dep in new_dependencies]

                if dry_run:
                    new_bundle_uuid = None
                else:
                    if new_info['bundle_type'] not in ('make', 'run'):
                        raise UsageError('Can\'t mimic %s since it is not make or run' % old_bundle_uuid)
                    new_bundle_uuid = self._derive_bundle(new_info['bundle_type'], \
                        targets, new_info['command'], new_metadata, worksheet_uuid)

                new_info['uuid'] = new_bundle_uuid
github codalab / codalab-worksheets / build / lib / codalab / lib / bundle_cli.py View on Github external
# Add metadata arguments for UploadedBundle and all of its subclasses.
        metadata_keys = set()
        metadata_util.add_arguments(UploadedBundle, metadata_keys, parser)
        for bundle_type in UPLOADED_TYPES:
            bundle_subclass = get_bundle_subclass(bundle_type)
            metadata_util.add_arguments(bundle_subclass, metadata_keys, parser)
        metadata_util.add_auto_argument(parser)
        args = parser.parse_args(argv)
        # Check that the upload path exists.
        path_util.check_isvalid(path_util.normalize(args.path), 'upload')
        # Pull out the upload bundle type from the arguments and validate it.
        if args.bundle_type not in UPLOADED_TYPES:
            raise UsageError('Invalid bundle type %s (options: [%s])' % (
              args.bundle_type, '|'.join(sorted(UPLOADED_TYPES)),
            ))
        bundle_subclass = get_bundle_subclass(args.bundle_type)
        metadata = metadata_util.request_missing_data(bundle_subclass, args)
        # Type-check the bundle metadata BEFORE uploading the bundle data.
        # This optimization will avoid file copies on failed bundle creations.
        bundle_subclass.construct(data_hash='', metadata=metadata).validate()
        print self.client.upload(args.bundle_type, args.path, metadata, worksheet_uuid)
github codalab / codalab-worksheets / codalab / rest / legacy.py View on Github external
    @staticmethod
    def get_bundle_info(uuid):
        bundle_info = get_bundle_info(uuid, True, True, True)

        if bundle_info is None:
            return None
        # Set permissions
        bundle_info['edit_permission'] = (bundle_info['permission'] == GROUP_OBJECT_PERMISSION_ALL)
        # Format permissions into strings
        bundle_info['permission_str'] = permission_str(bundle_info['permission'])
        for group_permission in bundle_info['group_permissions']:
            group_permission['permission_str'] = permission_str(group_permission['permission'])

        metadata = bundle_info['metadata']

        cls = get_bundle_subclass(bundle_info['bundle_type'])
        for key, value in worksheet_util.get_formatted_metadata(cls, metadata):
            metadata[key] = value

        bundle_info['metadata'] = metadata
        bundle_info['editable_metadata_fields'] = worksheet_util.get_editable_metadata_fields(cls)

        return bundle_info
github codalab / codalab-worksheets / build / lib / codalab / lib / bundle_cli.py View on Github external
def do_upload_command(self, argv, parser):
        worksheet_uuid = self.env_model.get_current_worksheet()
        help_text = 'bundle_type: [%s]' % ('|'.join(sorted(UPLOADED_TYPES)))
        parser.add_argument('bundle_type', help=help_text)
        parser.add_argument('path', help='path of the directory to upload')
        # Add metadata arguments for UploadedBundle and all of its subclasses.
        metadata_keys = set()
        metadata_util.add_arguments(UploadedBundle, metadata_keys, parser)
        for bundle_type in UPLOADED_TYPES:
            bundle_subclass = get_bundle_subclass(bundle_type)
            metadata_util.add_arguments(bundle_subclass, metadata_keys, parser)
        metadata_util.add_auto_argument(parser)
        args = parser.parse_args(argv)
        # Check that the upload path exists.
        path_util.check_isvalid(path_util.normalize(args.path), 'upload')
        # Pull out the upload bundle type from the arguments and validate it.
        if args.bundle_type not in UPLOADED_TYPES:
            raise UsageError('Invalid bundle type %s (options: [%s])' % (
              args.bundle_type, '|'.join(sorted(UPLOADED_TYPES)),
            ))
        bundle_subclass = get_bundle_subclass(args.bundle_type)
        metadata = metadata_util.request_missing_data(bundle_subclass, args)
        # Type-check the bundle metadata BEFORE uploading the bundle data.
        # This optimization will avoid file copies on failed bundle creations.
        bundle_subclass.construct(data_hash='', metadata=metadata).validate()
        print self.client.upload(args.bundle_type, args.path, metadata, worksheet_uuid)