How to use the codalab.lib.formatting.parse_size function in codalab

To help you get started, we’ve selected a few codalab examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github codalab / codalab-worksheets / codalab / worker / aws_batch / aws_batch_run_manager.py View on Github external
def memory_bytes(self):
        """
        Total installed memory of this RunManager
        """
        # TODO: Read from batch
        return parse_size('10000t')
github codalab / codalab-worksheets / codalab / objects / metadata.py View on Github external
for key in self._metadata_keys:
            if key not in expected_keys:
                raise UsageError('Unexpected metadata key: %s' % (key,))
        for spec in metadata_specs:
            if spec.key in self._metadata_keys:
                value = getattr(self, spec.key)
                if spec.type is float and isinstance(value, int):
                    # cast int to float
                    value = float(value)
                # Validate formatted string fields
                if issubclass(spec.type, str) and spec.formatting is not None and value:
                    try:
                        if spec.formatting == 'duration':
                            formatting.parse_duration(value)
                        elif spec.formatting == 'size':
                            formatting.parse_size(value)
                        elif spec.formatting == 'date':
                            formatting.parse_datetime(value)
                    except ValueError as e:
                        raise UsageError(str(e))
                if value is not None and not isinstance(value, spec.type):
                    raise UsageError(
                        'Metadata value for %s should be of type %s, was %s (type %s)'
                        % (spec.key, spec.type.__name__, value, type(value).__name__)
                    )
            elif not spec.generated:
                raise UsageError('Missing metadata key: %s' % (spec.key,))
github codalab / codalab-worksheets / codalab / server / bundle_manager.py View on Github external
def _compute_request_memory(bundle):
        """
        Compute the memory limit used for scheduling the run.
        The default of 2g is for backwards compatibilty for
        runs from before when we added client-side defaults
        """
        if not bundle.metadata.request_memory:
            return formatting.parse_size('2g')
        return formatting.parse_size(bundle.metadata.request_memory)
github codalab / codalab-worksheets / codalab / lib / codalab_manager.py View on Github external
def default_user_info(self):
        info = self.config['server'].get(
            'default_user_info', {'time_quota': '1y', 'disk_quota': '1t', 'parallel_run_quota': 3}
        )
        return {
            'time_quota': formatting.parse_duration(info['time_quota']),
            'disk_quota': formatting.parse_size(info['disk_quota']),
            'parallel_run_quota': info['parallel_run_quota'],
        }
github codalab / codalab-worksheets / codalab / worker / main.py View on Github external
password = getpass.getpass()

    # Set up logging.
    if args.verbose:
        logging.basicConfig(format='%(asctime)s %(message)s', level=logging.DEBUG)
    else:
        logging.basicConfig(format='%(asctime)s %(message)s', level=logging.INFO)

    try:
        bundle_service = BundleServiceClient(args.server, username, password)
    except BundleAuthException as ex:
        logger.error('Cannot log into the bundle service. Please check your worker credentials.\n')
        logger.debug('Auth error: {}'.format(ex))
        return

    max_work_dir_size_bytes = parse_size(args.max_work_dir_size)
    if args.max_image_cache_size is None:
        max_images_bytes = None
    else:
        max_images_bytes = parse_size(args.max_image_cache_size)

    if not os.path.exists(args.work_dir):
        logging.debug('Work dir %s doesn\'t exist, creating.', args.work_dir)
        os.makedirs(args.work_dir, 0o770)

    def create_local_run_manager(worker):
        """
        To avoid circular dependencies the Worker initializes takes a RunManager factory
        to initilize its run manager. This method creates a LocalFilesystem-Docker RunManager
        which is the default execution architecture Codalab uses
        """
        docker_runtime = docker_utils.get_available_runtime()
github codalab / codalab-worksheets / codalab / worker / main.py View on Github external
logging.basicConfig(format='%(asctime)s %(message)s', level=logging.DEBUG)
    else:
        logging.basicConfig(format='%(asctime)s %(message)s', level=logging.INFO)

    try:
        bundle_service = BundleServiceClient(args.server, username, password)
    except BundleAuthException as ex:
        logger.error('Cannot log into the bundle service. Please check your worker credentials.\n')
        logger.debug('Auth error: {}'.format(ex))
        return

    max_work_dir_size_bytes = parse_size(args.max_work_dir_size)
    if args.max_image_cache_size is None:
        max_images_bytes = None
    else:
        max_images_bytes = parse_size(args.max_image_cache_size)

    if not os.path.exists(args.work_dir):
        logging.debug('Work dir %s doesn\'t exist, creating.', args.work_dir)
        os.makedirs(args.work_dir, 0o770)

    def create_local_run_manager(worker):
        """
        To avoid circular dependencies the Worker initializes takes a RunManager factory
        to initilize its run manager. This method creates a LocalFilesystem-Docker RunManager
        which is the default execution architecture Codalab uses
        """
        docker_runtime = docker_utils.get_available_runtime()
        cpuset = parse_cpuset_args(args.cpuset)
        gpuset = parse_gpuset_args(args.gpuset)

        dependency_manager = LocalFileSystemDependencyManager(
github codalab / codalab-worksheets / codalab / server / bundle_manager.py View on Github external
def _compute_request_disk(self, bundle):
        """
        Compute the disk limit used for scheduling the run.
        The default is min(disk quota the user has left, global max)
        """
        if not bundle.metadata.request_disk:
            return min(
                self._model.get_user_disk_quota_left(bundle.owner_id) - 1, self._max_request_disk
            )
        return formatting.parse_size(bundle.metadata.request_disk)