Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def memory_bytes(self):
"""
Total installed memory of this RunManager
"""
# TODO: Read from batch
return parse_size('10000t')
for key in self._metadata_keys:
if key not in expected_keys:
raise UsageError('Unexpected metadata key: %s' % (key,))
for spec in metadata_specs:
if spec.key in self._metadata_keys:
value = getattr(self, spec.key)
if spec.type is float and isinstance(value, int):
# cast int to float
value = float(value)
# Validate formatted string fields
if issubclass(spec.type, str) and spec.formatting is not None and value:
try:
if spec.formatting == 'duration':
formatting.parse_duration(value)
elif spec.formatting == 'size':
formatting.parse_size(value)
elif spec.formatting == 'date':
formatting.parse_datetime(value)
except ValueError as e:
raise UsageError(str(e))
if value is not None and not isinstance(value, spec.type):
raise UsageError(
'Metadata value for %s should be of type %s, was %s (type %s)'
% (spec.key, spec.type.__name__, value, type(value).__name__)
)
elif not spec.generated:
raise UsageError('Missing metadata key: %s' % (spec.key,))
def _compute_request_memory(bundle):
"""
Compute the memory limit used for scheduling the run.
The default of 2g is for backwards compatibilty for
runs from before when we added client-side defaults
"""
if not bundle.metadata.request_memory:
return formatting.parse_size('2g')
return formatting.parse_size(bundle.metadata.request_memory)
def default_user_info(self):
info = self.config['server'].get(
'default_user_info', {'time_quota': '1y', 'disk_quota': '1t', 'parallel_run_quota': 3}
)
return {
'time_quota': formatting.parse_duration(info['time_quota']),
'disk_quota': formatting.parse_size(info['disk_quota']),
'parallel_run_quota': info['parallel_run_quota'],
}
password = getpass.getpass()
# Set up logging.
if args.verbose:
logging.basicConfig(format='%(asctime)s %(message)s', level=logging.DEBUG)
else:
logging.basicConfig(format='%(asctime)s %(message)s', level=logging.INFO)
try:
bundle_service = BundleServiceClient(args.server, username, password)
except BundleAuthException as ex:
logger.error('Cannot log into the bundle service. Please check your worker credentials.\n')
logger.debug('Auth error: {}'.format(ex))
return
max_work_dir_size_bytes = parse_size(args.max_work_dir_size)
if args.max_image_cache_size is None:
max_images_bytes = None
else:
max_images_bytes = parse_size(args.max_image_cache_size)
if not os.path.exists(args.work_dir):
logging.debug('Work dir %s doesn\'t exist, creating.', args.work_dir)
os.makedirs(args.work_dir, 0o770)
def create_local_run_manager(worker):
"""
To avoid circular dependencies the Worker initializes takes a RunManager factory
to initilize its run manager. This method creates a LocalFilesystem-Docker RunManager
which is the default execution architecture Codalab uses
"""
docker_runtime = docker_utils.get_available_runtime()
logging.basicConfig(format='%(asctime)s %(message)s', level=logging.DEBUG)
else:
logging.basicConfig(format='%(asctime)s %(message)s', level=logging.INFO)
try:
bundle_service = BundleServiceClient(args.server, username, password)
except BundleAuthException as ex:
logger.error('Cannot log into the bundle service. Please check your worker credentials.\n')
logger.debug('Auth error: {}'.format(ex))
return
max_work_dir_size_bytes = parse_size(args.max_work_dir_size)
if args.max_image_cache_size is None:
max_images_bytes = None
else:
max_images_bytes = parse_size(args.max_image_cache_size)
if not os.path.exists(args.work_dir):
logging.debug('Work dir %s doesn\'t exist, creating.', args.work_dir)
os.makedirs(args.work_dir, 0o770)
def create_local_run_manager(worker):
"""
To avoid circular dependencies the Worker initializes takes a RunManager factory
to initilize its run manager. This method creates a LocalFilesystem-Docker RunManager
which is the default execution architecture Codalab uses
"""
docker_runtime = docker_utils.get_available_runtime()
cpuset = parse_cpuset_args(args.cpuset)
gpuset = parse_gpuset_args(args.gpuset)
dependency_manager = LocalFileSystemDependencyManager(
def _compute_request_disk(self, bundle):
"""
Compute the disk limit used for scheduling the run.
The default is min(disk quota the user has left, global max)
"""
if not bundle.metadata.request_disk:
return min(
self._model.get_user_disk_quota_left(bundle.owner_id) - 1, self._max_request_disk
)
return formatting.parse_size(bundle.metadata.request_disk)