Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
converted_configuration = StrikeConfigurationV1(configuration).get_dict()
converted_configuration['version'] = CURRENT_VERSION
mount = converted_configuration['mount']
mount_path = mount.split(':')[1]
transfer_suffix = converted_configuration['transfer_suffix']
del converted_configuration['mount']
del converted_configuration['transfer_suffix']
auto_workspace_name = 'auto_wksp_for_%s' % mount.replace(':', '_').replace('/', '_')
auto_workspace_name = auto_workspace_name[:50] # Truncate to max name length of 50 chars
title = 'Auto Workspace for %s' % mount
title = title[:50] # Truncate to max title length of 50 chars
try:
Workspace.objects.get(name=auto_workspace_name)
except Workspace.DoesNotExist:
workspace = Workspace()
workspace.name = auto_workspace_name
workspace.title = title
desc = 'This workspace was automatically created for mount %s to support converting Strike from 1.0 to 2.0'
workspace.description = desc % mount
workspace.json_config = {'version': '1.0', 'broker': {'type': 'host', 'host_path': mount_path}}
workspace.save()
converted_configuration['workspace'] = auto_workspace_name
converted_configuration['monitor'] = {'type': 'dir-watcher', 'transfer_suffix': transfer_suffix}
for file_dict in converted_configuration['files_to_ingest']:
file_dict['new_workspace'] = file_dict['workspace_name']
file_dict['new_file_path'] = file_dict['workspace_path']
del file_dict['workspace_name']
del file_dict['workspace_path']
return converted_configuration
workspace_name = options.get('workspace')
logger.info('Command starting: scale_upload_file')
logger.info(' - Workspace: %s', workspace_name)
# Validate the file paths
file_name = os.path.basename(local_path)
if not os.path.exists(local_path):
logger.exception('Local file does not exist: %s', local_path)
sys.exit(1)
# Attempt to fetch the workspace model
try:
workspace = Workspace.objects.get(name=workspace_name)
except Workspace.DoesNotExist:
logger.exception('Workspace does not exist: %s', workspace_name)
sys.exit(1)
# Attempt to set up a file model
try:
scale_file = ScaleFile.objects.get(file_name=file_name)
except ScaleFile.DoesNotExist:
scale_file = ScaleFile()
scale_file.update_uuid(file_name)
scale_file.file_path = remote_path
try:
ScaleFile.objects.upload_files(workspace, [FileUpload(scale_file, local_path)])
except:
logger.exception('Unknown error occurred, exit code 1 returning')
sys.exit(1)
This method starts the file move process.
"""
logger.info('Command starting: scale_move_files')
# Register a listener to handle clean shutdowns
signal.signal(signal.SIGTERM, self._onsigterm)
file_ids = json.loads(os.environ.get('FILE_IDS'))
new_workspace_name = json.loads(os.environ.get('NEW_WORKSPACE'))
uri = json.loads(os.environ.get('NEW_PATH'))
new_workspace = None
if new_workspace_name:
try:
new_workspace = Workspace.objects.get(name=new_workspace_name)
except Workspace.DoesNotExist:
logger.error('Error running command scale_move_files: Workspace %s does not exist' % new_workspace_name)
sys.exit(1)
logger.info('Command starting: scale_move_files')
logger.info('File IDs: %s', file_ids)
move_files_job.move_files(file_ids=file_ids, new_workspace=new_workspace, new_file_path=uri)
logger.info('Command completed: scale_move_files')
sys.exit(0)
# Try converting from 1.0
converted_configuration = StrikeConfigurationV1(configuration).get_dict()
converted_configuration['version'] = CURRENT_VERSION
mount = converted_configuration['mount']
mount_path = mount.split(':')[1]
transfer_suffix = converted_configuration['transfer_suffix']
del converted_configuration['mount']
del converted_configuration['transfer_suffix']
auto_workspace_name = 'auto_wksp_for_%s' % mount.replace(':', '_').replace('/', '_')
auto_workspace_name = auto_workspace_name[:50] # Truncate to max name length of 50 chars
title = 'Auto Workspace for %s' % mount
title = title[:50] # Truncate to max title length of 50 chars
try:
Workspace.objects.get(name=auto_workspace_name)
except Workspace.DoesNotExist:
workspace = Workspace()
workspace.name = auto_workspace_name
workspace.title = title
desc = 'This workspace was automatically created for mount %s to support converting Strike from 1.0 to 2.0'
workspace.description = desc % mount
workspace.json_config = {'version': '1.0', 'broker': {'type': 'host', 'host_path': mount_path}}
workspace.save()
converted_configuration['workspace'] = auto_workspace_name
converted_configuration['monitor'] = {'type': 'dir-watcher', 'transfer_suffix': transfer_suffix}
for file_dict in converted_configuration['files_to_ingest']:
file_dict['new_workspace'] = file_dict['workspace_name']
file_dict['new_file_path'] = file_dict['workspace_path']
del file_dict['workspace_name']
del file_dict['workspace_path']
workspace_dict[name] = workspace_id
config = job_exe.job.get_job_configuration()
if config and (config.default_output_workspace or config.output_workspaces):
workspace_names_dict = {} # {Output name: workspace name}
# Do the new way, grabbing output workspaces from job configuration
config = job_exe.job.get_job_configuration()
for name in output_params:
if name in config.output_workspaces:
workspace_names_dict[name] = config.output_workspaces[name]
elif config.default_output_workspace:
workspace_names_dict[name] = config.default_output_workspace
else:
raise Exception('No output workspace configured for output \'%s\'' % name)
from storage.models import Workspace
workspace_mapping = {w.name: w.id for w in Workspace.objects.filter(name__in=workspace_names_dict.values())}
for output_name, workspace_name in workspace_names_dict.items():
workspace_dict[output_name] = workspace_mapping[workspace_name]
return workspace_dict
results = dict()
query = None
for config in configuration:
if not 'workspace_name' in config:
continue
name = config['workspace_name']
# Create a query filter for each workspace
if name not in results:
results[name] = None
query_filter = Q(name=name)
query = query | query_filter if query else query_filter
# Fetch each workspace model and map it
if query:
for workspace in Workspace.objects.filter(query):
results[workspace.name] = workspace
# Check for any missing workspace model declarations
for name, workspace in results.iteritems():
if not workspace:
raise InvalidTriggerRule('Unknown workspace reference: %s' % name)
return results
def _cache_workspace_names(self, workspace_ids):
"""Queries and caches the workspace names for the given IDs
:param workspace_ids: The set of workspace IDs
:type workspace_ids: set
"""
ids = []
for workspace_id in workspace_ids:
if workspace_id not in self._cached_workspace_names:
ids.append(workspace_id)
if ids:
for workspace in Workspace.objects.filter(id__in=ids).iterator():
self._cached_workspace_names[workspace.id] = workspace.name
def validate(self):
"""See :meth:`trigger.configuration.trigger_rule.TriggerRuleConfiguration.validate`
"""
workspace_name = self.get_workspace_name()
if Workspace.objects.filter(name=workspace_name).count() == 0:
raise InvalidTriggerRule('%s is an invalid workspace name' % workspace_name)
def get_workspaces(self, workspace_ids):
"""See :meth:`job.configuration.data.data_file.AbstractDataFileStore.get_workspaces`
"""
workspaces = Workspace.objects.filter(id__in=workspace_ids)
results = {}
for workspace in workspaces:
results[workspace.id] = workspace.is_active
return results
def _get_workspace_map(self, configuration):
"""Builds a mapping for a workspace and configuration of name to model instance.
:param configuration: A list of configurations that specify system names of models to fetch and map.
:type configuration: list[dict]
:returns: A mapping of workspace system names to associated model instances.
:rtype: dict[string, :class:`storage.models.Workspace`]
"""
# Build a mapping of required workspaces
results = {file_dict['workspace_name']: None for file_dict in configuration}
for workspace in Workspace.objects.filter(name__in=results.keys()):
if not workspace.is_active:
raise InvalidStrikeConfiguration('Workspace is not active: %s' % workspace.name)
results[workspace.name] = workspace
# Check for any missing workspace model declarations
for name, workspace in results.iteritems():
if not workspace:
raise InvalidStrikeConfiguration('Unknown workspace reference: %s' % name)
return results