How to use the storage.models.Workspace function in storage

To help you get started, we’ve selected a few storage examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github ngageoint / scale / scale / ingest / strike / configuration / json / configuration_2_0.py View on Github external
converted_configuration = StrikeConfigurationV1(configuration).get_dict()
        converted_configuration['version'] = CURRENT_VERSION

        mount = converted_configuration['mount']
        mount_path = mount.split(':')[1]
        transfer_suffix = converted_configuration['transfer_suffix']
        del converted_configuration['mount']
        del converted_configuration['transfer_suffix']
        auto_workspace_name = 'auto_wksp_for_%s' % mount.replace(':', '_').replace('/', '_')
        auto_workspace_name = auto_workspace_name[:50]  # Truncate to max name length of 50 chars
        title = 'Auto Workspace for %s' % mount
        title = title[:50]  # Truncate to max title length of 50 chars
        try:
            Workspace.objects.get(name=auto_workspace_name)
        except Workspace.DoesNotExist:
            workspace = Workspace()
            workspace.name = auto_workspace_name
            workspace.title = title
            desc = 'This workspace was automatically created for mount %s to support converting Strike from 1.0 to 2.0'
            workspace.description = desc % mount
            workspace.json_config = {'version': '1.0', 'broker': {'type': 'host', 'host_path': mount_path}}
            workspace.save()

        converted_configuration['workspace'] = auto_workspace_name
        converted_configuration['monitor'] = {'type': 'dir-watcher', 'transfer_suffix': transfer_suffix}
        for file_dict in converted_configuration['files_to_ingest']:
            file_dict['new_workspace'] = file_dict['workspace_name']
            file_dict['new_file_path'] = file_dict['workspace_path']
            del file_dict['workspace_name']
            del file_dict['workspace_path']

        return converted_configuration
github ngageoint / scale / scale / storage / management / commands / scale_upload_file.py View on Github external
workspace_name = options.get('workspace')

        logger.info('Command starting: scale_upload_file')
        logger.info(' - Workspace: %s', workspace_name)

        # Validate the file paths
        file_name = os.path.basename(local_path)
        if not os.path.exists(local_path):
            logger.exception('Local file does not exist: %s', local_path)
            sys.exit(1)

        # Attempt to fetch the workspace model
        try:
            workspace = Workspace.objects.get(name=workspace_name)
        except Workspace.DoesNotExist:
            logger.exception('Workspace does not exist: %s', workspace_name)
            sys.exit(1)

        # Attempt to set up a file model
        try:
            scale_file = ScaleFile.objects.get(file_name=file_name)
        except ScaleFile.DoesNotExist:
            scale_file = ScaleFile()
            scale_file.update_uuid(file_name)
        scale_file.file_path = remote_path

        try:
            ScaleFile.objects.upload_files(workspace, [FileUpload(scale_file, local_path)])
        except:
            logger.exception('Unknown error occurred, exit code 1 returning')
            sys.exit(1)
github ngageoint / scale / scale / storage / management / commands / scale_move_files.py View on Github external
This method starts the file move process.
        """

        logger.info('Command starting: scale_move_files')
        # Register a listener to handle clean shutdowns
        signal.signal(signal.SIGTERM, self._onsigterm)

        file_ids = json.loads(os.environ.get('FILE_IDS'))
        new_workspace_name = json.loads(os.environ.get('NEW_WORKSPACE'))
        uri = json.loads(os.environ.get('NEW_PATH'))

        new_workspace = None
        if new_workspace_name:
            try:
                new_workspace = Workspace.objects.get(name=new_workspace_name)
            except Workspace.DoesNotExist:
                logger.error('Error running command scale_move_files: Workspace %s does not exist' % new_workspace_name)
                sys.exit(1)

        logger.info('Command starting: scale_move_files')
        logger.info('File IDs: %s', file_ids)
        
        move_files_job.move_files(file_ids=file_ids, new_workspace=new_workspace, new_file_path=uri)

        logger.info('Command completed: scale_move_files')

        sys.exit(0)
github ngageoint / scale / scale / ingest / strike / configuration / json / configuration_2_0.py View on Github external
# Try converting from 1.0
        converted_configuration = StrikeConfigurationV1(configuration).get_dict()
        converted_configuration['version'] = CURRENT_VERSION

        mount = converted_configuration['mount']
        mount_path = mount.split(':')[1]
        transfer_suffix = converted_configuration['transfer_suffix']
        del converted_configuration['mount']
        del converted_configuration['transfer_suffix']
        auto_workspace_name = 'auto_wksp_for_%s' % mount.replace(':', '_').replace('/', '_')
        auto_workspace_name = auto_workspace_name[:50]  # Truncate to max name length of 50 chars
        title = 'Auto Workspace for %s' % mount
        title = title[:50]  # Truncate to max title length of 50 chars
        try:
            Workspace.objects.get(name=auto_workspace_name)
        except Workspace.DoesNotExist:
            workspace = Workspace()
            workspace.name = auto_workspace_name
            workspace.title = title
            desc = 'This workspace was automatically created for mount %s to support converting Strike from 1.0 to 2.0'
            workspace.description = desc % mount
            workspace.json_config = {'version': '1.0', 'broker': {'type': 'host', 'host_path': mount_path}}
            workspace.save()

        converted_configuration['workspace'] = auto_workspace_name
        converted_configuration['monitor'] = {'type': 'dir-watcher', 'transfer_suffix': transfer_suffix}
        for file_dict in converted_configuration['files_to_ingest']:
            file_dict['new_workspace'] = file_dict['workspace_name']
            file_dict['new_file_path'] = file_dict['workspace_path']
            del file_dict['workspace_name']
            del file_dict['workspace_path']
github ngageoint / scale / scale / job / configuration / data / job_data.py View on Github external
workspace_dict[name] = workspace_id
        config = job_exe.job.get_job_configuration()
        if config and (config.default_output_workspace or config.output_workspaces):
            workspace_names_dict = {}  # {Output name: workspace name}
            # Do the new way, grabbing output workspaces from job configuration
            config = job_exe.job.get_job_configuration()
            for name in output_params:
                if name in config.output_workspaces:
                    workspace_names_dict[name] = config.output_workspaces[name]
                elif config.default_output_workspace:
                    workspace_names_dict[name] = config.default_output_workspace
                else:
                    raise Exception('No output workspace configured for output \'%s\'' % name)

            from storage.models import Workspace
            workspace_mapping = {w.name: w.id for w in Workspace.objects.filter(name__in=workspace_names_dict.values())}
            for output_name, workspace_name in workspace_names_dict.items():
                workspace_dict[output_name] = workspace_mapping[workspace_name]

        return workspace_dict
github ngageoint / scale / scale / ingest / triggers / ingest_rule.py View on Github external
results = dict()
        query = None
        for config in configuration:
            if not 'workspace_name' in config:
                continue
            name = config['workspace_name']

            # Create a query filter for each workspace
            if name not in results:
                results[name] = None
                query_filter = Q(name=name)
                query = query | query_filter if query else query_filter

        # Fetch each workspace model and map it
        if query:
            for workspace in Workspace.objects.filter(query):
                results[workspace.name] = workspace

        # Check for any missing workspace model declarations
        for name, workspace in results.iteritems():
            if not workspace:
                raise InvalidTriggerRule('Unknown workspace reference: %s' % name)

        return results
github ngageoint / scale / scale / job / execution / configuration / configurators.py View on Github external
def _cache_workspace_names(self, workspace_ids):
        """Queries and caches the workspace names for the given IDs

        :param workspace_ids: The set of workspace IDs
        :type workspace_ids: set
        """

        ids = []
        for workspace_id in workspace_ids:
            if workspace_id not in self._cached_workspace_names:
                ids.append(workspace_id)

        if ids:
            for workspace in Workspace.objects.filter(id__in=ids).iterator():
                self._cached_workspace_names[workspace.id] = workspace.name
github ngageoint / scale / scale / source / triggers / configuration / parse_trigger_rule_1_0.py View on Github external
def validate(self):
        """See :meth:`trigger.configuration.trigger_rule.TriggerRuleConfiguration.validate`
        """

        workspace_name = self.get_workspace_name()

        if Workspace.objects.filter(name=workspace_name).count() == 0:
            raise InvalidTriggerRule('%s is an invalid workspace name' % workspace_name)
github ngageoint / scale / scale / product / configuration / product_data_file.py View on Github external
def get_workspaces(self, workspace_ids):
        """See :meth:`job.configuration.data.data_file.AbstractDataFileStore.get_workspaces`
        """

        workspaces = Workspace.objects.filter(id__in=workspace_ids)

        results = {}
        for workspace in workspaces:
            results[workspace.id] = workspace.is_active

        return results
github ngageoint / scale / scale / ingest / strike / configuration / strike_configuration_1_0.py View on Github external
def _get_workspace_map(self, configuration):
        """Builds a mapping for a workspace and configuration of name to model instance.

        :param configuration: A list of configurations that specify system names of models to fetch and map.
        :type configuration: list[dict]
        :returns: A mapping of workspace system names to associated model instances.
        :rtype: dict[string, :class:`storage.models.Workspace`]
        """

        # Build a mapping of required workspaces
        results = {file_dict['workspace_name']: None for file_dict in configuration}
        for workspace in Workspace.objects.filter(name__in=results.keys()):
            if not workspace.is_active:
                raise InvalidStrikeConfiguration('Workspace is not active: %s' % workspace.name)
            results[workspace.name] = workspace

        # Check for any missing workspace model declarations
        for name, workspace in results.iteritems():
            if not workspace:
                raise InvalidStrikeConfiguration('Unknown workspace reference: %s' % name)
        return results