How to use the dcicutils.ff_utils.get_metadata function in dcicutils

To help you get started, we’ve selected a few dcicutils examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github 4dn-dcic / foursight / chalicelib / checks / wfr_cgap_checks.py View on Github external
start = datetime.utcnow()
    action = ActionResult(connection, 'fastqcCGAP_start')
    action_logs = {'runs_started': [], 'runs_failed': []}
    my_auth = connection.ff_keys
    fastqcCGAP_check_result = action.get_associated_check_result(kwargs).get('full_output', {})
    targets = []
    if kwargs.get('start_missing_run'):
        targets.extend(fastqcCGAP_check_result.get('files_without_run', []))
    if kwargs.get('start_missing_meta'):
        targets.extend(fastqcCGAP_check_result.get('files_without_changes', []))
    for a_target in targets:
        now = datetime.utcnow()
        if (now-start).seconds > lambda_limit:
            action.description = 'Did not complete action due to time limitations'
            break
        a_file = ff_utils.get_metadata(a_target, key=my_auth)
        attributions = cgap_utils.get_attribution(a_file)
        inp_f = {'input_fastq': a_file['@id']}
        wfr_setup = wfrset_cgap_utils.step_settings('fastqc', 'no_organism', attributions)
        url = cgap_utils.run_missing_wfr(wfr_setup, inp_f, a_file['accession'], connection.ff_keys, connection.ff_env, mount=True)
        # aws run url
        if url.startswith('http'):
            action_logs['runs_started'].append(url)
        else:
            action_logs['runs_failed'].append([a_target, url])
    action.output = action_logs
    action.status = 'DONE'
    return action
github 4dn-dcic / foursight / chalicelib / checks / helpers / wfr_utils.py View on Github external
a_wfr['run_version'] = wfr_version.strip()
    my_workflows = [i for i in my_workflows if i['run_version'] in versions]
    if not my_workflows:
        return {'status': "no workflow in file with accepted version"}
    my_workflows = sorted(my_workflows, key=lambda k: k['run_hours'])
    same_type_wfrs = [i for i in my_workflows if i['run_type'] == wfr_name]

    if not same_type_wfrs:
        return {'status': "no workflow on file"}
    last_wfr = same_type_wfrs[0]

    # get metadata for the last wfr
    if all_wfrs:
        wfr = [i for i in all_wfrs if i['uuid'] == last_wfr['uuid']][0]
    else:
        wfr = ff_utils.get_metadata(last_wfr['uuid'], key)
    run_duration = last_wfr['run_hours']
    run_status = wfr['run_status']

    if run_status == 'complete':
        outputs = wfr.get('output_files')
        # some runs, like qc, don't have a real file output
        if md_qc:
            return {'status': 'complete'}
        # if expected output files, return a dictionary of argname:file_id
        else:
            out_files = {}
            for output in outputs:
                if output.get('format'):
                    # get the arg name
                    arg_name = output['workflow_argument_name']
                    try:
github 4dn-dcic / foursight / chalicelib / checks / badge_checks.py View on Github external
- full_output[output_keys[0]] should be a list of item @ids and message to patch into badge.
    - badges can also be edited to change the message.
    '''
    patches = {'add_badge_success': [], 'add_badge_failure': [],
               'remove_badge_success': [], 'remove_badge_failure': []}
    badge_id = '/badges/' + badge_name + '/'
    output_keys = ['Add badge', 'Remove badge']
    if isinstance(full_output[output_keys[0]], list):
        add_list = full_output[output_keys[0]]
    elif isinstance(full_output[output_keys[0]], dict):
        patches['edit_badge_success'] = []
        patches['edit_badge_failure'] = []
        output_keys.append('Keep badge and edit messages')
        add_list = full_output[output_keys[0]].keys()
    for add_key in add_list:
        add_result = ff_utils.get_metadata(add_key + '?frame=object&field=badges', key=ff_keys)
        badges = add_result['badges'] if add_result.get('badges') else []
        badges.append({'badge': badge_id, 'messages': [single_message] if single_message else full_output[output_keys[0]][add_key]})
        if [b['badge'] for b in badges].count(badge_id) > 1:
            # print an error message?
            patches['add_badge_failure'].append('{} already has badge'.format(add_key))
            continue
        try:
            response = ff_utils.patch_metadata({"badges": badges}, add_key[1:], key=ff_keys)
            if response['status'] == 'success':
                patches['add_badge_success'].append(add_key)
            else:
                patches['add_badge_failure'].append(add_key)
        except Exception:
            patches['add_badge_failure'].append(add_key)
    for remove_key, remove_val in full_output[output_keys[1]].items():
        # delete field if no badges?
github 4dn-dcic / foursight / chalicelib / checks / wfr_checks.py View on Github external
start = datetime.utcnow()
    action = ActionResult(connection, 'fastqc_start')
    action_logs = {'runs_started': [], 'runs_failed': []}
    my_auth = connection.ff_keys
    fastqc_check_result = action.get_associated_check_result(kwargs).get('full_output', {})
    targets = []
    if kwargs.get('start_missing_run'):
        targets.extend(fastqc_check_result.get('files_without_run', []))
    if kwargs.get('start_missing_meta'):
        targets.extend(fastqc_check_result.get('files_without_changes', []))
    for a_target in targets:
        now = datetime.utcnow()
        if (now-start).seconds > lambda_limit:
            action.description = 'Did not complete action due to time limitations'
            break
        a_file = ff_utils.get_metadata(a_target, key=my_auth)
        attributions = wfr_utils.get_attribution(a_file)
        inp_f = {'input_fastq': a_file['@id']}
        wfr_setup = wfrset_utils.step_settings('fastqc-0-11-4-1', 'no_organism', attributions)
        url = wfr_utils.run_missing_wfr(wfr_setup, inp_f, a_file['accession'], connection.ff_keys, connection.ff_env)
        # aws run url
        if url.startswith('http'):
            action_logs['runs_started'].append(url)
        else:
            action_logs['runs_failed'].append([a_target, url])
    action.output = action_logs
    action.status = 'DONE'
    return action
github 4dn-dcic / foursight / chalicelib / checks / helpers / wfr_utils.py View on Github external
template = {"workflow_argument_name": arg_name}
    if rename:
        change_from = rename[0]
        change_to = rename[1]

    # if it is list of items, change the structure
    if isinstance(obj_id, list):
        # if it is list of list, change the structure, for RNAseq
        if isinstance(obj_id[0], list):
            # will only work with single item in first list (was implemented for RNA seq)
            assert len(obj_id) == 1
            object_key = []
            uuid = []
            buckets = []
            for obj in obj_id[0]:
                metadata = ff_utils.get_metadata(obj, key=auth)
                object_key.append(metadata['display_title'])
                uuid.append(metadata['uuid'])
                # get the bucket
                if 'FileProcessed' in metadata['@type']:
                    my_bucket = out_bucket
                else:  # covers cases of FileFastq, FileReference, FileMicroscopy
                    my_bucket = raw_bucket
                buckets.append(my_bucket)
            # check bucket consistency
            assert len(list(set(buckets))) == 1
            template['object_key'] = [object_key]
            template['uuid'] = [uuid]
            template['bucket_name'] = buckets[0]
            if rename:
                template['rename'] = [i.replace(change_from, change_to) for i in template['object_key'][0]]
        # if it is just a list
github 4dn-dcic / foursight / chalicelib / checks / badge_checks.py View on Github external
'''
    check = CheckResult(connection, 'exp_has_raw_files')
    # search all experiments except microscopy experiments for missing files field
    no_files = ff_utils.search_metadata('search/?type=Experiment&%40type%21=ExperimentMic&files.uuid=No+value',
                                        key=connection.ff_keys)
    # also check sequencing experiments whose files items are all uploading/archived/deleted
    bad_status = ff_utils.search_metadata('search/?status=uploading&status=archived&status=deleted&status=upload+failed'
                                          '&type=FileFastq&experiments.uuid%21=No+value',
                                          key=connection.ff_keys)
    bad_status_ids = {item['@id']: item['status'] for item in bad_status}
    exps = list(set([exp['@id'] for fastq in bad_status for exp in
                     fastq.get('experiments') if fastq.get('experiments')]))
    missing_files_released = [e['@id'] for e in no_files if e.get('status') not in REV]
    missing_files_in_rev = [e['@id'] for e in no_files if e.get('status') in REV]
    for expt in exps:
        result = ff_utils.get_metadata(expt, key=connection.ff_keys)
        raw_files = False
        if result.get('files'):
            for fastq in result.get('files'):
                if fastq['@id'] not in bad_status_ids or result['status'] == bad_status_ids[fastq['@id']]:
                    raw_files = True
                    break
        if not raw_files:
            if result.get('status') in REV:
                missing_files_in_rev.append(expt)
            else:
                missing_files_released.append(expt)

    to_add, to_remove, ok = compare_badges(missing_files_released, 'Experiment', 'no-raw-files', connection.ff_keys)

    if to_add or to_remove:
        check.status = 'WARN'
github 4dn-dcic / Submit4DN / wranglertools / import_data.py View on Github external
if e.get('status') == 'error':
        return e
    if file_to_upload:
        # get s3 credentials
        if verb == 'PATCH':
            creds = get_upload_creds(e['@graph'][0]['accession'], connection)
            e['@graph'][0]['upload_credentials'] = creds
        # upload
        upload_file_item(e, filename_to_post)
        if ftp_download:
            os.remove(filename_to_post)
    if extrafiles:
        extcreds = e['@graph'][0].get('extra_files_creds')
        for fformat, filepath in extrafiles.items():
            try:
                file_format = ff_utils.get_metadata(fformat, key=connection.key)
                ff_uuid = file_format.get('uuid')
            except:
                raise "Can't find file_format item for %s" % fformat
            for ecred in extcreds:
                if ff_uuid == ecred.get('file_format'):
                    upload_creds = ecred.get('upload_credentials')
                    upload_extra_file(upload_creds, filepath)
    return e