Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
start = datetime.utcnow()
action = ActionResult(connection, 'fastqcCGAP_start')
action_logs = {'runs_started': [], 'runs_failed': []}
my_auth = connection.ff_keys
fastqcCGAP_check_result = action.get_associated_check_result(kwargs).get('full_output', {})
targets = []
if kwargs.get('start_missing_run'):
targets.extend(fastqcCGAP_check_result.get('files_without_run', []))
if kwargs.get('start_missing_meta'):
targets.extend(fastqcCGAP_check_result.get('files_without_changes', []))
for a_target in targets:
now = datetime.utcnow()
if (now-start).seconds > lambda_limit:
action.description = 'Did not complete action due to time limitations'
break
a_file = ff_utils.get_metadata(a_target, key=my_auth)
attributions = cgap_utils.get_attribution(a_file)
inp_f = {'input_fastq': a_file['@id']}
wfr_setup = wfrset_cgap_utils.step_settings('fastqc', 'no_organism', attributions)
url = cgap_utils.run_missing_wfr(wfr_setup, inp_f, a_file['accession'], connection.ff_keys, connection.ff_env, mount=True)
# aws run url
if url.startswith('http'):
action_logs['runs_started'].append(url)
else:
action_logs['runs_failed'].append([a_target, url])
action.output = action_logs
action.status = 'DONE'
return action
a_wfr['run_version'] = wfr_version.strip()
my_workflows = [i for i in my_workflows if i['run_version'] in versions]
if not my_workflows:
return {'status': "no workflow in file with accepted version"}
my_workflows = sorted(my_workflows, key=lambda k: k['run_hours'])
same_type_wfrs = [i for i in my_workflows if i['run_type'] == wfr_name]
if not same_type_wfrs:
return {'status': "no workflow on file"}
last_wfr = same_type_wfrs[0]
# get metadata for the last wfr
if all_wfrs:
wfr = [i for i in all_wfrs if i['uuid'] == last_wfr['uuid']][0]
else:
wfr = ff_utils.get_metadata(last_wfr['uuid'], key)
run_duration = last_wfr['run_hours']
run_status = wfr['run_status']
if run_status == 'complete':
outputs = wfr.get('output_files')
# some runs, like qc, don't have a real file output
if md_qc:
return {'status': 'complete'}
# if expected output files, return a dictionary of argname:file_id
else:
out_files = {}
for output in outputs:
if output.get('format'):
# get the arg name
arg_name = output['workflow_argument_name']
try:
- full_output[output_keys[0]] should be a list of item @ids and message to patch into badge.
- badges can also be edited to change the message.
'''
patches = {'add_badge_success': [], 'add_badge_failure': [],
'remove_badge_success': [], 'remove_badge_failure': []}
badge_id = '/badges/' + badge_name + '/'
output_keys = ['Add badge', 'Remove badge']
if isinstance(full_output[output_keys[0]], list):
add_list = full_output[output_keys[0]]
elif isinstance(full_output[output_keys[0]], dict):
patches['edit_badge_success'] = []
patches['edit_badge_failure'] = []
output_keys.append('Keep badge and edit messages')
add_list = full_output[output_keys[0]].keys()
for add_key in add_list:
add_result = ff_utils.get_metadata(add_key + '?frame=object&field=badges', key=ff_keys)
badges = add_result['badges'] if add_result.get('badges') else []
badges.append({'badge': badge_id, 'messages': [single_message] if single_message else full_output[output_keys[0]][add_key]})
if [b['badge'] for b in badges].count(badge_id) > 1:
# print an error message?
patches['add_badge_failure'].append('{} already has badge'.format(add_key))
continue
try:
response = ff_utils.patch_metadata({"badges": badges}, add_key[1:], key=ff_keys)
if response['status'] == 'success':
patches['add_badge_success'].append(add_key)
else:
patches['add_badge_failure'].append(add_key)
except Exception:
patches['add_badge_failure'].append(add_key)
for remove_key, remove_val in full_output[output_keys[1]].items():
# delete field if no badges?
start = datetime.utcnow()
action = ActionResult(connection, 'fastqc_start')
action_logs = {'runs_started': [], 'runs_failed': []}
my_auth = connection.ff_keys
fastqc_check_result = action.get_associated_check_result(kwargs).get('full_output', {})
targets = []
if kwargs.get('start_missing_run'):
targets.extend(fastqc_check_result.get('files_without_run', []))
if kwargs.get('start_missing_meta'):
targets.extend(fastqc_check_result.get('files_without_changes', []))
for a_target in targets:
now = datetime.utcnow()
if (now-start).seconds > lambda_limit:
action.description = 'Did not complete action due to time limitations'
break
a_file = ff_utils.get_metadata(a_target, key=my_auth)
attributions = wfr_utils.get_attribution(a_file)
inp_f = {'input_fastq': a_file['@id']}
wfr_setup = wfrset_utils.step_settings('fastqc-0-11-4-1', 'no_organism', attributions)
url = wfr_utils.run_missing_wfr(wfr_setup, inp_f, a_file['accession'], connection.ff_keys, connection.ff_env)
# aws run url
if url.startswith('http'):
action_logs['runs_started'].append(url)
else:
action_logs['runs_failed'].append([a_target, url])
action.output = action_logs
action.status = 'DONE'
return action
template = {"workflow_argument_name": arg_name}
if rename:
change_from = rename[0]
change_to = rename[1]
# if it is list of items, change the structure
if isinstance(obj_id, list):
# if it is list of list, change the structure, for RNAseq
if isinstance(obj_id[0], list):
# will only work with single item in first list (was implemented for RNA seq)
assert len(obj_id) == 1
object_key = []
uuid = []
buckets = []
for obj in obj_id[0]:
metadata = ff_utils.get_metadata(obj, key=auth)
object_key.append(metadata['display_title'])
uuid.append(metadata['uuid'])
# get the bucket
if 'FileProcessed' in metadata['@type']:
my_bucket = out_bucket
else: # covers cases of FileFastq, FileReference, FileMicroscopy
my_bucket = raw_bucket
buckets.append(my_bucket)
# check bucket consistency
assert len(list(set(buckets))) == 1
template['object_key'] = [object_key]
template['uuid'] = [uuid]
template['bucket_name'] = buckets[0]
if rename:
template['rename'] = [i.replace(change_from, change_to) for i in template['object_key'][0]]
# if it is just a list
'''
check = CheckResult(connection, 'exp_has_raw_files')
# search all experiments except microscopy experiments for missing files field
no_files = ff_utils.search_metadata('search/?type=Experiment&%40type%21=ExperimentMic&files.uuid=No+value',
key=connection.ff_keys)
# also check sequencing experiments whose files items are all uploading/archived/deleted
bad_status = ff_utils.search_metadata('search/?status=uploading&status=archived&status=deleted&status=upload+failed'
'&type=FileFastq&experiments.uuid%21=No+value',
key=connection.ff_keys)
bad_status_ids = {item['@id']: item['status'] for item in bad_status}
exps = list(set([exp['@id'] for fastq in bad_status for exp in
fastq.get('experiments') if fastq.get('experiments')]))
missing_files_released = [e['@id'] for e in no_files if e.get('status') not in REV]
missing_files_in_rev = [e['@id'] for e in no_files if e.get('status') in REV]
for expt in exps:
result = ff_utils.get_metadata(expt, key=connection.ff_keys)
raw_files = False
if result.get('files'):
for fastq in result.get('files'):
if fastq['@id'] not in bad_status_ids or result['status'] == bad_status_ids[fastq['@id']]:
raw_files = True
break
if not raw_files:
if result.get('status') in REV:
missing_files_in_rev.append(expt)
else:
missing_files_released.append(expt)
to_add, to_remove, ok = compare_badges(missing_files_released, 'Experiment', 'no-raw-files', connection.ff_keys)
if to_add or to_remove:
check.status = 'WARN'
if e.get('status') == 'error':
return e
if file_to_upload:
# get s3 credentials
if verb == 'PATCH':
creds = get_upload_creds(e['@graph'][0]['accession'], connection)
e['@graph'][0]['upload_credentials'] = creds
# upload
upload_file_item(e, filename_to_post)
if ftp_download:
os.remove(filename_to_post)
if extrafiles:
extcreds = e['@graph'][0].get('extra_files_creds')
for fformat, filepath in extrafiles.items():
try:
file_format = ff_utils.get_metadata(fformat, key=connection.key)
ff_uuid = file_format.get('uuid')
except:
raise "Can't find file_format item for %s" % fformat
for ecred in extcreds:
if ff_uuid == ecred.get('file_format'):
upload_creds = ecred.get('upload_credentials')
upload_extra_file(upload_creds, filepath)
return e