Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
"%s sort -@ %d -n %s %s" \
% (samtools, cpu_count(), input_bam_filename, final_nmsrt_bam_prefix)
logger.info(command)
subprocess.check_call(shlex.split(command))
final_BEDPE_filename = input_bam_basename + ".bedpe.gz"
out, err = common.run_pipe([
"bamToBed -bedpe -mate1 -i %s" % (final_nmsrt_bam_filename),
"gzip -cn"],
outfile=final_BEDPE_filename)
subprocess.check_output('ls -l', shell=True)
tagAlign_file = dxpy.upload_local_file(final_TA_filename)
if paired_end:
BEDPE_file = dxpy.upload_local_file(final_BEDPE_filename)
output = {}
output["tagAlign_file"] = dxpy.dxlink(tagAlign_file)
if paired_end:
output["BEDPE_file"] = dxpy.dxlink(BEDPE_file)
return output
line_count = wc_output.split()[0]
n_peaks = int(line_count)
#TODO batch consistency plot
# The following line(s) use the Python bindings to upload your file outputs
# after you have created them on the local file system. It assumes that you
# have used the output field name for the filename for each output, but you
# can change that behavior to suit your needs.
output = {}
EM_fit_output = None
empirical_curves_output = None
overlapped_peaks = None
EM_parameters_log = dxpy.upload_local_file(rep1_vs_rep2_prefix + '.log.txt')
IDR2_plot = dxpy.upload_local_file(pooled_common_peaks_IDR_filename + '.png')
output.update({
"IDR2_plot": dxpy.dxlink(IDR2_plot)
})
npeaks_pass = dxpy.upload_local_file(npeaks_pass_filename)
IDR_output = dxpy.upload_local_file(compress(pooled_common_peaks_IDR_filename))
IDR_peaks = dxpy.upload_local_file(compress(final_IDR_thresholded_filename))
#
# return { "app_output_field": postprocess_job.get_output_ref("answer"), ...}
#
subprocess.check_output('ls -l', shell=True, stderr=subprocess.STDOUT)
output.update({
fieldnames = [
'analysis',
'experiment',
'dx_pipeline',
'files',
'error'
]
output_writer = csv.DictWriter(fh, fieldnames, delimiter='\t')
output_writer.writeheader()
for output_row in output_rows:
output_writer.writerow(output_row)
for line in open(outfn, 'r'):
print(line)
output = {"outfile": dxpy.upload_local_file(outfn)}
return output
def upload_file(ovalue):
sh("unset DX_WORKSPACE_ID && dx cd $DX_PROJECT_CONTEXT_ID: && dx mkdir -p {}".format(folder))
return dxpy.dxlink(dxpy.upload_local_file(remove_prefix(ovalue['location'], "file://"), wait_on_close=True, project=dxpy.PROJECT_CONTEXT_ID, folder=folder))
def upload_local_file(local_path, project, destFolder, **kwargs):
for i in range(0,max_num_retries):
try:
return dxpy.upload_local_file(filename = local_path,
project = project.get_id(),
folder = destFolder,
show_progress = True,
wait_on_close=True,
**kwargs)
except:
print("Sleeping for 5 seconds before trying again")
time.sleep(5)
raise Exception("Error uploading file {}".format(local_path))
# count peaks
npeaks_in = common.count_lines(common.uncompress(pooled_peaks_fn))
npeaks_out = common.count_lines(overlapping_peaks_fn)
npeaks_rejected = common.count_lines(rejected_peaks_fn)
# make bigBed files for visualization
overlapping_peaks_bb_fn = common.bed2bb(
overlapping_peaks_fn, chrom_sizes_fn, as_file_fn, bed_type=bed_type)
rejected_peaks_bb_fn = common.bed2bb(
rejected_peaks_fn, chrom_sizes_fn, as_file_fn, bed_type=bed_type)
# Upload file outputs from the local file system.
overlapping_peaks = dxpy.upload_local_file(common.compress(overlapping_peaks_fn))
overlapping_peaks_bb = dxpy.upload_local_file(overlapping_peaks_bb_fn)
rejected_peaks = dxpy.upload_local_file(common.compress(rejected_peaks_fn))
rejected_peaks_bb = dxpy.upload_local_file(rejected_peaks_bb_fn)
output = {
"overlapping_peaks" : dxpy.dxlink(overlapping_peaks),
"overlapping_peaks_bb" : dxpy.dxlink(overlapping_peaks_bb),
"rejected_peaks" : dxpy.dxlink(rejected_peaks),
"rejected_peaks_bb" : dxpy.dxlink(rejected_peaks_bb),
"npeaks_in" : npeaks_in,
"npeaks_out" : npeaks_out,
"npeaks_rejected" : npeaks_rejected,
"frip_nreads" : n_reads,
"frip_nreads_in_peaks" : n_reads_in_peaks,
"frip_score" : frip_score
}
return output
dat = tar_tmp_fh.read(io.DEFAULT_BUFFER_SIZE)
while dat:
targz_gzf.write(dat)
dat = tar_tmp_fh.read(io.DEFAULT_BUFFER_SIZE)
targz_gzf.flush()
targz_gzf.close()
targz_fh.close()
if 'folder' in applet_spec:
try:
dxpy.get_handler(dest_project).new_folder(applet_spec['folder'], parents=True)
except dxpy.exceptions.DXAPIError:
pass # TODO: make this better
dx_resource_archive = dxpy.upload_local_file(
targz_fh.name,
wait_on_close=True,
project=dest_project,
folder=target_folder,
hidden=True,
properties=properties_dict
)
os.unlink(targz_fh.name)
# end compressed file creation and upload
archive_link = dxpy.dxlink(dx_resource_archive.get_id())
# end tempfile.NamedTemporaryFile(suffix=".tar") as tar_fh
steps.extend([
"grep -v 'chrM'",
"sort",
"uniq -c",
r"""awk 'BEGIN{mt=0;m0=0;m1=0;m2=0} ($1==1){m1=m1+1} ($1==2){m2=m2+1} {m0=m0+1} {mt=mt+$1} END{printf "%d\t%d\t%d\t%d\t%f\t%f\t%f\n",mt,m0,m1,m2,m0/mt,m1/m0,m1/m2}'"""
])
out, err = common.run_pipe(steps, pbc_file_qc_filename)
if err:
logger.error("PBC file error: %s" % (err))
logger.info("Uploading results files to the project")
filtered_bam = dxpy.upload_local_file(final_bam_filename)
filtered_bam_index = dxpy.upload_local_file(final_bam_index_filename)
filtered_mapstats = \
dxpy.upload_local_file(final_bam_file_mapstats_filename)
dup_file = dxpy.upload_local_file(dup_file_qc_filename)
pbc_file = dxpy.upload_local_file(pbc_file_qc_filename)
dup_qc = dup_parse(dup_file_qc_filename)
pbc_qc = pbc_parse(pbc_file_qc_filename)
logger.info("dup_qc: %s" % (dup_qc))
logger.info("pbc_qc: %s" % (pbc_qc))
# Return links to the output files
output = {
"filtered_bam": dxpy.dxlink(filtered_bam),
"filtered_bam_index": dxpy.dxlink(filtered_bam_index),
"filtered_mapstats": dxpy.dxlink(filtered_mapstats),
"dup_file_qc": dxpy.dxlink(dup_file),
"pbc_file_qc": dxpy.dxlink(pbc_file),
"paired_end": paired_end,
"NRF": pbc_qc.get('NRF'),
"PBC1": pbc_qc.get('PBC1'),
# These are optional outputs to see what's being removed by the blacklist
if blacklist:
output.update({
"pre_bl_stable_set":
dxpy.dxlink(dxpy.upload_local_file(common.compress(
r1pr_peaks_filename)))}
)
# bedtobigbed often fails, so skip creating the bb if it does
stable_set_bb_filename = \
common.bed2bb(stable_set_filename, chrom_sizes_filename,
as_file_filename)
if stable_set_bb_filename:
stable_set_bb_output = \
dxpy.upload_local_file(stable_set_bb_filename)
output.update(
{"stable_set_bb": dxpy.dxlink(stable_set_bb_output)})
output.update({
"N1": N1,
"stable_set": dxpy.dxlink(dxpy.upload_local_file(common.compress(stable_set_filename))),
"Ns": Nsb
})
# These are just passed through for convenience so that signals and tracks
# are available in one place. Both input and output are optional.
if rep1_signal:
output.update({"rep1_signal": rep1_signal})
return output