How to use the dxpy.upload_local_file function in dxpy

To help you get started, we’ve selected a few dxpy examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github ENCODE-DCC / chip-seq-pipeline / dnanexus / bam2tagAlign / src / bam2tagAlign.py View on Github external
"%s sort -@ %d -n %s %s" \
            % (samtools, cpu_count(), input_bam_filename, final_nmsrt_bam_prefix)
        logger.info(command)
        subprocess.check_call(shlex.split(command))

        final_BEDPE_filename = input_bam_basename + ".bedpe.gz"
        out, err = common.run_pipe([
            "bamToBed -bedpe -mate1 -i %s" % (final_nmsrt_bam_filename),
            "gzip -cn"],
            outfile=final_BEDPE_filename)

    subprocess.check_output('ls -l', shell=True)

    tagAlign_file = dxpy.upload_local_file(final_TA_filename)
    if paired_end:
        BEDPE_file = dxpy.upload_local_file(final_BEDPE_filename)

    output = {}
    output["tagAlign_file"] = dxpy.dxlink(tagAlign_file)
    if paired_end:
        output["BEDPE_file"] = dxpy.dxlink(BEDPE_file)

    return output
github ENCODE-DCC / chip-seq-pipeline / dnanexus / idr2 / src / idr2.py View on Github external
line_count = wc_output.split()[0]
    n_peaks = int(line_count)

    #TODO batch consistency plot

    # The following line(s) use the Python bindings to upload your file outputs
    # after you have created them on the local file system.  It assumes that you
    # have used the output field name for the filename for each output, but you
    # can change that behavior to suit your needs.

    output = {}

    EM_fit_output = None
    empirical_curves_output = None
    overlapped_peaks = None
    EM_parameters_log = dxpy.upload_local_file(rep1_vs_rep2_prefix + '.log.txt')
    IDR2_plot = dxpy.upload_local_file(pooled_common_peaks_IDR_filename + '.png')
    output.update({
        "IDR2_plot": dxpy.dxlink(IDR2_plot)
        })

    npeaks_pass = dxpy.upload_local_file(npeaks_pass_filename)
    IDR_output = dxpy.upload_local_file(compress(pooled_common_peaks_IDR_filename))
    IDR_peaks = dxpy.upload_local_file(compress(final_IDR_thresholded_filename))

    #
    # return { "app_output_field": postprocess_job.get_output_ref("answer"), ...}
    #

    subprocess.check_output('ls -l', shell=True, stderr=subprocess.STDOUT)

    output.update({
github ENCODE-DCC / chip-seq-pipeline / dnanexus / accession_analysis / src / accession_analysis.py View on Github external
fieldnames = [
            'analysis',
            'experiment',
            'dx_pipeline',
            'files',
            'error'
        ]
        output_writer = csv.DictWriter(fh, fieldnames, delimiter='\t')
        output_writer.writeheader()
        for output_row in output_rows:
            output_writer.writerow(output_row)

    for line in open(outfn, 'r'):
        print(line)

    output = {"outfile": dxpy.upload_local_file(outfn)}

    return output
github dnanexus / dx-cwl / dx-cwl-applet-code.py View on Github external
def upload_file(ovalue):
                    sh("unset DX_WORKSPACE_ID && dx cd $DX_PROJECT_CONTEXT_ID: && dx mkdir -p {}".format(folder))
                    return dxpy.dxlink(dxpy.upload_local_file(remove_prefix(ovalue['location'], "file://"), wait_on_close=True, project=dxpy.PROJECT_CONTEXT_ID, folder=folder))
github dnanexus / dxWDL / scripts / util.py View on Github external
def upload_local_file(local_path, project, destFolder, **kwargs):
    for i in range(0,max_num_retries):
        try:
            return dxpy.upload_local_file(filename = local_path,
                                          project = project.get_id(),
                                          folder = destFolder,
                                          show_progress = True,
                                          wait_on_close=True,
                                          **kwargs)
        except:
            print("Sleeping for 5 seconds before trying again")
            time.sleep(5)
    raise Exception("Error uploading file {}".format(local_path))
github ENCODE-DCC / chip-seq-pipeline / dnanexus / overlap_peaks / src / overlap_peaks.py View on Github external
# count peaks
    npeaks_in        = common.count_lines(common.uncompress(pooled_peaks_fn))
    npeaks_out       = common.count_lines(overlapping_peaks_fn)
    npeaks_rejected  = common.count_lines(rejected_peaks_fn)

    # make bigBed files for visualization
    overlapping_peaks_bb_fn = common.bed2bb(
        overlapping_peaks_fn, chrom_sizes_fn, as_file_fn, bed_type=bed_type)
    rejected_peaks_bb_fn = common.bed2bb(
        rejected_peaks_fn, chrom_sizes_fn, as_file_fn, bed_type=bed_type)

    # Upload file outputs from the local file system.

    overlapping_peaks       = dxpy.upload_local_file(common.compress(overlapping_peaks_fn))
    overlapping_peaks_bb    = dxpy.upload_local_file(overlapping_peaks_bb_fn)
    rejected_peaks          = dxpy.upload_local_file(common.compress(rejected_peaks_fn))
    rejected_peaks_bb       = dxpy.upload_local_file(rejected_peaks_bb_fn)

    output = {
        "overlapping_peaks"     : dxpy.dxlink(overlapping_peaks),
        "overlapping_peaks_bb"  : dxpy.dxlink(overlapping_peaks_bb),
        "rejected_peaks"        : dxpy.dxlink(rejected_peaks),
        "rejected_peaks_bb"     : dxpy.dxlink(rejected_peaks_bb),
        "npeaks_in"             : npeaks_in,
        "npeaks_out"            : npeaks_out,
        "npeaks_rejected"       : npeaks_rejected,
        "frip_nreads"           : n_reads,
        "frip_nreads_in_peaks"  : n_reads_in_peaks,
        "frip_score"            : frip_score
    }

    return output
github dnanexus / dx-toolkit / src / python / dxpy / app_builder.py View on Github external
dat = tar_tmp_fh.read(io.DEFAULT_BUFFER_SIZE)
                while dat:
                    targz_gzf.write(dat)
                    dat = tar_tmp_fh.read(io.DEFAULT_BUFFER_SIZE)

                targz_gzf.flush()
                targz_gzf.close()
                targz_fh.close()

                if 'folder' in applet_spec:
                    try:
                        dxpy.get_handler(dest_project).new_folder(applet_spec['folder'], parents=True)
                    except dxpy.exceptions.DXAPIError:
                        pass # TODO: make this better

                dx_resource_archive = dxpy.upload_local_file(
                    targz_fh.name,
                    wait_on_close=True,
                    project=dest_project,
                    folder=target_folder,
                    hidden=True,
                    properties=properties_dict
                )

                os.unlink(targz_fh.name)

                # end compressed file creation and upload

            archive_link = dxpy.dxlink(dx_resource_archive.get_id())

        # end tempfile.NamedTemporaryFile(suffix=".tar") as tar_fh
github ENCODE-DCC / chip-seq-pipeline / dnanexus / filter_qc / src / filter_qc.py View on Github external
steps.extend([
        "grep -v 'chrM'",
        "sort",
        "uniq -c",
        r"""awk 'BEGIN{mt=0;m0=0;m1=0;m2=0} ($1==1){m1=m1+1} ($1==2){m2=m2+1} {m0=m0+1} {mt=mt+$1} END{printf "%d\t%d\t%d\t%d\t%f\t%f\t%f\n",mt,m0,m1,m2,m0/mt,m1/m0,m1/m2}'"""
        ])
    out, err = common.run_pipe(steps, pbc_file_qc_filename)
    if err:
        logger.error("PBC file error: %s" % (err))

    logger.info("Uploading results files to the project")
    filtered_bam = dxpy.upload_local_file(final_bam_filename)
    filtered_bam_index = dxpy.upload_local_file(final_bam_index_filename)
    filtered_mapstats = \
        dxpy.upload_local_file(final_bam_file_mapstats_filename)
    dup_file = dxpy.upload_local_file(dup_file_qc_filename)
    pbc_file = dxpy.upload_local_file(pbc_file_qc_filename)
    dup_qc = dup_parse(dup_file_qc_filename)
    pbc_qc = pbc_parse(pbc_file_qc_filename)
    logger.info("dup_qc: %s" % (dup_qc))
    logger.info("pbc_qc: %s" % (pbc_qc))

    # Return links to the output files
    output = {
        "filtered_bam": dxpy.dxlink(filtered_bam),
        "filtered_bam_index": dxpy.dxlink(filtered_bam_index),
        "filtered_mapstats": dxpy.dxlink(filtered_mapstats),
        "dup_file_qc": dxpy.dxlink(dup_file),
        "pbc_file_qc": dxpy.dxlink(pbc_file),
        "paired_end": paired_end,
        "NRF": pbc_qc.get('NRF'),
        "PBC1": pbc_qc.get('PBC1'),
github ENCODE-DCC / chip-seq-pipeline / dnanexus / encode_idr / src / encode_idr.py View on Github external
# These are optional outputs to see what's being removed by the blacklist
    if blacklist:
        output.update({
            "pre_bl_stable_set":
                dxpy.dxlink(dxpy.upload_local_file(common.compress(
                    r1pr_peaks_filename)))}
        )

    # bedtobigbed often fails, so skip creating the bb if it does
    stable_set_bb_filename = \
        common.bed2bb(stable_set_filename, chrom_sizes_filename,
                      as_file_filename)
    if stable_set_bb_filename:
        stable_set_bb_output = \
            dxpy.upload_local_file(stable_set_bb_filename)
        output.update(
            {"stable_set_bb": dxpy.dxlink(stable_set_bb_output)})

    output.update({
        "N1": N1,
        "stable_set": dxpy.dxlink(dxpy.upload_local_file(common.compress(stable_set_filename))),
        "Ns": Nsb
    })

    # These are just passed through for convenience so that signals and tracks
    # are available in one place.  Both input and output are optional.
    if rep1_signal:
        output.update({"rep1_signal": rep1_signal})

    return output