How to use the dcicutils.ff_utils.authorized_request function in dcicutils

To help you get started, we’ve selected a few dcicutils examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github 4dn-dcic / foursight / chalicelib / checks / wrangler_checks.py View on Github external
# specifically formatted for FF health page
        ret = {}
        split_str = count_str.split()
        ret[split_str[0].strip(':')] = int(split_str[1])
        ret[split_str[2].strip(':')] = int(split_str[3])
        return ret

    check = CheckResult(connection, 'item_counts_by_type')
    # add random wait
    wait = round(random.uniform(0.1, random_wait), 1)
    time.sleep(wait)
    # run the check
    item_counts = {}
    warn_item_counts = {}
    req_location = ''.join([connection.ff_server, 'counts?format=json'])
    counts_res = ff_utils.authorized_request(req_location, auth=connection.ff_keys)
    if counts_res.status_code >= 400:
        check.status = 'ERROR'
        check.description = 'Error (bad status code %s) connecting to the counts endpoint at: %s.' % (counts_res.status_code, req_location)
        return check
    counts_json = json.loads(counts_res.text)
    for index in counts_json['db_es_compare']:
        counts = process_counts(counts_json['db_es_compare'][index])
        item_counts[index] = counts
        if counts['DB'] != counts['ES']:
            warn_item_counts[index] = counts
    # add ALL for total counts
    total_counts = process_counts(counts_json['db_es_total'])
    item_counts['ALL'] = total_counts
    # set fields, store result
    if not item_counts:
        check.status = 'FAIL'
github 4dn-dcic / foursight / chalicelib / checks / wrangler_checks.py View on Github external
# specifically formatted for FF health page
        ret = {}
        split_str = count_str.split()
        ret[split_str[0].strip(':')] = int(split_str[1])
        ret[split_str[2].strip(':')] = int(split_str[3])
        return ret

    check = CheckResult(connection, 'item_counts_by_type')
    # add random wait
    wait = round(random.uniform(0.1, random_wait), 1)
    time.sleep(wait)
    # run the check
    item_counts = {}
    warn_item_counts = {}
    req_location = ''.join([connection.ff_server, 'counts?format=json'])
    counts_res = ff_utils.authorized_request(req_location, auth=connection.ff_keys)
    if counts_res.status_code >= 400:
        check.status = 'ERROR'
        check.description = 'Error (bad status code %s) connecting to the counts endpoint at: %s.' % (counts_res.status_code, req_location)
        return check
    counts_json = json.loads(counts_res.text)
    for index in counts_json['db_es_compare']:
        counts = process_counts(counts_json['db_es_compare'][index])
        item_counts[index] = counts
        if counts['DB'] != counts['ES']:
            warn_item_counts[index] = counts
    # add ALL for total counts
    total_counts = process_counts(counts_json['db_es_total'])
    item_counts['ALL'] = total_counts
    # set fields, store result
    if not item_counts:
        check.status = 'FAIL'
github 4dn-dcic / foursight / chalicelib / checks / system_checks.py View on Github external
'bar_plot_aggregations/type=ExperimentSetReplicate&experimentset_type=replicate/?field=experiments_in_set.experiment_type',
        'browse/?type=ExperimentSetReplicate&experimentset_type=replicate',
        'experiment-set-replicates/4DNESIE5R9HS/',
        'experiment-set-replicates/4DNESIE5R9HS/?datastore=database',
        'experiment-set-replicates/4DNESQWI9K2F/',
        'experiment-set-replicates/4DNESQWI9K2F/?datastore=database',
        'workflow-runs-awsem/ba50d240-5312-4aa7-b600-6b18d8230311/',
        'workflow-runs-awsem/ba50d240-5312-4aa7-b600-6b18d8230311/?datastore=database',
        'files-fastq/4DNFIX75FSJM/',
        'files-fastq/4DNFIX75FSJM/?datastore=database'
    ]
    for check_url in check_urls:
        performance[check_url] = {}
        try:
            # set timeout really high
            ff_resp = ff_utils.authorized_request(connection.ff_server + check_url,
                                                  auth=connection.ff_keys, timeout=1000)
        except Exception as e:
            performance[check_url]['error'] = str(e)
        if ff_resp and hasattr(ff_resp, 'headers') and 'X-stats' in ff_resp.headers:
            x_stats = ff_resp.headers['X-stats']
            if not isinstance(x_stats, basestring):
                performance[check_url]['error'] = 'Stats response is not a string.'
                continue
            # X-stats in form: 'db_count=148&db_time=1215810&es_count=4& ... '
            split_stats = x_stats.strip().split('&')
            parse_stats = [stat.split('=') for stat in split_stats]
            # stats can be strings or integers
            for stat in parse_stats:
                if not len(stat) == 2:
                    continue
                try:
github 4dn-dcic / foursight / chalicelib / checks / system_checks.py View on Github external
time_limit = kwargs['time_limit']
    t0 = time.time()
    sent = 0
    deleted = 0
    deduplicated = 0
    total_msgs = 0
    replaced = 0
    repeat_replaced = 0
    problem_msgs = []
    elapsed = round(time.time() - t0, 2)
    failed = []
    seen_uuids = set()
    # this is a bit of a hack -- send maximum sid with every message we replace
    # get the maximum sid at the start of deduplication and update it if we
    # encounter a higher sid
    max_sid_resp = ff_utils.authorized_request(connection.ff_server + 'max-sid',
                                               auth=connection.ff_keys).json()
    if max_sid_resp['status'] != 'success':
        check.status = 'FAIL'
        check.summary = 'Could not retrieve max_sid from the server'
        return check
    max_sid = max_sid_resp['max_sid']

    exit_reason = 'out of time'
    dedup_msg = 'FS dedup uuid: %s' % kwargs['uuid']
    while elapsed < time_limit:
        # end if we are spinning our wheels replacing the same uuids
        if (replaced + repeat_replaced) >= starting_count:
            exit_reason = 'starting uuids fully covered'
            break
        send_uuids = set()
        to_send = []