Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
# specifically formatted for FF health page
ret = {}
split_str = count_str.split()
ret[split_str[0].strip(':')] = int(split_str[1])
ret[split_str[2].strip(':')] = int(split_str[3])
return ret
check = CheckResult(connection, 'item_counts_by_type')
# add random wait
wait = round(random.uniform(0.1, random_wait), 1)
time.sleep(wait)
# run the check
item_counts = {}
warn_item_counts = {}
req_location = ''.join([connection.ff_server, 'counts?format=json'])
counts_res = ff_utils.authorized_request(req_location, auth=connection.ff_keys)
if counts_res.status_code >= 400:
check.status = 'ERROR'
check.description = 'Error (bad status code %s) connecting to the counts endpoint at: %s.' % (counts_res.status_code, req_location)
return check
counts_json = json.loads(counts_res.text)
for index in counts_json['db_es_compare']:
counts = process_counts(counts_json['db_es_compare'][index])
item_counts[index] = counts
if counts['DB'] != counts['ES']:
warn_item_counts[index] = counts
# add ALL for total counts
total_counts = process_counts(counts_json['db_es_total'])
item_counts['ALL'] = total_counts
# set fields, store result
if not item_counts:
check.status = 'FAIL'
# specifically formatted for FF health page
ret = {}
split_str = count_str.split()
ret[split_str[0].strip(':')] = int(split_str[1])
ret[split_str[2].strip(':')] = int(split_str[3])
return ret
check = CheckResult(connection, 'item_counts_by_type')
# add random wait
wait = round(random.uniform(0.1, random_wait), 1)
time.sleep(wait)
# run the check
item_counts = {}
warn_item_counts = {}
req_location = ''.join([connection.ff_server, 'counts?format=json'])
counts_res = ff_utils.authorized_request(req_location, auth=connection.ff_keys)
if counts_res.status_code >= 400:
check.status = 'ERROR'
check.description = 'Error (bad status code %s) connecting to the counts endpoint at: %s.' % (counts_res.status_code, req_location)
return check
counts_json = json.loads(counts_res.text)
for index in counts_json['db_es_compare']:
counts = process_counts(counts_json['db_es_compare'][index])
item_counts[index] = counts
if counts['DB'] != counts['ES']:
warn_item_counts[index] = counts
# add ALL for total counts
total_counts = process_counts(counts_json['db_es_total'])
item_counts['ALL'] = total_counts
# set fields, store result
if not item_counts:
check.status = 'FAIL'
'bar_plot_aggregations/type=ExperimentSetReplicate&experimentset_type=replicate/?field=experiments_in_set.experiment_type',
'browse/?type=ExperimentSetReplicate&experimentset_type=replicate',
'experiment-set-replicates/4DNESIE5R9HS/',
'experiment-set-replicates/4DNESIE5R9HS/?datastore=database',
'experiment-set-replicates/4DNESQWI9K2F/',
'experiment-set-replicates/4DNESQWI9K2F/?datastore=database',
'workflow-runs-awsem/ba50d240-5312-4aa7-b600-6b18d8230311/',
'workflow-runs-awsem/ba50d240-5312-4aa7-b600-6b18d8230311/?datastore=database',
'files-fastq/4DNFIX75FSJM/',
'files-fastq/4DNFIX75FSJM/?datastore=database'
]
for check_url in check_urls:
performance[check_url] = {}
try:
# set timeout really high
ff_resp = ff_utils.authorized_request(connection.ff_server + check_url,
auth=connection.ff_keys, timeout=1000)
except Exception as e:
performance[check_url]['error'] = str(e)
if ff_resp and hasattr(ff_resp, 'headers') and 'X-stats' in ff_resp.headers:
x_stats = ff_resp.headers['X-stats']
if not isinstance(x_stats, basestring):
performance[check_url]['error'] = 'Stats response is not a string.'
continue
# X-stats in form: 'db_count=148&db_time=1215810&es_count=4& ... '
split_stats = x_stats.strip().split('&')
parse_stats = [stat.split('=') for stat in split_stats]
# stats can be strings or integers
for stat in parse_stats:
if not len(stat) == 2:
continue
try:
time_limit = kwargs['time_limit']
t0 = time.time()
sent = 0
deleted = 0
deduplicated = 0
total_msgs = 0
replaced = 0
repeat_replaced = 0
problem_msgs = []
elapsed = round(time.time() - t0, 2)
failed = []
seen_uuids = set()
# this is a bit of a hack -- send maximum sid with every message we replace
# get the maximum sid at the start of deduplication and update it if we
# encounter a higher sid
max_sid_resp = ff_utils.authorized_request(connection.ff_server + 'max-sid',
auth=connection.ff_keys).json()
if max_sid_resp['status'] != 'success':
check.status = 'FAIL'
check.summary = 'Could not retrieve max_sid from the server'
return check
max_sid = max_sid_resp['max_sid']
exit_reason = 'out of time'
dedup_msg = 'FS dedup uuid: %s' % kwargs['uuid']
while elapsed < time_limit:
# end if we are spinning our wheels replacing the same uuids
if (replaced + repeat_replaced) >= starting_count:
exit_reason = 'starting uuids fully covered'
break
send_uuids = set()
to_send = []