Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
s = s.map(lambda x: [tuple(q)
for q in iterutils.chunked(sorted(x), size=2)])
if hasattr(args, "print_configs") and args.print_configs:
for job in jobs:
print(job)
if args.submit_jobs:
random.shuffle(jobs)
jobids = []
try:
n_gpus = args.n_gpus
except AttributeError:
n_gpus = 1
# submit = {
# 'slurm': slurm_submit,
# 'grid_engine': grid_engine_submit
# }[args.cluster_scheduler]
positional_arg = "command"
for batch in chunked(jobs, args.configs_per_job):
if args.cluster_scheduler == 'slurm':
jobid = slurm_submit(
batch,
positional_arg=positional_arg,
partition=args.partition,
exclude=args.exclude,
ntasks_per_node=args.ntasks_per_node,
cpus_per_task=args.cpus_per_task,
n_gpus=n_gpus,
time=args.time,
job_name=args.job_name)
elif args.cluster_scheduler == 'grid_engine':
jobid = grid_engine_submit(
batch, positional_arg=positional_arg)
print(jobid)
time.sleep(1)
def print_hexdump(data, address=0, stream=None):
"""
Print data to stdout in a visually pleasant hex format.
:param str data: The data to print.
:param int address: The base address to display for *data*.
:param stream: The object to write the data to be displayed to.
"""
stream = stream or sys.stdout
data = bytearray(data)
divider = 8
chunk_size = 16
for row, chunk in enumerate(boltons.iterutils.chunked(data, chunk_size, fill=-1)):
offset_col = "{0:04x}".format((row * chunk_size) + address)
ascii_col = ''
hex_col = ''
pos = 0
for pos, byte in enumerate(chunk):
hex_col += ' ' if byte == -1 else "{0:02x} ".format(byte)
if divider and pos and (pos + 1) % divider == 0:
hex_col += ' '
if byte == -1:
ascii_col += ' '
elif byte < 32 or byte > 126:
ascii_col += '.'
else:
ascii_col += chr(byte)
if divider and pos and (pos + 1) % divider == 0:
:return: a pair (would_be_formatted, black_failed)
"""
py_files = [x for x in files if should_format(str(x), ['*.py'], exclude_patterns)[0]]
black_failed = False
would_be_formatted = False
if py_files:
if check:
click.secho(f'Checking black on {len(py_files)} files...', fg='cyan')
else:
click.secho(f'Running black on {len(py_files)} files...', fg='cyan')
# on Windows there's a limit on the command-line size, so we call black in batches
# this should only be an issue when executing fix-format over the entire repository,
# not on day to day usage
chunk_size = 100
for chunked_files in boltons.iterutils.chunked(py_files, chunk_size):
args = ['black']
if check:
args.append('--check')
if verbose:
args.append('--verbose')
args.extend(str(x) for x in chunked_files)
status = subprocess.call(args)
if not black_failed:
black_failed = not check and status != 0
if not would_be_formatted:
would_be_formatted = check and status == 1
return would_be_formatted, black_failed
ip_counter = collections.Counter()
for visit_node in _get_graphql_campaign_visits(rpc, campaign_id):
visit = visit_node['node']
ip_counter.update((visit['ip'],))
visitor_ip = ipaddress.ip_address(visit['ip'])
if not isinstance(visitor_ip, ipaddress.IPv4Address):
continue
if visitor_ip.is_loopback or visitor_ip.is_private:
continue
if not visitor_ip in ips_for_georesolution:
ips_for_georesolution[visitor_ip] = visit['firstSeen']
elif ips_for_georesolution[visitor_ip] > visit['firstSeen']:
ips_for_georesolution[visitor_ip] = visit['firstSeen']
ips_for_georesolution = [ip for (ip, _) in sorted(ips_for_georesolution.items(), key=lambda x: x[1])]
locations = {}
for ip_addresses in iterutils.chunked(ips_for_georesolution, 50):
locations.update(rpc.geoip_lookup_multi(ip_addresses))
points = []
for ip, location in locations.items():
if not (location.coordinates and location.coordinates[0] and location.coordinates[1]):
continue
points.append(geojson.Feature(geometry=location, properties={'count': ip_counter[ip], 'ip-address': ip}))
feature_collection = geojson.FeatureCollection(points)
with open(geojson_file, 'w') as file_h:
serializers.JSON.dump(feature_collection, file_h, pretty=True)
@tlog.wrap('info', inject_as='log_rec')
def add_extras(articles, lang, project, log_rec):
'''\
Add images and summaries to articles in groups.
'''
log_rec['len_art'] = len(articles)
ret = []
article_groups = chunked(articles, DEFAULT_GROUP_SIZE)
for article_group in article_groups:
titles = [a['article'] for a in article_group]
images = get_images(titles, lang, project)
summaries = get_summaries(titles, lang, project)
for article in article_group:
title = article['title']
article['image_url'] = images.get(title, DEFAULT_IMAGE)
if word_filter(title) or word_filter(article['image_url']):
article['image_url'] = DEFAULT_IMAGE
summary = summaries.get(title, '')
summary = crisco.shorten(summary, lang, 400)
article['summary'] = summary
ret.append(article)
log_rec.success('finished adding images/summaries for {len_art} articles')
def print_hexdump(data, stream=None):
stream = stream or sys.stdout
data = bytearray(data)
divider = 8
chunk_size = 16
for row, chunk in enumerate(boltons.iterutils.chunked(data, chunk_size, fill=-1)):
offset_col = "{0:04x}".format(row * chunk_size)
ascii_col = ''
hex_col = ''
pos = 0
for pos, byte in enumerate(chunk):
hex_col += ' ' if byte == -1 else "{0:02x} ".format(byte)
if divider and pos and (pos + 1) % divider == 0:
hex_col += ' '
if byte == -1:
ascii_col += ' '
elif byte < 32 or byte > 126:
ascii_col += '.'
else:
ascii_col += chr(byte)
if divider and pos and (pos + 1) % divider == 0:
# Copy the contents of df_rank into this shared memory block using row-major ordering.
array[:] = df_rnk.values.flatten(order='C')
# The resulting AUCs are returned via a synchronize array.
auc_mtx = Array('d', len(cells) * len(signatures)) # Double precision floats.
# Convert the modules to modules with uniform weights if necessary.
if noweights:
signatures = list(map(lambda m: m.noweights(), signatures))
# Do the analysis in separate child processes.
chunk_size = ceil(float(len(signatures)) / num_workers)
processes = [Process(target=_enrichment, args=(shared_ro_memory_array, chunk,
genes, cells, auc_threshold,
auc_mtx, (chunk_size*len(cells))*idx))
for idx, chunk in enumerate(chunked(signatures, chunk_size))]
for p in processes:
p.start()
for p in processes:
p.join()
# Reconstitute the results array. Using C or row-major ordering.
aucs = pd.DataFrame(data=np.ctypeslib.as_array(auc_mtx.get_obj()).reshape(len(signatures), len(cells)),
columns=pd.Index(data=cells, name='Cell'),
index=pd.Index(data=list(map(attrgetter("name"), signatures)), name='Regulon')).T
return aucs/aucs.max(axis=0) if normalize else aucs