How to use the boltons.iterutils.chunked function in boltons

To help you get started, we’ve selected a few boltons examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github AxFoundation / strax / strax / testutils.py View on Github external
s = s.map(lambda x: [tuple(q)
                         for q in iterutils.chunked(sorted(x), size=2)])
github bheinzerling / dougu / dougu / gpunode.py View on Github external
if hasattr(args, "print_configs") and args.print_configs:
            for job in jobs:
                print(job)
        if args.submit_jobs:
            random.shuffle(jobs)
            jobids = []
            try:
                n_gpus = args.n_gpus
            except AttributeError:
                n_gpus = 1
            # submit = {
            #     'slurm': slurm_submit,
            #     'grid_engine': grid_engine_submit
            #     }[args.cluster_scheduler]
            positional_arg = "command"
            for batch in chunked(jobs, args.configs_per_job):
                if args.cluster_scheduler == 'slurm':
                    jobid = slurm_submit(
                        batch,
                        positional_arg=positional_arg,
                        partition=args.partition,
                        exclude=args.exclude,
                        ntasks_per_node=args.ntasks_per_node,
                        cpus_per_task=args.cpus_per_task,
                        n_gpus=n_gpus,
                        time=args.time,
                        job_name=args.job_name)
                elif args.cluster_scheduler == 'grid_engine':
                    jobid = grid_engine_submit(
                        batch, positional_arg=positional_arg)
                print(jobid)
                time.sleep(1)
github zeroSteiner / mayhem / mayhem / utilities.py View on Github external
def print_hexdump(data, address=0, stream=None):
	"""
	Print data to stdout in a visually pleasant hex format.

	:param str data: The data to print.
	:param int address: The base address to display for *data*.
	:param stream: The object to write the data to be displayed to.
	"""
	stream = stream or sys.stdout
	data = bytearray(data)
	divider = 8
	chunk_size = 16
	for row, chunk in enumerate(boltons.iterutils.chunked(data, chunk_size, fill=-1)):
		offset_col = "{0:04x}".format((row * chunk_size) + address)
		ascii_col = ''
		hex_col = ''
		pos = 0
		for pos, byte in enumerate(chunk):
			hex_col += '   ' if byte == -1 else "{0:02x} ".format(byte)
			if divider and pos and (pos + 1) % divider == 0:
				hex_col += ' '

			if byte == -1:
				ascii_col += ' '
			elif byte < 32 or byte > 126:
				ascii_col += '.'
			else:
				ascii_col += chr(byte)
			if divider and pos and (pos + 1) % divider == 0:
github ESSS / esss_fix_format / src / esss_fix_format / cli.py View on Github external
:return: a pair (would_be_formatted, black_failed)
    """
    py_files = [x for x in files if should_format(str(x), ['*.py'], exclude_patterns)[0]]
    black_failed = False
    would_be_formatted = False
    if py_files:
        if check:
            click.secho(f'Checking black on {len(py_files)} files...', fg='cyan')
        else:
            click.secho(f'Running black on {len(py_files)} files...', fg='cyan')
        # on Windows there's a limit on the command-line size, so we call black in batches
        # this should only be an issue when executing fix-format over the entire repository,
        # not on day to day usage
        chunk_size = 100
        for chunked_files in boltons.iterutils.chunked(py_files, chunk_size):
            args = ['black']
            if check:
                args.append('--check')
            if verbose:
                args.append('--verbose')
            args.extend(str(x) for x in chunked_files)
            status = subprocess.call(args)
            if not black_failed:
                black_failed = not check and status != 0
            if not would_be_formatted:
                would_be_formatted = check and status == 1

    return would_be_formatted, black_failed
github securestate / king-phisher / king_phisher / client / export.py View on Github external
ip_counter = collections.Counter()
	for visit_node in _get_graphql_campaign_visits(rpc, campaign_id):
		visit = visit_node['node']
		ip_counter.update((visit['ip'],))
		visitor_ip = ipaddress.ip_address(visit['ip'])
		if not isinstance(visitor_ip, ipaddress.IPv4Address):
			continue
		if visitor_ip.is_loopback or visitor_ip.is_private:
			continue
		if not visitor_ip in ips_for_georesolution:
			ips_for_georesolution[visitor_ip] = visit['firstSeen']
		elif ips_for_georesolution[visitor_ip] > visit['firstSeen']:
			ips_for_georesolution[visitor_ip] = visit['firstSeen']
	ips_for_georesolution = [ip for (ip, _) in sorted(ips_for_georesolution.items(), key=lambda x: x[1])]
	locations = {}
	for ip_addresses in iterutils.chunked(ips_for_georesolution, 50):
		locations.update(rpc.geoip_lookup_multi(ip_addresses))
	points = []
	for ip, location in locations.items():
		if not (location.coordinates and location.coordinates[0] and location.coordinates[1]):
			continue
		points.append(geojson.Feature(geometry=location, properties={'count': ip_counter[ip], 'ip-address': ip}))
	feature_collection = geojson.FeatureCollection(points)
	with open(geojson_file, 'w') as file_h:
		serializers.JSON.dump(feature_collection, file_h, pretty=True)
github hatnote / top / top / get_data.py View on Github external
@tlog.wrap('info', inject_as='log_rec')
def add_extras(articles, lang, project, log_rec):
    '''\
    Add images and summaries to articles in groups.
    '''
    log_rec['len_art'] = len(articles)
    ret = []
    article_groups = chunked(articles, DEFAULT_GROUP_SIZE)
    for article_group in article_groups:
        titles = [a['article'] for a in article_group]
        images = get_images(titles, lang, project)
        summaries = get_summaries(titles, lang, project)

        for article in article_group:
            title = article['title']
            article['image_url'] = images.get(title, DEFAULT_IMAGE)
            if word_filter(title) or word_filter(article['image_url']):
                article['image_url'] = DEFAULT_IMAGE
            summary = summaries.get(title, '')
            summary = crisco.shorten(summary, lang, 400)
            article['summary'] = summary
            ret.append(article)

    log_rec.success('finished adding images/summaries for {len_art} articles')
github zeroSteiner / protocon / lib / protocon / color.py View on Github external
def print_hexdump(data, stream=None):
	stream = stream or sys.stdout
	data = bytearray(data)
	divider = 8
	chunk_size = 16
	for row, chunk in enumerate(boltons.iterutils.chunked(data, chunk_size, fill=-1)):
		offset_col = "{0:04x}".format(row * chunk_size)
		ascii_col = ''
		hex_col = ''
		pos = 0
		for pos, byte in enumerate(chunk):
			hex_col += '   ' if byte == -1 else "{0:02x} ".format(byte)
			if divider and pos and (pos + 1) % divider == 0:
				hex_col += ' '

			if byte == -1:
				ascii_col += ' '
			elif byte < 32 or byte > 126:
				ascii_col += '.'
			else:
				ascii_col += chr(byte)
			if divider and pos and (pos + 1) % divider == 0:
github aertslab / pySCENIC / src / pyscenic / aucell.py View on Github external
# Copy the contents of df_rank into this shared memory block using row-major ordering.
        array[:] = df_rnk.values.flatten(order='C')

        # The resulting AUCs are returned via a synchronize array.
        auc_mtx = Array('d', len(cells) * len(signatures))  # Double precision floats.

        # Convert the modules to modules with uniform weights if necessary.
        if noweights:
            signatures = list(map(lambda m: m.noweights(), signatures))

        # Do the analysis in separate child processes.
        chunk_size = ceil(float(len(signatures)) / num_workers)
        processes = [Process(target=_enrichment, args=(shared_ro_memory_array, chunk,
                                                       genes, cells, auc_threshold,
                                                       auc_mtx, (chunk_size*len(cells))*idx))
                     for idx, chunk in enumerate(chunked(signatures, chunk_size))]
        for p in processes:
            p.start()
        for p in processes:
            p.join()

        # Reconstitute the results array. Using C or row-major ordering.
        aucs = pd.DataFrame(data=np.ctypeslib.as_array(auc_mtx.get_obj()).reshape(len(signatures), len(cells)),
                            columns=pd.Index(data=cells, name='Cell'),
                            index=pd.Index(data=list(map(attrgetter("name"), signatures)), name='Regulon')).T
    return aucs/aucs.max(axis=0) if normalize else aucs