How to use the multiprocess.Pool function in multiprocess

To help you get started, we’ve selected a few multiprocess examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github cvxgrp / cvxportfolio / cvxportfolio / simulator.py View on Github external
def run_multiple_backtest(self, initial_portf, start_time,
                              end_time, policies,
                              loglevel=logging.WARNING, parallel=True):
        """Backtest multiple policies.
        """

        def _run_backtest(policy):
            return self.run_backtest(initial_portf, start_time, end_time,
                                     policy, loglevel=loglevel)

        num_workers = min(multiprocess.cpu_count(), len(policies))
        if parallel:
            workers = multiprocess.Pool(num_workers)
            results = workers.map(_run_backtest, policies)
            workers.close()
            return results
        else:
            return list(map(_run_backtest, policies))
github ttinoco / GRIDOPT / gridopt / stochastic / two_stage_DCOPF / saa_cutting_plane_risk.py View on Github external
z_inf = params['z_inf']
        quiet = params['quiet']
        gamma = params['gamma']

        # Problem
        problemRA = self.create_problem(net,params)
        problem = problemRA.ts_dcopf
        self.problem = problemRA
        if not quiet:
            problemRA.show()

        # Scenarios
        scenarios = [problem.sample_w() for i in range(num_sce)]

        # Pool
        pool = Pool(num_procs)

        # Constants
        num_p = problem.num_p
        H0 = problem.H0
        g0 = problem.g0
        p_min = problem.p_min
        p_max = problem.p_max
        t_min = params['t_min']*problemRA.Qref
        t_max = params['t_max']*problemRA.Qref
        op = np.zeros(num_p)

        # Header
        if not quiet:
            print('\nSAA Cutting-Plane Risk')
            print('-----------------------')
            print('{0:^8s}'.format('iter'), end=' ')
github uqfoundation / multiprocess / py2.6 / multiprocess / managers.py View on Github external
of threads, plus `dict`, `list` and `Namespace`.

    The `multiprocessing.Manager()` function creates started instances of
    this class.
    '''

SyncManager.register('Queue', Queue.Queue)
SyncManager.register('JoinableQueue', Queue.Queue)
SyncManager.register('Event', threading.Event, EventProxy)
SyncManager.register('Lock', threading.Lock, AcquirerProxy)
SyncManager.register('RLock', threading.RLock, AcquirerProxy)
SyncManager.register('Semaphore', threading.Semaphore, AcquirerProxy)
SyncManager.register('BoundedSemaphore', threading.BoundedSemaphore,
                     AcquirerProxy)
SyncManager.register('Condition', threading.Condition, ConditionProxy)
SyncManager.register('Pool', Pool, PoolProxy)
SyncManager.register('list', list, ListProxy)
SyncManager.register('dict', dict, DictProxy)
SyncManager.register('Value', Value, ValueProxy)
SyncManager.register('Array', Array, ArrayProxy)
SyncManager.register('Namespace', Namespace, NamespaceProxy)

# types returned by methods of PoolProxy
SyncManager.register('Iterator', proxytype=IteratorProxy, create_method=False)
SyncManager.register('AsyncResult', create_method=False)
github simetenn / uncertainpy / src / uncertainpy / core / run_model.py View on Github external
"""
        if self.model.suppress_graphics:
            if not prerequisites:
                raise ImportError("Running with suppress_graphics require: xvfbwrapper")

            vdisplay = Xvfb()
            vdisplay.start()

        results = []

        model_parameters = self.create_model_parameters(nodes, uncertain_parameters)

        if self.CPUs:
            import multiprocess as mp

            pool = mp.Pool(processes=self.CPUs)

            # pool.map(self._parallel.run, model_parameters)
            # chunksize = int(np.ceil(len(model_parameters)/self.CPUs))
            chunksize = 1
            for result in tqdm(pool.imap(self._parallel.run, model_parameters, chunksize),
                               desc="Running model",
                               total=len(nodes.T)):

                results.append(result)

            pool.close()

        else:
            for result in tqdm(imap(self._parallel.run, model_parameters),
                               desc="Running model",
                               total=len(nodes.T)):
github spencerahill / aospy / examples / main.py View on Github external
def main(main_params, exec_calcs=True, prompt_verify=True):
    """Main script for interfacing with aospy."""
    # Instantiate objects and load default/all models, runs, and regions.
    cs = CalcSuite(MainParamsParser(main_params, projs))
    cs.print_params()
    if prompt_verify:
        try:
            cs.prompt_user_verify()
        except IOError as e:
            logging.warn(repr(e))
            return
    param_combos = cs.create_params_all_calcs()
    if main_params.parallelize and exec_calcs:
        calcs = cs.create_calcs(param_combos, exec_calcs=False)
        p = multiprocess.Pool()
        return p.map(lambda calc: calc.compute(), calcs)
    else:
        calcs = cs.create_calcs(param_combos, exec_calcs=exec_calcs)
    return calcs
github citp / BlockSci / blockscipy / blocksci / __init__.py View on Github external
raw_segments = chain._segment_indexes(start, end, cpu_count)
    config = chain._config

    segments = [(raw_segment, config) for raw_segment in raw_segments]

    def real_map_func(input):
        local_chain = Blockchain(input[1])
        file = io.BytesIO()
        pickler = Pickler(file)
        mapped = mapFunc(local_chain[input[0][0]:input[0][1]])
        pickler.dump(mapped)
        file.seek(0)
        return file

    with Pool(cpu_count - 1) as p:
        results_future = p.map_async(real_map_func, segments[1:])
        first = mapFunc(chain[raw_segments[0][0]:raw_segments[0][1]])
        results = results_future.get()
        results = [Unpickler(res, chain).load() for res in results]
    results.insert(0, first)
    if type(init) == type(missing_param): 
        return reduce(reduceFunc, results)
    else:
        return reduce(reduceFunc, results, init)
github artur-deluca / psopt / psopt / commons / optimizer.py View on Github external
def _optimize(self):

        start = time.time()
        if not self._n_jobs or self._n_jobs > 1:
            pool = multiprocess.Pool(self._n_jobs)
        else:
            pool = MockPool()

        # Initialize storage arrays
        self._init_storage_fields()

        # Generate particles
        iteration = 0
        seeds = get_seeds(self.swarm_population)
        self._generate_particles(pool, seeds)

        while iteration < self._max_iter:

            # Add empty placeholders for the calculation by copying templates
            # already defined at `init_storage_fields`
            self._particles.append(self._template_position.copy())
github yasufumy / pipelib / pipelib / parallel.py View on Github external
def __call__(self, dataset):
        with multiprocess.Pool(self._n) as p:
            yield from chain.from_iterable(
                getattr(p, self._map_method)(
                    self._func, dataset, self._chunksize))
github ttinoco / GRIDOPT / gridopt / stochastic / multi_stage_DCOPF / problem.py View on Github external
if not outfile:
            outfile = 'evaluation.csv'

        csvfile = open(outfile,'wb')
        writer = csv.writer(csvfile)

        np.random.seed(seed)

        print('Evaluating policies with %d processes' %num_procs)
                   
        # Eval
        self.policies = policies
        self.samples = [self.sample_W(self.T-1) for j in range(num_sims)]
        if num_procs > 1:
            pool = Pool(num_procs)
            func = pool.map
        else:
            func = map
        t0 = time.time()
        results = func(lambda i: self.simulate_policies(i), range(num_sims))
        t1 = time.time()            
        print('Total time: %.2f min' %((t1-t0)/60.))

        # Process
        num_pol = len(policies)
        dtot,rtot,cost,ptot,qtot,stot = list(zip(*results))
        dtot = np.average(np.array(dtot),axis=0)
        rtot = np.average(np.array(rtot),axis=0)
        cost = dict([(i,np.average(np.array([cost[j][i] for j in range(num_sims)]),axis=0)) for i in range(num_pol)])
        ptot = dict([(i,np.average(np.array([ptot[j][i] for j in range(num_sims)]),axis=0)) for i in range(num_pol)])
        qtot = dict([(i,np.average(np.array([qtot[j][i] for j in range(num_sims)]),axis=0)) for i in range(num_pol)])