How to use the multiprocessing.Pipe function in multiprocessing

To help you get started, we’ve selected a few multiprocessing examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github mila-iqia / babyai / babyai / multienv / multienv.py View on Github external
def _init_connections(self):
        self.locals, self.remotes = zip(*[mp.Pipe() for _ in range(self.num_menvs)])
github serverdensity / sd-agent / win32 / agent.py View on Github external
self.hWaitStop = win32event.CreateEvent(None, 0, 0, None)
        config = get_config(parse_args=False)

        # Setup the correct options so the agent will use the forwarder
        opts, args = Values({
            'autorestart': False,
            'sd_account': None,
            'use_forwarder': True,
            'disabled_dd': False,
            'profile': False
        }), []
        agentConfig = get_config(parse_args=False, options=opts)
        self.hostname = get_hostname(agentConfig)

        # Watchdog for Windows
        self._collector_heartbeat, self._collector_send_heartbeat = multiprocessing.Pipe(False)
        self._collector_failed_heartbeats = 0
        self._max_failed_heartbeats = \
            MAX_FAILED_HEARTBEATS * agentConfig['check_freq'] / SERVICE_SLEEP_INTERVAL

        # Watch JMXFetch restarts
        self._MAX_JMXFETCH_RESTARTS = 3
        self._count_jmxfetch_restarts = 0

        # Keep a list of running processes so we can start/end as needed.
        # Processes will start started in order and stopped in reverse order.
        self.procs = {
            # 'forwarder': ProcessWatchDog("forwarder", DDForwarder(config, self.hostname)),
            'collector': ProcessWatchDog("collector", DDAgent(agentConfig, self.hostname,
                                         heartbeat=self._collector_send_heartbeat)),
            # 'dogstatsd': ProcessWatchDog("dogstatsd", DogstatsdProcess(config, self.hostname)),
github Xion / seejoo / src / seejoo / commands / standard.py View on Github external
@command('c') 
def evaluate_expression(exp):
    '''
    Evaluates given expression.
    '''    
    if not exp:         return "No expression supplied."
    exp = str(exp)
    
    # Setup evaluation process if it's not present
    global eval_process, eval_pipe_parent, eval_pipe_child
    if not eval_process:
        eval_pipe_parent, eval_pipe_child = Pipe()
        eval_process = Process(name = "seejoo_eval", target = _eval_worker, args = (eval_pipe_child,))
        eval_process.daemon = True
        eval_process.start()
    
    # Push expression through the pipe and wait for result
    eval_pipe_parent.send(exp)
    if eval_pipe_parent.poll(EVAL_TIMEOUT):
        res = str(eval_pipe_parent.recv())
        res = filter(lambda x: ord(x) >= 32, res)   # Sanitize result
        return res        
    else:
        # Evaluation timed out; kill the process and return error
        os.kill(eval_process.pid, 9)    # This will leave defunct process; take care of it later
        eval_process = None
        return "Operation timed out."
github jopohl / urh / src / urh / dev / native / Device.py View on Github external
def start_rx_mode(self):
        self.init_recv_buffer()
        self.parent_data_conn, self.child_data_conn = Pipe(duplex=False)
        self.parent_ctrl_conn, self.child_ctrl_conn = Pipe()

        self.is_receiving = True
        logger.info("{0}: Starting RX Mode".format(self.__class__.__name__))
        self.receive_process = Process(target=self.receive_process_function,
                                       args=self.receive_process_arguments)
        self.receive_process.daemon = True
        self._start_read_rcv_buffer_thread()
        self._start_read_message_thread()
        try:
            self.receive_process.start()
        except OSError as e:
            logger.error(repr(e))
            self.device_messages.append(repr(e))
github openmodal / OpenModal / OpenModal / daqprocess.py View on Github external
def start_process(self):
        """Start a separate measurement process, do not start collecting the data just yet (run_measurement)."""
        # . Prepare separate measurement process.

        # Trigger, keeps the thread alive.
        # self.live_flag = mp.Value('b', True)
        self.live_flag.value = True

        # .. Trigger, starts the measurement - shared variable.
        self.run_flag = mp.Value('b', False)

        # .. Trigger, signals the signal level trigger was tripped. This trigger is picked up in the parent app.
        self.triggered = mp.Value('b', False)

        # Pipe to send parameters to the separate thread.
        self.properties_out, self.properties_in = mp.Pipe(False)

        # Pipe to send sampling_rate through. Later maybe also other data.
        self.task_info_out, self.task_info_in = mp.Pipe(False)

        # Start the thread. Pipe end and shared variables are passed.
        self.process = mp.Process(target=ThreadedDAQ, args=(self.live_flag, self.run_flag, self.properties_out,
                                                            self.task_info_in, self.triggered))
        self.process.start()
github sailthru / relay.mesos / relay_mesos / main.py View on Github external
log.warn(
            "You didn't define '--mesos_task_resources'."
            "  Tasks may not start on slaves",
            extra=dict(mesos_framework_name=ns.mesos_framework_name))
    log.info(
        "Starting Relay Mesos!",
        extra={k: str(v) for k, v in ns.__dict__.items()})

    # a distributed value storing the num and type of tasks mesos scheduler
    # should create at any given moment in time.
    # Sign of MV determines task type: warmer or cooler
    # ie. A positive value of n means n warmer tasks
    MV = mp.Array('d', [0, 0])  # max_val is a ctypes.c_int64

    # store exceptions that may be raised
    exception_receiver, exception_sender = mp.Pipe(False)
    # notify relay when mesos framework is ready
    mesos_ready = mp.Condition()

    # copy and then override warmer and cooler
    ns_relay = ns.__class__(**{k: v for k, v in ns.__dict__.items()})
    if ns.warmer:
        ns_relay.warmer = warmer_cooler_wrapper(MV, ns)
    if ns.cooler:
        ns_relay.cooler = warmer_cooler_wrapper(MV, ns)

    mesos_name = "Relay.Mesos Scheduler"
    mesos = mp.Process(
        target=catch(init_mesos_scheduler, exception_sender),
        kwargs=dict(ns=ns, MV=MV, exception_sender=exception_sender,
                    mesos_ready=mesos_ready),
        name=mesos_name)
github zhkzyth / BT-Share / libs / util.py View on Github external
def call_subprocess(self, func, callback=None, args=[], kwargs={}):
        self.ioloop = tornado.ioloop.IOLoop.instance()
        self.pipe, child_conn = Pipe()

        def wrap(func, pipe, args, kwargs):
            try:
                pipe.send(func(*args, **kwargs))
            except Exception, e:
                logging.error(traceback.format_exc())
                pipe.send(e)

        self.ioloop.add_handler(self.pipe.fileno(),
                  self.async_callback(self.on_pipe_result, callback),
                  self.ioloop.READ)
        thread.start_new_thread(wrap, (func, child_conn, args, kwargs))
github FFMG / myoddweb.piger / monitor / api / python / Lib / concurrent / futures / process.py View on Github external
def __init__(self):
        self._reader, self._writer = mp.Pipe(duplex=False)
github peteshadbolt / qy / qy / hardware / wrappers / photon_elf.py View on Github external
def __init__(self):
        ''' Constructor '''
        # Start up the GUI process and build the communication network
        self.pipe, their_pipe = Pipe()
        self.gui = Process(target=photon_elf, name='photon_elf', args=(their_pipe,))
        self.gui.start()
github ANSSI-FR / packetweaver / packetweaver / abilities / osi / transport_l4 / tls_server.py View on Github external
def _accept_new_connection(self, s):
        # accepting the connection
        clt_sock, clt_info = s.accept()

        # Getting the service ability
        new_abl = self.callback()

        # Giving to the service ability the informations about the client
        new_abl.set_opt(self.client_info_name, '{}:{}'.format(clt_info[0],
                                                              clt_info[1]))

        # Creating the pipes
        in_pipe_in, in_pipe_out = multiprocessing.Pipe()
        out_pipe_in, out_pipe_out = multiprocessing.Pipe()
        new_abl.add_in_pipe(in_pipe_out)
        new_abl.add_out_pipe(out_pipe_in)

        # Starting the service ability
        new_abl.start()

        return clt_sock, in_pipe_in, out_pipe_out, new_abl