Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
else:
tasks, count = tasks_to_process[name]
tasks.append(task)
tasks_to_process[name] = (tasks, count + point_count)
processed_points = 0
points_in_progress = 0
previous_percent = 0
points_in_pnts = 0
max_splitting_jobs_count = max(1, jobs // 2)
# zmq setup
context = zmq.Context()
zmq_skt = context.socket(zmq.ROUTER)
zmq_skt.bind('ipc:///tmp/py3dtiles1')
zmq_idle_clients = []
state = State(infos['portions'])
zmq_processes_killed = -1
zmq_processes = [multiprocessing.Process(
target=zmq_process,
args=(
graph, projection, node_store, octree_metadata, outfolder, rgb, verbose)) for i in range(jobs)]
for p in zmq_processes:
p.start()
activities = [p.pid for p in zmq_processes]
def _receive_message(self):
"""
Internal coroutine for receiving messages
"""
while True:
try:
if self._socket.getsockopt(zmq.TYPE) == zmq.ROUTER:
zmq_identity, msg_bytes = \
yield from self._socket.recv_multipart()
self._received_from_identity(zmq_identity)
self._dispatcher_queue.put_nowait(
(zmq_identity, msg_bytes))
else:
msg_bytes = yield from self._socket.recv()
self._last_message_time = time.time()
self._dispatcher_queue.put_nowait((None, msg_bytes))
except CancelledError:
# The concurrent.futures.CancelledError is caught by asyncio
# when the Task associated with the coroutine is cancelled.
# The raise is required to stop this component.
raise
except Exception as e: # pylint: disable=broad-except
self.shell_socket = context.socket(zmq.ROUTER)
self.shell_socket.linger = 1000
self.shell_port = self._bind_socket(self.shell_socket, self.shell_port)
self.log.debug("shell ROUTER Channel on port: %i" % self.shell_port)
self.iopub_socket = context.socket(zmq.PUB)
self.iopub_socket.linger = 1000
self.iopub_port = self._bind_socket(self.iopub_socket, self.iopub_port)
self.log.debug("iopub PUB Channel on port: %i" % self.iopub_port)
self.stdin_socket = context.socket(zmq.ROUTER)
self.stdin_socket.linger = 1000
self.stdin_port = self._bind_socket(self.stdin_socket, self.stdin_port)
self.log.debug("stdin ROUTER Channel on port: %i" % self.stdin_port)
self.control_socket = context.socket(zmq.ROUTER)
self.control_socket.linger = 1000
self.control_port = self._bind_socket(self.control_socket, self.control_port)
self.log.debug("control ROUTER Channel on port: %i" % self.control_port)
def start(self):
context = zmq.Context()
frontend = context.socket(zmq.ROUTER)
frontend.bind(self.front_url)
backend = context.socket(zmq.DEALER)
backend.bind(self.back_url)
workers = []
for i in xrange(self.num_workers):
worker = Worker(context, self.db, self.back_url)
worker.start()
workers.append(worker)
poller = zmq.Poller()
poller.register(frontend, zmq.POLLIN)
poller.register(backend, zmq.POLLIN)
try:
while True:
def initialize(self):
# initialize controller
# Initialize ZMQ Sockets
self.ctrl_socket = self.context.socket(zmq.ROUTER)
self.ctrl_socket.bind(self.endpoint)
self.ctrl_socket.linger = 0
# support chown'ing the zmq endpoint on unix platforms
if self.endpoint_owner_mode:
uid = to_uid(self.endpoint_owner)
sockpath = self.endpoint[6:] # length of 'ipc://' prefix
os.chown(sockpath, uid, -1)
self._init_stream()
# Initialize UDP Socket
if self.multicast_endpoint:
self._init_multicast_endpoint()
def __init__(self, handlers=None, default_host="", transforms=None,
wsgi=False, **settings):
# ZMQWEB NOTE: This method is overriden from the base class.
# ZMQWEB NOTE: We have added new context and loop settings.
self.context = settings.pop('context', zmq.Context.instance())
self.loop = settings.pop('loop', IOLoop.instance())
super(ZMQApplication,self).__init__(
handlers=handlers, default_host=default_host,
transforms=transforms, wsgi=wsgi, **settings
)
# ZMQWEB NOTE: Here we create the zmq socket and stream and setup a
# list of urls that are bound/connected to.
self.socket = self.context.socket(zmq.ROUTER)
self.stream = ZMQStream(self.socket, self.loop)
self.stream.on_recv(self._handle_request)
self.urls = []
ident = f.session.bsession
# disambiguate url, in case of *
monitor_url = disambiguate_url(f.monitor_url)
# maybe_inproc = 'inproc://monitor' if self.use_threads else monitor_url
# IOPub relay (in a Process)
q = mq(zmq.PUB, zmq.SUB, zmq.PUB, b'N/A', b'iopub')
q.bind_in(f.client_url('iopub'))
q.setsockopt_in(zmq.IDENTITY, ident + b"_iopub")
q.bind_out(f.engine_url('iopub'))
q.setsockopt_out(zmq.SUBSCRIBE, b'')
q.connect_mon(monitor_url)
q.daemon = True
children.append(q)
# Multiplexer Queue (in a Process)
q = mq(zmq.ROUTER, zmq.ROUTER, zmq.PUB, b'in', b'out')
q.bind_in(f.client_url('mux'))
q.setsockopt_in(zmq.IDENTITY, b'mux_in')
q.bind_out(f.engine_url('mux'))
q.setsockopt_out(zmq.IDENTITY, b'mux_out')
q.connect_mon(monitor_url)
q.daemon = True
children.append(q)
# Control Queue (in a Process)
q = mq(zmq.ROUTER, zmq.ROUTER, zmq.PUB, b'incontrol', b'outcontrol')
q.bind_in(f.client_url('control'))
q.setsockopt_in(zmq.IDENTITY, b'control_in')
q.bind_out(f.engine_url('control'))
q.setsockopt_out(zmq.IDENTITY, b'control_out')
q.connect_mon(monitor_url)
def socket_type(self):
if self.mode == zmq.PULL:
return 'PULL'
elif self.mode == zmq.PUSH:
return 'PUSH'
elif self.mode == zmq.PUB:
return 'PUB'
elif self.mode == zmq.SUB:
return 'SUB'
elif self.mode == zmq.PAIR:
return 'PAIR'
elif self.mode == zmq.REQ:
return 'REQ'
elif self.mode == zmq.REP:
return 'REP'
elif self.mode == zmq.ROUTER:
return 'ROUTER'
elif self.mode == zmq.DEALER:
return 'DEALER'
def stdin_proxy_server(context, ports):
stdin_frontend = context.socket(zmq.DEALER)
stdin_frontend.bind("tcp://127.0.0.1:{}".format(ports["stdin_port"]))
stdin_backend = context.socket(zmq.ROUTER)
stdin_backend.bind("tcp://127.0.0.1:{}".format(ports["stdin_back_port"]))
zmq.device(zmq.QUEUE, stdin_frontend, stdin_backend)
def __init__(self, context, main_ep, opt_ep=None):
"""Init MDPBroker instance.
"""
l = logger.Logger('mq_broker')
self.log = l.get_logger()
self.log.info("MDP broker startup...")
socket = ZmqSocket(context, zmq.ROUTER)
socket.bind(main_ep)
self.main_stream = ZMQStream(socket)
self.main_stream.on_recv(self.on_message)
if opt_ep:
socket = ZmqSocket(context, zmq.ROUTER)
socket.bind(opt_ep)
self.client_stream = ZMQStream(socket)
self.client_stream.on_recv(self.on_message)
else:
self.client_stream = self.main_stream
self.log.debug("Socket created...")
self._workers = {}
# services contain the worker queue and the request queue
self._services = {}
self._worker_cmds = { b'\x01': self.on_ready,
b'\x03': self.on_reply,