Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def run(self):
context = zmq.Context()
push_socket = context.socket(zmq.PUSH)
pull_socket = context.socket(zmq.PULL)
identity = u'worker-%d' % self.id
push_socket.identity = identity.encode('ascii')
push_socket.connect('tcp://localhost:5555')
pull_socket.identity = identity.encode('ascii')
pull_socket.connect('tcp://localhost:5556')
print('Client %s started' % (identity))
poll = zmq.Poller()
poll.register(pull_socket, zmq.POLLIN)
reqs = 0
while True:
reqs = reqs + 1
if reqs % 100 == 0:
print('Req #%d sent..' % (reqs))
_, frame = self.stream.read()
md = dict(
dtype = str(frame.dtype),
shape = frame.shape,
)
push_socket.send_json(md, zmq.SNDMORE)
push_socket.send(memoryview(frame.data), 0, copy=True, track=False)
sockets = dict(poll.poll(10))
if pull_socket in sockets:
exec_status_socket = context.socket(zmq.PULL)
exec_status_socket.bind(sutils.BIND_ADDR_TEMPLATE % (sutils.STATUS_PORT))
sched_update_socket = context.socket(zmq.PULL)
sched_update_socket.bind(sutils.BIND_ADDR_TEMPLATE %
(sutils.SCHED_UPDATE_PORT))
pin_accept_socket = context.socket(zmq.PULL)
pin_accept_socket.setsockopt(zmq.RCVTIMEO, 500)
pin_accept_socket.bind(sutils.BIND_ADDR_TEMPLATE %
(sutils.PIN_ACCEPT_PORT))
requestor_cache = SocketCache(context, zmq.REQ)
pusher_cache = SocketCache(context, zmq.PUSH)
poller = zmq.Poller()
poller.register(connect_socket, zmq.POLLIN)
poller.register(func_create_socket, zmq.POLLIN)
poller.register(func_call_socket, zmq.POLLIN)
poller.register(dag_create_socket, zmq.POLLIN)
poller.register(dag_call_socket, zmq.POLLIN)
poller.register(dag_delete_socket, zmq.POLLIN)
poller.register(list_socket, zmq.POLLIN)
poller.register(exec_status_socket, zmq.POLLIN)
poller.register(sched_update_socket, zmq.POLLIN)
# Start the policy engine.
policy = DefaultDropletSchedulerPolicy(pin_accept_socket, pusher_cache,
kvs, ip)
policy.update()
start = time.time()
def _ann_loop(self):
ctlsocket = self._make_signal_socket(self._announce_ctl_endpoint) #ctlsocket to receive shutdown signal
# subscriber - a SUB socket to get announcements from server
subscriber = self.context.socket(zmq.SUB)
subscriber.setsockopt(zmq.SUBSCRIBE, '')
subscriber.connect(self.upstream_announce_endpoint)
# publisher - a PUB socket to send announcements to clients
publisher = self.context.socket(zmq.PUB)
publisher.bind(self.downstream_announce_endpoint)
poller = zmq.Poller()
poller.register(ctlsocket, zmq.POLLIN)
poller.register(subscriber, zmq.POLLIN)
self._signal_thread(self._startup_ctl_endpoint) # signal successful startup
last_server_ping = None
#'Work' loop
try:
while True:
#TODO: add wait time - so that router shuts down if no response from server after specified time
# for now, just simple receipt/publishing of announcements from server
waittime = (self.server_heartbeat_interval*1000) #time to wait for server ping
poll_results = dict(poller.poll(waittime))
now = time.time()
def __init__(self, socket, io_loop=None):
self.socket = socket
self.io_loop = io_loop or IOLoop.instance()
self.poller = zmq.Poller()
self._send_queue = Queue()
self._recv_callback = None
self._send_callback = None
self._close_callback = None
self._recv_copy = False
self._flushed = False
self._state = self.io_loop.ERROR
self._init_io_state()
# shortcircuit some socket methods
self.bind = self.socket.bind
self.bind_to_random_port = self.socket.bind_to_random_port
self.connect = self.socket.connect
self.setsockopt = self.socket.setsockopt
def create_sockets(self):
"""
Creates the ZeroMQ sockets used by the vPoller Worker Manager
"""
logger.debug('Creating Worker Manager sockets')
self.zcontext = zmq.Context()
self.mgmt_socket = self.zcontext.socket(zmq.REP)
self.mgmt_socket.bind(self.config.get('mgmt'))
self.zpoller = zmq.Poller()
self.zpoller.register(self.mgmt_socket, zmq.POLLIN)
def _send_request(self, request):
"""
Send the given request to the server.
This is used from a thread safe loop in order to avoid sending a
request without receiving a response from a previous one.
:param request: the request to send.
:type request: str
"""
# logger.debug("Signaling '{0}'".format(request))
self._socket.send(request)
poll = zmq.Poller()
poll.register(self._socket, zmq.POLLIN)
reply = None
tries = 0
while True:
socks = dict(poll.poll(self.POLL_TIMEOUT))
if socks.get(self._socket) == zmq.POLLIN:
reply = self._socket.recv()
break
tries += 1
if tries < self.POLL_TRIES:
logger.warning('Retrying receive... {0}/{1}'.format(
tries, self.POLL_TRIES))
else:
if not os.path.exists(ROUTER_CONFIG_PATH):
t = self.token_handler.token_create_hunter()
with open(ROUTER_CONFIG_PATH, 'w') as f:
f.write('hunter_token: %s' % t)
self.router.connect(self.store_addr)
self.router_write.connect(STORE_WRITE_ADDR)
self.router_write_h.connect(STORE_WRITE_H_ADDR)
poller = zmq.Poller()
poller.register(self.router, zmq.POLLIN)
poller_write = zmq.Poller()
poller_write.register(self.router_write, zmq.POLLIN)
poller_write_h = zmq.Poller()
poller_write_h.register(self.router_write_h, zmq.POLLIN)
last_flushed = time.time()
while not self.exit.is_set():
try:
s = dict(poller.poll(5))
except KeyboardInterrupt:
break
if self.router in s:
m = Msg().recv(self.router)
try:
self.handle_message(m)
except Exception as e:
logger.error(e)
logger.debug(m)
def __init__(self, socket):
self.profiler = None
self._poller = zmq.Poller()
self._running = False
self._running_thread = None
self._socket = socket
self._direction = zmq.POLLIN
self._pipe = os.pipe()
fcntl.fcntl(self._pipe[0], fcntl.F_SETFL, os.O_NONBLOCK)
fcntl.fcntl(self._pipe[1], fcntl.F_SETFL, os.O_NONBLOCK)
self._poller.register(self._pipe[0], zmq.POLLIN)
self._poller.register(socket.zmq(), self._direction)
self._output_hook_kernel,
ip.display_pub.session,
ip.display_pub.pub_socket,
ip.display_pub.parent_header,
)
if output_hook is None:
# default: redisplay plain-text outputs
output_hook = self._output_hook_default
# set deadline based on timeout
if timeout is not None:
deadline = monotonic() + timeout
else:
timeout_ms = None
poller = zmq.Poller()
iopub_socket = self.iopub_channel.socket
poller.register(iopub_socket, zmq.POLLIN)
if allow_stdin:
stdin_socket = self.stdin_channel.socket
poller.register(stdin_socket, zmq.POLLIN)
else:
stdin_socket = None
# wait for output and redisplay it
# NOTE: Commenting out the following lines so that we can continue to proxy all iopub messages over Wamp
# while True:
# if timeout is not None:
# timeout = max(0, deadline - monotonic())
# timeout_ms = 1e3 * timeout
# events = dict(poller.poll(timeout_ms))
# if not events: