Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def setUp(self):
# create the io_loop
self._io_loop = IOLoop.instance()
# and the context
self._ctx = zmq.Context(1)
# setup the mgmt sockets
self._setup_mgmt_sockets()
# setup the data sockets
self._setup_data_sockets()
# setup the management interface
self._mgmt = ZmqMgmt( self._mgmt_sockets['worker_sub'],
self._mgmt_sockets['worker_pub'], io_loop=self._io_loop)
self._mgmt.start()
self._mgmt.add_callback(ZMQ_SPYDER_MGMT_WORKER, self.on_mgmt_end)
def __init__(self):
MQAsyncSub.__init__(self, zmq.Context(), 'test', ['metrics.browser'])
IOLoop.instance().start()
def __init__(self, settings, identity, insocket, outsocket, mgmt, frontier,
log_handler, log_level, io_loop):
"""
Initialize the master.
"""
LoggingMixin.__init__(self, log_handler, log_level)
self._identity = identity
self._io_loop = io_loop or IOLoop.instance()
self._in_stream = ZMQStream(insocket, io_loop)
self._out_stream = ZMQStream(outsocket, io_loop)
self._mgmt = mgmt
self._frontier = frontier
self._running = False
self._available_workers = []
# periodically check if there are pending URIs to crawl
self._periodic_update = PeriodicCallback(self._send_next_uri,
settings.MASTER_PERIODIC_UPDATE_INTERVAL, io_loop=io_loop)
# start this periodic callback when you are waiting for the workers to
# finish
self._periodic_shutdown = PeriodicCallback(self._shutdown_wait, 500,
def start(self):
# Launch worker subprocesses
self.processes = dict(zip(self.args, map(lambda x: self.target(x,self.uri), self.args)))
for d,p in self.processes.iteritems():
print "Launching Worker: ", d
p.start()
# Start IOLoop
self.stream = ZMQStream(self.socket)
self.ioloop = ioloop.IOLoop.instance()
self.ioloop.call_later(1.0, self.on_start)
self.stream.on_recv_stream(self.on_rcv)
try:
self.ioloop.start()
except KeyboardInterrupt:
print " ** Got keyboard interrupt (^C) shutting down cleanly ... ** "
self.shutdown()
self.ioloop.close()
for d,p in self.processes.iteritems():
p.join()
def _eventloop_changed(self, name, old, new):
"""schedule call to eventloop from IOLoop"""
loop = ioloop.IOLoop.instance()
loop.add_callback(self.enter_eventloop)
def serve(self):
''' start server listening on port, launch browser if requested & start the ioloop
'''
self.http_server = httpserver.HTTPServer(self.app)
self.http_server.listen(self.options.port)
if not self.options.serveronly:
launch_browser(self.options.port, self.options.browser)
try:
ioloop.IOLoop.instance().start()
except KeyboardInterrupt:
DEBUG('interrupt received, shutting down.')
def _shutdown(self):
""" Stop all subprocesses and exit. """
self.server_manager.cleanup()
ioloop.IOLoop.instance().add_timeout(time.time() + 5, sys.exit)
def launch_scheduler(in_addr, out_addr, mon_addr, not_addr, config=None,
logname='root', log_url=None, loglevel=logging.DEBUG,
identity=b'task', in_thread=False):
ZMQStream = zmqstream.ZMQStream
if config:
# unwrap dict back into Config
config = Config(config)
if in_thread:
# use instance() to get the same Context/Loop as our parent
ctx = zmq.Context.instance()
loop = ioloop.IOLoop.instance()
else:
# in a process, don't use instance()
# for safety with multiprocessing
ctx = zmq.Context()
loop = ioloop.IOLoop()
ins = ZMQStream(ctx.socket(zmq.ROUTER),loop)
ins.setsockopt(zmq.IDENTITY, identity)
ins.bind(in_addr)
outs = ZMQStream(ctx.socket(zmq.ROUTER),loop)
outs.setsockopt(zmq.IDENTITY, identity)
outs.bind(out_addr)
mons = zmqstream.ZMQStream(ctx.socket(zmq.PUB),loop)
mons.connect(mon_addr)
nots = zmqstream.ZMQStream(ctx.socket(zmq.SUB),loop)
nots.setsockopt(zmq.SUBSCRIBE, b'')