Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
"""Creates the remote reference for each worker node"""
def _make_callback(_factory, _name, _url):
def _errback(_reason):
log.msg("Could not get remote node %s in %s: %s." % (_name, _url, _reason), log.ERROR)
d = _factory.getRootObject()
d.addCallbacks(callback=lambda obj: self.add_node(obj, _name), errback=_errback)
server, port = url.split(":")
port = int(port)
log.msg("Connecting to cluster worker %s..." % name)
log.msg("Server: %s, Port: %s" % (server, port))
factory = ScrapyPBClientFactory(self, name)
try:
reactor.connectTCP(server, port, factory)
except Exception, err:
log.msg("Could not connect to node %s in %s: %s." % (name, url, reason), log.ERROR)
else:
_make_callback(factory, name, url)
def send_changes():
# Submit the changes, if any
if not changes:
logging.warning("No changes found")
return
host, port = master.split(':')
port = int(port)
f = pb.PBClientFactory()
d = f.login(credentials.UsernamePassword(username.encode('utf-8'), auth.encode('utf-8')))
reactor.connectTCP(host, port, f)
d.addErrback(connectFailed)
d.addCallback(connected)
d.addBoth(cleanup)
reactor.run()
def callback(result):
name, port = random.choice(sorted((str(x.payload.target), x.payload.port) for x in result[0]))
reactor.connectTCP(name, port, f, **tcpkwargs)
def errback(err):
def setServiceParent(self, service):
if service:
internet.TCPClient(self.server, self.port, self).setServiceParent(service)
else:
reactor.connectTCP(self.server, self.port, self)
def createIRC(*args, **kwargs):
f = QWebIRCFactory(*args, **kwargs)
tcpkwargs = {}
if hasattr(config, "OUTGOING_IP"):
tcpkwargs["bindAddress"] = (config.OUTGOING_IP, 0)
if CONNECTION_RESOLVER is None:
reactor.connectTCP(config.IRCSERVER, config.IRCPORT, f, **tcpkwargs)
return f
def callback(result):
name, port = random.choice(sorted((str(x.payload.target), x.payload.port) for x in result[0]))
reactor.connectTCP(name, port, f, **tcpkwargs)
def errback(err):
f.clientConnectionFailed(None, err) # None?!
d = CONNECTION_RESOLVER.lookupService(config.IRCSERVER, (1, 3, 11))
d.addCallbacks(callback, errback)
return f
def get_store(self):
factory = pb.PBClientFactory()
reactor.connectTCP(self.host, 8789, factory)
return factory.getRootObject()
def run(main):
factory = MPDFactory()
factory.connectionMade = main
reactor.connectTCP(os.getenv("MPD_HOST", "localhost"), os.getenv("MPD_PORT", 6600), factory)
reactor.run()
@defer.inlineCallbacks
def do_this():
for keyspace in keyspaces:
factory = ManagedCassandraClientFactory(keyspace)
reactor.connectTCP("localhost", 9160, factory)
client = CassandraClient(factory)
k = yield client.describe_keyspace(keyspace)
for cf in [c.name for c in k.cf_defs]:
n = 0
db_rows = yield client.get_range_slices(column_family=cf, count=10000000)
with gzip.GzipFile("%s.%s.csv.gz" % (keyspace, cf), "w") as f:
out = csv.DictWriter(f, fieldnames=["keyspace", "cf", "key", "col", "val"])
out.writeheader()
for row in db_rows:
for col in row.columns:
out.writerow({"keyspace": keyspace,
"cf": cf,
"key": row.key.encode("string_escape"),
"col": col.column.name,
"val": col.column.value.encode("string_escape")})
n += 1
else:
print "reactor.connectTCP(", config.yabiadminserver, ",", config.yabiadminport, ",", os.path.join(config.yabiadminpath, self.BLOCKED_URL), ")"
# now if the page fails for some reason. deal with it
def _doFailure(data):
if VERBOSE:
print "No more unblock requests. Sleeping for", self.JOBLESS_PAUSE
# no more tasks. we should wait for the next task.
#self.pausechannel_unblock.put(self.JOBLESS_PAUSE)
d = factory.deferred.addCallback(self.start_unblock).addErrback(_doFailure)
if config.yabiadminscheme == 'https':
reactor.connectSSL(config.yabiadminserver, config.yabiadminport, factory, ServerContextFactory())
else:
reactor.connectTCP(config.yabiadminserver, config.yabiadminport, factory)
return d