Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
long_timeout -= 0.2
if long_timeout < 0:
break
if master_rpcport in [-1, 'N/A']:
raise Exception(
"""The application master JVM process failed to report back. This can mean:
- that the YARN cluster cannot scheduler adequate resources - check
k.yarn_api.cluster_metrics() and other diagnostic methods;
- that the ApplicationMaster crashed - check the application logs, k.logs();
- that the cluster is otherwise unhealthy - check the RM and NN logs
(use k.yarn_api.system_logs() to find these on a one-node system
""")
master_rpchost = self.client.masterRPCHost()
gateway = JavaGateway(GatewayClient(
address=master_rpchost, port=master_rpcport), auto_convert=True)
self.master = gateway.entry_point
rfiles = [triple_slash(f) if f.startswith('hdfs://') else
'/'.join(['hdfs://', self.hdfs_home, '.knitDeps',
os.path.basename(f)])
for f in files]
logger.debug("Resource files: %s" % rfiles)
jfiles = ListConverter().convert(rfiles, gateway._gateway_client)
jenv = MapConverter().convert(envvars, gateway._gateway_client)
self.master.init(jfiles, jenv, cmd, num_containers,
virtual_cores, memory)
return self.app_id
if __name__ == "__main__":
host = '127.0.0.1'
port_number = 25334
connectionTimeout = 15
while port_number < 25335:
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind(('', 0))
port_number = s.getsockname()[1]
s.close()
try:
p = subprocess.Popen(['java', '-jar', 'FifteenPuzzleGateway.jar', str(port_number)])
count = 0;
print("Connecting to server. Please wait.")
while count < connectionTimeout:
time.sleep(1)
gateway_server = JavaGateway(GatewayClient(address=host, port=port_number))
count += 1
connected = True
try:
gateway_server.entry_point.isConnected()
except:
connected = False
if connected:
break
elif count % 2 == 0 and count < connectionTimeout:
print(str(count) + " seconds passed, continue to wait.")
if not connected:
print("Connection time out over " + str(connectionTimeout) + " seconds")
gateway_server.shutdown()
p.kill()
sys.exit()
else:
proc = Popen(command, cwd=path, stdin=PIPE, preexec_fn=preexec_func, env=env)
gateway_port = None
while gateway_port is None and proc.poll() is None:
timeout = 1
readable, _, _ = select.select([callback], [], [], timeout)
if callback in readable:
connection = callback.accept()[0]
gateway_port = read_int(connection.makefile(mode="rb"))
connection.close()
callback.close()
if gateway_port is None:
raise Exception("Gateway process exited before sending the driver its port")
gateway = JavaGateway(GatewayClient(port=gateway_port), auto_convert=False)
'''
java_import(gateway.jvm, "geotrellis.raster.*")
java_import(gateway.jvm, "geotrellis.proj4.*")
java_import(gateway.jvm, "geotrellis.raster.io.geotiff.*")
java_import(gateway.jvm, "geotrellis.raster.io.geotiff.reader.*")
java_import(gateway.jvm, "geotrellis.spark.io.hadoop.*")
java_import(gateway.jvm, "org.apache.spark.*")
java_import(gateway.jvm, "org.apache.spark.api.java.*")
'''
return gateway
if callback_socket in readable:
gateway_connection = callback_socket.accept()[0]
# Determine which ephemeral port the server started on:
gateway_port = read_int(gateway_connection.makefile(mode="rb"))
gateway_connection.close()
callback_socket.close()
_isremote = not fork
if gateway_port is None:
raise Exception("Java gateway process exited before sending the driver its port number")
print("Talking with MrGeo on port " + str(gateway_port))
# Connect to the gateway
gateway_client = GatewayClient(address=requesthost, port=gateway_port)
gateway = JavaGateway(gateway_client=gateway_client, auto_convert=True)
# Import the classes used by MrGeo
java_import(gateway.jvm, "org.mrgeo.python.*")
# Import classes used by Spark
java_import(gateway.jvm, "org.apache.spark.SparkConf")
java_import(gateway.jvm, "org.apache.spark.api.java.*")
java_import(gateway.jvm, "org.apache.spark.api.python.*")
java_import(gateway.jvm, "org.apache.spark.mllib.api.python.*")
return gateway, gateway_client
class JavaOutputThread(Thread):
def __init__(self, stream):
Thread.__init__(self)
self.daemon = True
self.stream = stream
def run(self):
while True:
line = self.stream.readline()
sys.stderr.write(line)
JavaOutputThread(process.stdout).start()
# connect to the gateway server
gateway = JavaGateway(GatewayClient(port=port), auto_convert=False)
java_import(gateway.jvm, 'io.ddf.*')
java_import(gateway.jvm, 'io.ddf.spark.*')
return gateway
main_class = "com.continuumio.seqreaderapp.App"
port = int(os.getenv('NUTCHPY_GATEWAY_PORT',0))
cmd_dict = {"jar_full": jar_full, "main_class": main_class, 'port': port}
java_cmd = "/usr/bin/java -cp ::{jar_full} -Xms512m -Xmx512m {main_class} {port}".format(**cmd_dict)
ps = subprocess.Popen(java_cmd, shell=os.name != 'nt',
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
# wait for JVM to start
time.sleep(1)
logger.debug(java_cmd)
port = int(ps.stdout.readline())
gateway = JavaGateway(GatewayClient(port=port),auto_convert=True)
logger.info("JAVA GATEWAY STARTED ON PORT: %d"% (port,) )
java_import(gateway.jvm, 'com.continuumio.seqreaderapp.LinkReader')
java_import(gateway.jvm, 'com.continuumio.seqreaderapp.NodeReader')
java_import(gateway.jvm, 'com.continuumio.seqreaderapp.SequenceReader')
## STOLEN SHAMELESS FROM APACHE/SPARK
# Create a thread to echo output from the GatewayServer, which is required
# for Java log output to show up:
class EchoOutputThread(Thread):
def __init__(self, stream):
Thread.__init__(self)
self.daemon = True
self.stream = stream
if completionList is None or len(completionList) <= 0:
self.interpreter.setStatementsFinished("", False)
else:
result = json.dumps(list(filter(lambda x : not re.match("^__.*", x), list(completionList))))
self.interpreter.setStatementsFinished(result, False)
host = sys.argv[1]
port = int(sys.argv[2])
if "PY4J_GATEWAY_SECRET" in os.environ:
from py4j.java_gateway import GatewayParameters
gateway_secret = os.environ["PY4J_GATEWAY_SECRET"]
gateway = JavaGateway(gateway_parameters=GatewayParameters(
address=host, port=port, auth_token=gateway_secret, auto_convert=True))
else:
gateway = JavaGateway(GatewayClient(address=host, port=port), auto_convert=True)
intp = gateway.entry_point
_zcUserQueryNameSpace = {}
completion = PythonCompletion(intp, _zcUserQueryNameSpace)
_zcUserQueryNameSpace["__zeppelin_completion__"] = completion
_zcUserQueryNameSpace["gateway"] = gateway
from zeppelin_context import PyZeppelinContext
if intp.getZeppelinContext():
z = __zeppelin__ = PyZeppelinContext(intp.getZeppelinContext(), gateway)
__zeppelin__._setup_matplotlib()
_zcUserQueryNameSpace["z"] = z
_zcUserQueryNameSpace["__zeppelin__"] = __zeppelin__
intp.onPythonScriptInitialized(os.getpid())
proc = Popen(command, cwd=path, stdin=PIPE, preexec_fn=preexec_func, env=env)
gateway_port = None
while gateway_port is None and proc.poll() is None:
timeout = 1
readable, _, _ = select.select([callback], [], [], timeout)
if callback in readable:
connection = callback.accept()[0]
gateway_port = read_int(connection.makefile(mode="rb"))
connection.close()
callback.close()
if gateway_port is None:
raise Exception("Gateway process exited before sending the driver its port")
gateway = JavaGateway(GatewayClient(port=gateway_port), auto_convert=False)
'''
java_import(gateway.jvm, "geotrellis.raster.*")
java_import(gateway.jvm, "geotrellis.proj4.*")
java_import(gateway.jvm, "geotrellis.raster.io.geotiff.*")
java_import(gateway.jvm, "geotrellis.raster.io.geotiff.reader.*")
java_import(gateway.jvm, "geotrellis.spark.io.hadoop.*")
java_import(gateway.jvm, "org.apache.spark.*")
java_import(gateway.jvm, "org.apache.spark.api.java.*")
'''
return gateway
# We use select() here in order to avoid blocking indefinitely if the subprocess dies
# before connecting
while gateway_port is None and proc.poll() is None:
timeout = 1 # (seconds)
readable, _, _ = select.select([callback_socket], [], [], timeout)
if callback_socket in readable:
gateway_connection = callback_socket.accept()[0]
# Determine which ephemeral port the server started on:
gateway_port = read_int(gateway_connection.makefile(mode="rb"))
gateway_connection.close()
callback_socket.close()
if gateway_port is None:
raise Exception("Java gateway process exited before sending the driver its port number")
# Connect to the gateway
gateway = JavaGateway(GatewayClient(port=gateway_port), auto_convert=True)
return gateway