How to use the py4j.java_gateway.GatewayClient function in py4j

To help you get started, we’ve selected a few py4j examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github dask / knit / knit / core.py View on Github external
long_timeout -= 0.2
            if long_timeout < 0:
                break

        if master_rpcport in [-1, 'N/A']:
            raise Exception(
"""The application master JVM process failed to report back. This can mean:
 - that the YARN cluster cannot scheduler adequate resources - check
   k.yarn_api.cluster_metrics() and other diagnostic methods;
 - that the ApplicationMaster crashed - check the application logs, k.logs();
 - that the cluster is otherwise unhealthy - check the RM and NN logs 
   (use k.yarn_api.system_logs() to find these on a one-node system
""")
        master_rpchost = self.client.masterRPCHost()

        gateway = JavaGateway(GatewayClient(
            address=master_rpchost, port=master_rpcport), auto_convert=True)
        self.master = gateway.entry_point
        rfiles = [triple_slash(f) if f.startswith('hdfs://') else
                  '/'.join(['hdfs://', self.hdfs_home, '.knitDeps',
                            os.path.basename(f)])
                  for f in files]
        logger.debug("Resource files: %s" % rfiles)
        jfiles = ListConverter().convert(rfiles, gateway._gateway_client)
        jenv = MapConverter().convert(envvars, gateway._gateway_client)
        self.master.init(jfiles, jenv, cmd, num_containers,
                         virtual_cores, memory)

        return self.app_id
github mwong510ca / 15Puzzle_OptimalSolver / gui(pyqt5) / app15PuzzleGameSolver.py View on Github external
if __name__ == "__main__":
    host = '127.0.0.1'
    port_number = 25334
    connectionTimeout = 15
    while port_number < 25335:
        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        s.bind(('', 0))
        port_number = s.getsockname()[1]
        s.close()
    try:
        p = subprocess.Popen(['java', '-jar', 'FifteenPuzzleGateway.jar', str(port_number)])
        count = 0;
        print("Connecting to server.  Please wait.")
        while count < connectionTimeout:
            time.sleep(1)
            gateway_server = JavaGateway(GatewayClient(address=host, port=port_number))
            count += 1
            connected = True
            try:
                gateway_server.entry_point.isConnected()
            except:
                connected = False
            if connected:
                break
            elif count % 2 == 0 and count < connectionTimeout:
                print(str(count) + " seconds passed, continue to wait.")
        if not connected:
            print("Connection time out over " + str(connectionTimeout) + " seconds")
            gateway_server.shutdown()
            p.kill()
            sys.exit()
        else:
github locationtech-labs / geopyspark / geopyspark / java_gateway.py View on Github external
proc = Popen(command, cwd=path, stdin=PIPE, preexec_fn=preexec_func, env=env)

    gateway_port = None
    while gateway_port is None and proc.poll() is None:
        timeout = 1
        readable, _, _ = select.select([callback], [], [], timeout)
        if callback in readable:
            connection = callback.accept()[0]
            gateway_port = read_int(connection.makefile(mode="rb"))
            connection.close()
            callback.close()

    if gateway_port is None:
        raise Exception("Gateway process exited before sending the driver its port")

    gateway = JavaGateway(GatewayClient(port=gateway_port), auto_convert=False)

    '''
    java_import(gateway.jvm, "geotrellis.raster.*")
    java_import(gateway.jvm, "geotrellis.proj4.*")
    java_import(gateway.jvm, "geotrellis.raster.io.geotiff.*")
    java_import(gateway.jvm, "geotrellis.raster.io.geotiff.reader.*")
    java_import(gateway.jvm, "geotrellis.spark.io.hadoop.*")
    java_import(gateway.jvm, "org.apache.spark.*")
    java_import(gateway.jvm, "org.apache.spark.api.java.*")
    '''

    return gateway
github ngageoint / mrgeo / mrgeo-python / src / main / python / pymrgeo / java_gateway.py View on Github external
if callback_socket in readable:
                gateway_connection = callback_socket.accept()[0]
                # Determine which ephemeral port the server started on:
                gateway_port = read_int(gateway_connection.makefile(mode="rb"))
                gateway_connection.close()
                callback_socket.close()

        _isremote = not fork

        if gateway_port is None:
                    raise Exception("Java gateway process exited before sending the driver its port number")

    print("Talking with MrGeo on port " + str(gateway_port))

    # Connect to the gateway
    gateway_client = GatewayClient(address=requesthost, port=gateway_port)
    gateway = JavaGateway(gateway_client=gateway_client, auto_convert=True)

    # Import the classes used by MrGeo
    java_import(gateway.jvm, "org.mrgeo.python.*")

    # Import classes used by Spark
    java_import(gateway.jvm, "org.apache.spark.SparkConf")
    java_import(gateway.jvm, "org.apache.spark.api.java.*")
    java_import(gateway.jvm, "org.apache.spark.api.python.*")
    java_import(gateway.jvm, "org.apache.spark.mllib.api.python.*")

    return gateway, gateway_client
github ddf-project / DDF / python / ddf / gateway.py View on Github external
class JavaOutputThread(Thread):
        def __init__(self, stream):
            Thread.__init__(self)
            self.daemon = True
            self.stream = stream

        def run(self):
            while True:
                line = self.stream.readline()
                sys.stderr.write(line)

    JavaOutputThread(process.stdout).start()

    # connect to the gateway server
    gateway = JavaGateway(GatewayClient(port=port), auto_convert=False)
    java_import(gateway.jvm, 'io.ddf.*')
    java_import(gateway.jvm, 'io.ddf.spark.*')
    return gateway
github ContinuumIO / nutchpy / nutchpy / JVM.py View on Github external
main_class = "com.continuumio.seqreaderapp.App"

    port = int(os.getenv('NUTCHPY_GATEWAY_PORT',0))

    cmd_dict = {"jar_full": jar_full, "main_class": main_class, 'port': port}

    java_cmd = "/usr/bin/java -cp ::{jar_full} -Xms512m -Xmx512m {main_class} {port}".format(**cmd_dict)
    ps = subprocess.Popen(java_cmd, shell=os.name != 'nt',
                              stdout=subprocess.PIPE, stderr=subprocess.PIPE)

    # wait for JVM to start
    time.sleep(1)
    logger.debug(java_cmd)
    port = int(ps.stdout.readline())

    gateway = JavaGateway(GatewayClient(port=port),auto_convert=True)
    logger.info("JAVA GATEWAY STARTED ON PORT: %d"% (port,) )

    java_import(gateway.jvm, 'com.continuumio.seqreaderapp.LinkReader')
    java_import(gateway.jvm, 'com.continuumio.seqreaderapp.NodeReader')
    java_import(gateway.jvm, 'com.continuumio.seqreaderapp.SequenceReader')

    ## STOLEN SHAMELESS FROM APACHE/SPARK
    # Create a thread to echo output from the GatewayServer, which is required
    # for Java log output to show up:
    class EchoOutputThread(Thread):

        def __init__(self, stream):
            Thread.__init__(self)
            self.daemon = True
            self.stream = stream
github apache / zeppelin / python / src / main / resources / python / zeppelin_python.py View on Github external
if completionList is None or len(completionList) <= 0:
      self.interpreter.setStatementsFinished("", False)
    else:
      result = json.dumps(list(filter(lambda x : not re.match("^__.*", x), list(completionList))))
      self.interpreter.setStatementsFinished(result, False)

host = sys.argv[1]
port = int(sys.argv[2])

if "PY4J_GATEWAY_SECRET" in os.environ:
  from py4j.java_gateway import GatewayParameters
  gateway_secret = os.environ["PY4J_GATEWAY_SECRET"]
  gateway = JavaGateway(gateway_parameters=GatewayParameters(
    address=host, port=port, auth_token=gateway_secret, auto_convert=True))
else:
  gateway = JavaGateway(GatewayClient(address=host, port=port), auto_convert=True)

intp = gateway.entry_point
_zcUserQueryNameSpace = {}

completion = PythonCompletion(intp, _zcUserQueryNameSpace)
_zcUserQueryNameSpace["__zeppelin_completion__"] = completion
_zcUserQueryNameSpace["gateway"] = gateway

from zeppelin_context import PyZeppelinContext
if intp.getZeppelinContext():
  z = __zeppelin__ = PyZeppelinContext(intp.getZeppelinContext(), gateway)
  __zeppelin__._setup_matplotlib()
  _zcUserQueryNameSpace["z"] = z
  _zcUserQueryNameSpace["__zeppelin__"] = __zeppelin__

intp.onPythonScriptInitialized(os.getpid())
github locationtech-labs / geopyspark / geopyspark / java_gateway.py View on Github external
proc = Popen(command, cwd=path, stdin=PIPE, preexec_fn=preexec_func, env=env)

    gateway_port = None
    while gateway_port is None and proc.poll() is None:
        timeout = 1
        readable, _, _ = select.select([callback], [], [], timeout)
        if callback in readable:
            connection = callback.accept()[0]
            gateway_port = read_int(connection.makefile(mode="rb"))
            connection.close()
            callback.close()

    if gateway_port is None:
        raise Exception("Gateway process exited before sending the driver its port")

    gateway = JavaGateway(GatewayClient(port=gateway_port), auto_convert=False)

    '''
    java_import(gateway.jvm, "geotrellis.raster.*")
    java_import(gateway.jvm, "geotrellis.proj4.*")
    java_import(gateway.jvm, "geotrellis.raster.io.geotiff.*")
    java_import(gateway.jvm, "geotrellis.raster.io.geotiff.reader.*")
    java_import(gateway.jvm, "geotrellis.spark.io.hadoop.*")
    java_import(gateway.jvm, "org.apache.spark.*")
    java_import(gateway.jvm, "org.apache.spark.api.java.*")
    '''

    return gateway
github Angel-ML / angel / angel-ps / python / pyangel / java_gateway.py View on Github external
# We use select() here in order to avoid blocking indefinitely if the subprocess dies
        # before connecting
        while gateway_port is None and proc.poll() is None:
            timeout = 1  # (seconds)
            readable, _, _ = select.select([callback_socket], [], [], timeout)
            if callback_socket in readable:
                gateway_connection = callback_socket.accept()[0]
                # Determine which ephemeral port the server started on:
                gateway_port = read_int(gateway_connection.makefile(mode="rb"))
                gateway_connection.close()
                callback_socket.close()
        if gateway_port is None:
            raise Exception("Java gateway process exited before sending the driver its port number")

    # Connect to the gateway
    gateway = JavaGateway(GatewayClient(port=gateway_port), auto_convert=True)

    return gateway