How to use the py4j.protocol.Py4JError function in py4j

To help you get started, we’ve selected a few py4j examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github qubole / spark-on-lambda / python / pyspark / sql / tests.py View on Github external
def setUpClass(cls):
        ReusedPySparkTestCase.setUpClass()
        cls.tempdir = tempfile.NamedTemporaryFile(delete=False)
        try:
            cls.sc._jvm.org.apache.hadoop.hive.conf.HiveConf()
        except py4j.protocol.Py4JError:
            cls.tearDownClass()
            raise unittest.SkipTest("Hive is not available")
        except TypeError:
            cls.tearDownClass()
            raise unittest.SkipTest("Hive is not available")
        os.unlink(cls.tempdir.name)
        cls.spark = HiveContext._createForTesting(cls.sc)
        cls.testData = [Row(key=i, value=str(i)) for i in range(100)]
        cls.df = cls.sc.parallelize(cls.testData).toDF()
github apache / systemml / src / main / python / systemml / classloader.py View on Github external
x = _getLoaderInstance(
                    sc,
                    jar_file_name,
                    'org.apache.sysml.api.dl.Caffe2DMLLoader',
                    hint + 'systemml-*-extra.jar')
                x.loadCaffe2DML(jar_file_name)
            else:
                x = _getLoaderInstance(
                    sc,
                    jar_file_name,
                    'org.apache.sysml.utils.SystemMLLoaderUtils',
                    hint + 'systemml-*.jar')
                x.loadSystemML(jar_file_name)
        try:
            ret = _createJavaObject(sc, obj_type)
        except (py4j.protocol.Py4JError, TypeError):
            raise ImportError(err_msg + ' Hint: ' + hint + jar_file_name)
        return ret
github bartdag / py4j / py4j-python / src / py4j / java_gateway.py View on Github external
proto.HELP_CLASS_SUBCOMMAND_NAME +\
            var._fqn + "\n" +\
            get_command_part(pattern) +\
            get_command_part(short_name) +\
            proto.END_COMMAND_PART
        answer = gateway_client.send_command(command)
    elif hasattr2(var, "container") and hasattr2(var, "name"):
        if pattern is not None:
            raise Py4JError("pattern should be None with var is a JavaMember")
        pattern = var.name + "(*"
        var = var.container
        return gateway_help(
            gateway_client, var, pattern, short_name=short_name,
            display=display)
    else:
        raise Py4JError(
            "var is none of Java Object, Java Class or Java Member")

    help_page = get_return_value(answer, gateway_client, None, None)
    if (display):
        pager(help_page)
    else:
        return help_page
github dask / knit / knit / core.py View on Github external
def kill(self):
        """
        Method to kill a yarn application

        Returns
        -------
        bool:
            True if successful, False otherwise.
        """
        if self.client is None:
            # never started, can't stop - should be warning or exception?
            return False
        try:
            self.client.kill()
        except Py4JError:
            logger.debug("Error while attempting to kill", exc_info=1)
            # fallback
            self.yarn_api.kill(self.app_id)
        if self.proc is not None:
            self.client_gateway.shutdown()
            if on_windows:
                call(["cmd", "/c", "taskkill", "/f", "/t", "/pid",
                      str(self.proc.pid)])
            self.proc.terminate()
            self.proc.communicate()
            self.proc = None
        self.client = None
        out = self.runtime_status() == 'KILLED'
        return out
github qubole / spark-on-lambda / python / pyspark / shell.py View on Github external
from pyspark.context import SparkContext
from pyspark.sql import SparkSession, SQLContext
from pyspark.storagelevel import StorageLevel

if os.environ.get("SPARK_EXECUTOR_URI"):
    SparkContext.setSystemProperty("spark.executor.uri", os.environ["SPARK_EXECUTOR_URI"])

SparkContext._ensure_initialized()

try:
    # Try to access HiveConf, it will raise exception if Hive is not added
    SparkContext._jvm.org.apache.hadoop.hive.conf.HiveConf()
    spark = SparkSession.builder\
        .enableHiveSupport()\
        .getOrCreate()
except py4j.protocol.Py4JError:
    spark = SparkSession.builder.getOrCreate()
except TypeError:
    spark = SparkSession.builder.getOrCreate()

sc = spark.sparkContext
sql = spark.sql
atexit.register(lambda: sc.stop())

# for compatibility
sqlContext = spark._wrapped
sqlCtx = sqlContext

print("""Welcome to
      ____              __
     / __/__  ___ _____/ /__
    _\ \/ _ \/ _ `/ __/  '_/
github bartdag / py4j / py4j-python / src / py4j / java_gateway.py View on Github external
"""
    popen_kwargs = {}

    if not jarpath:
        jarpath = find_jar_path()

    if not java_path:
        java_home = os.environ.get("JAVA_HOME")
        if java_home:
            java_path = os.path.join(java_home, "bin", "java")
        else:
            java_path = "java"

    # Fail if the jar does not exist.
    if not os.path.exists(jarpath):
        raise Py4JError("Could not find py4j jar at {0}".format(jarpath))

    # Launch the server in a subprocess.
    classpath = os.pathsep.join((jarpath, classpath))
    command = [java_path, "-classpath", classpath] + javaopts + \
              ["py4j.GatewayServer"]
    if die_on_exit:
        command.append("--die-on-broken-pipe")
    if enable_auth:
        command.append("--enable-auth")
    command.append(str(port))
    logger.debug("Launching gateway with command {0}".format(command))

    # stderr redirection
    close_stderr = False
    if redirect_stderr is None:
        stderr = open(os.devnull, "w")
github bartdag / py4j / py4j-python / src / py4j / java_gateway.py View on Github external
def shutdown_gateway(self):
        """Sends a shutdown command to the gateway. This will close the gateway
           server: all active connections will be closed. This may be useful
           if the lifecycle of the Java program must be tied to the Python
           program.
        """
        if not self.is_connected:
            raise Py4JError("Gateway must be connected to send shutdown cmd.")

        try:
            quiet_close(self.stream)
            self.socket.sendall(
                proto.SHUTDOWN_GATEWAY_COMMAND_NAME.encode("utf-8"))
            quiet_close(self.socket)
            self.is_connected = False
        except Exception:
            # Do nothing! Exceptions might occur anyway.
            logger.debug("Exception occurred while shutting down gateway",
                         exc_info=True)
github bartdag / py4j / py4j-python / src / py4j / java_collections.py View on Github external
def next(self):
        """This next method wraps the `next` method in Java iterators.

        The `Iterator.next()` method is called and if an exception occur (e.g.,
        NoSuchElementException), a StopIteration exception is raised."""
        if self._next_name not in self._methods:
            self._methods[self._next_name] = JavaMember(
                self._next_name, self,
                self._target_id, self._gateway_client)
        try:
            return self._methods[self._next_name]()
        except Py4JError:
            raise StopIteration()
github bartdag / py4j / py4j-python / src / py4j / java_gateway.py View on Github external
var._get_object_id() + "\n" +\
            get_command_part(pattern) +\
            get_command_part(short_name) +\
            proto.END_COMMAND_PART
        answer = gateway_client.send_command(command)
    elif hasattr2(var, "_fqn"):
        command = proto.HELP_COMMAND_NAME +\
            proto.HELP_CLASS_SUBCOMMAND_NAME +\
            var._fqn + "\n" +\
            get_command_part(pattern) +\
            get_command_part(short_name) +\
            proto.END_COMMAND_PART
        answer = gateway_client.send_command(command)
    elif hasattr2(var, "container") and hasattr2(var, "name"):
        if pattern is not None:
            raise Py4JError("pattern should be None with var is a JavaMember")
        pattern = var.name + "(*"
        var = var.container
        return gateway_help(
            gateway_client, var, pattern, short_name=short_name,
            display=display)
    else:
        raise Py4JError(
            "var is none of Java Object, Java Class or Java Member")

    help_page = get_return_value(answer, gateway_client, None, None)
    if (display):
        pager(help_page)
    else:
        return help_page
github bartdag / py4j / py4j-python / src / py4j / java_gateway.py View on Github external
def _java_lang_class(self):
        """Gets the java.lang.Class of the current JavaClass. This is
        equivalent to calling .class in Java.
        """
        command = proto.REFLECTION_COMMAND_NAME +\
            proto.REFL_GET_JAVA_LANG_CLASS_SUB_COMMAND_NAME +\
            self._fqn + "\n" + proto.END_COMMAND_PART
        answer = self._gateway_client.send_command(command)

        if len(answer) > 1 and answer[0] == proto.SUCCESS:
            return get_return_value(
                answer, self._gateway_client, self._fqn, "_java_lang_class")
        else:
            raise Py4JError(
                "{0} does not exist in the JVM".format(self._fqn))