Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def setUpClass(cls):
ReusedPySparkTestCase.setUpClass()
cls.tempdir = tempfile.NamedTemporaryFile(delete=False)
try:
cls.sc._jvm.org.apache.hadoop.hive.conf.HiveConf()
except py4j.protocol.Py4JError:
cls.tearDownClass()
raise unittest.SkipTest("Hive is not available")
except TypeError:
cls.tearDownClass()
raise unittest.SkipTest("Hive is not available")
os.unlink(cls.tempdir.name)
cls.spark = HiveContext._createForTesting(cls.sc)
cls.testData = [Row(key=i, value=str(i)) for i in range(100)]
cls.df = cls.sc.parallelize(cls.testData).toDF()
x = _getLoaderInstance(
sc,
jar_file_name,
'org.apache.sysml.api.dl.Caffe2DMLLoader',
hint + 'systemml-*-extra.jar')
x.loadCaffe2DML(jar_file_name)
else:
x = _getLoaderInstance(
sc,
jar_file_name,
'org.apache.sysml.utils.SystemMLLoaderUtils',
hint + 'systemml-*.jar')
x.loadSystemML(jar_file_name)
try:
ret = _createJavaObject(sc, obj_type)
except (py4j.protocol.Py4JError, TypeError):
raise ImportError(err_msg + ' Hint: ' + hint + jar_file_name)
return ret
proto.HELP_CLASS_SUBCOMMAND_NAME +\
var._fqn + "\n" +\
get_command_part(pattern) +\
get_command_part(short_name) +\
proto.END_COMMAND_PART
answer = gateway_client.send_command(command)
elif hasattr2(var, "container") and hasattr2(var, "name"):
if pattern is not None:
raise Py4JError("pattern should be None with var is a JavaMember")
pattern = var.name + "(*"
var = var.container
return gateway_help(
gateway_client, var, pattern, short_name=short_name,
display=display)
else:
raise Py4JError(
"var is none of Java Object, Java Class or Java Member")
help_page = get_return_value(answer, gateway_client, None, None)
if (display):
pager(help_page)
else:
return help_page
def kill(self):
"""
Method to kill a yarn application
Returns
-------
bool:
True if successful, False otherwise.
"""
if self.client is None:
# never started, can't stop - should be warning or exception?
return False
try:
self.client.kill()
except Py4JError:
logger.debug("Error while attempting to kill", exc_info=1)
# fallback
self.yarn_api.kill(self.app_id)
if self.proc is not None:
self.client_gateway.shutdown()
if on_windows:
call(["cmd", "/c", "taskkill", "/f", "/t", "/pid",
str(self.proc.pid)])
self.proc.terminate()
self.proc.communicate()
self.proc = None
self.client = None
out = self.runtime_status() == 'KILLED'
return out
from pyspark.context import SparkContext
from pyspark.sql import SparkSession, SQLContext
from pyspark.storagelevel import StorageLevel
if os.environ.get("SPARK_EXECUTOR_URI"):
SparkContext.setSystemProperty("spark.executor.uri", os.environ["SPARK_EXECUTOR_URI"])
SparkContext._ensure_initialized()
try:
# Try to access HiveConf, it will raise exception if Hive is not added
SparkContext._jvm.org.apache.hadoop.hive.conf.HiveConf()
spark = SparkSession.builder\
.enableHiveSupport()\
.getOrCreate()
except py4j.protocol.Py4JError:
spark = SparkSession.builder.getOrCreate()
except TypeError:
spark = SparkSession.builder.getOrCreate()
sc = spark.sparkContext
sql = spark.sql
atexit.register(lambda: sc.stop())
# for compatibility
sqlContext = spark._wrapped
sqlCtx = sqlContext
print("""Welcome to
____ __
/ __/__ ___ _____/ /__
_\ \/ _ \/ _ `/ __/ '_/
"""
popen_kwargs = {}
if not jarpath:
jarpath = find_jar_path()
if not java_path:
java_home = os.environ.get("JAVA_HOME")
if java_home:
java_path = os.path.join(java_home, "bin", "java")
else:
java_path = "java"
# Fail if the jar does not exist.
if not os.path.exists(jarpath):
raise Py4JError("Could not find py4j jar at {0}".format(jarpath))
# Launch the server in a subprocess.
classpath = os.pathsep.join((jarpath, classpath))
command = [java_path, "-classpath", classpath] + javaopts + \
["py4j.GatewayServer"]
if die_on_exit:
command.append("--die-on-broken-pipe")
if enable_auth:
command.append("--enable-auth")
command.append(str(port))
logger.debug("Launching gateway with command {0}".format(command))
# stderr redirection
close_stderr = False
if redirect_stderr is None:
stderr = open(os.devnull, "w")
def shutdown_gateway(self):
"""Sends a shutdown command to the gateway. This will close the gateway
server: all active connections will be closed. This may be useful
if the lifecycle of the Java program must be tied to the Python
program.
"""
if not self.is_connected:
raise Py4JError("Gateway must be connected to send shutdown cmd.")
try:
quiet_close(self.stream)
self.socket.sendall(
proto.SHUTDOWN_GATEWAY_COMMAND_NAME.encode("utf-8"))
quiet_close(self.socket)
self.is_connected = False
except Exception:
# Do nothing! Exceptions might occur anyway.
logger.debug("Exception occurred while shutting down gateway",
exc_info=True)
def next(self):
"""This next method wraps the `next` method in Java iterators.
The `Iterator.next()` method is called and if an exception occur (e.g.,
NoSuchElementException), a StopIteration exception is raised."""
if self._next_name not in self._methods:
self._methods[self._next_name] = JavaMember(
self._next_name, self,
self._target_id, self._gateway_client)
try:
return self._methods[self._next_name]()
except Py4JError:
raise StopIteration()
var._get_object_id() + "\n" +\
get_command_part(pattern) +\
get_command_part(short_name) +\
proto.END_COMMAND_PART
answer = gateway_client.send_command(command)
elif hasattr2(var, "_fqn"):
command = proto.HELP_COMMAND_NAME +\
proto.HELP_CLASS_SUBCOMMAND_NAME +\
var._fqn + "\n" +\
get_command_part(pattern) +\
get_command_part(short_name) +\
proto.END_COMMAND_PART
answer = gateway_client.send_command(command)
elif hasattr2(var, "container") and hasattr2(var, "name"):
if pattern is not None:
raise Py4JError("pattern should be None with var is a JavaMember")
pattern = var.name + "(*"
var = var.container
return gateway_help(
gateway_client, var, pattern, short_name=short_name,
display=display)
else:
raise Py4JError(
"var is none of Java Object, Java Class or Java Member")
help_page = get_return_value(answer, gateway_client, None, None)
if (display):
pager(help_page)
else:
return help_page
def _java_lang_class(self):
"""Gets the java.lang.Class of the current JavaClass. This is
equivalent to calling .class in Java.
"""
command = proto.REFLECTION_COMMAND_NAME +\
proto.REFL_GET_JAVA_LANG_CLASS_SUB_COMMAND_NAME +\
self._fqn + "\n" + proto.END_COMMAND_PART
answer = self._gateway_client.send_command(command)
if len(answer) > 1 and answer[0] == proto.SUCCESS:
return get_return_value(
answer, self._gateway_client, self._fqn, "_java_lang_class")
else:
raise Py4JError(
"{0} does not exist in the JVM".format(self._fqn))