How to use the sparkmagic.utils.constants.LONG_RANDOM_VARIABLE_NAME function in sparkmagic
To help you get started, we’ve selected a few sparkmagic examples, based on popular ways it is used in public projects.
Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
jupyter-incubator / sparkmagic / sparkmagic / sparkmagic / livyclientlib / sqlquery.py View on Github 
def _pyspark_command(self, sql_context_variable_name, encode_result=True):
command = u'{}.sql(u"""{} """).toJSON()'.format(sql_context_variable_name, self.query)
if self.samplemethod == u'sample':
command = u'{}.sample(False, {})'.format(command, self.samplefraction)
if self.maxrows >= 0:
command = u'{}.take({})'.format(command, self.maxrows)
else:
command = u'{}.collect()'.format(command)
# Unicode support has improved in Python 3 so we don't need to encode.
if encode_result:
print_command = '{}.encode("{}")'.format(constants.LONG_RANDOM_VARIABLE_NAME,
conf.pyspark_dataframe_encoding())
else:
print_command = constants.LONG_RANDOM_VARIABLE_NAME
command = u'for {} in {}: print({})'.format(constants.LONG_RANDOM_VARIABLE_NAME,
command,
print_command)
return Command(command)def _pyspark_command(self, sql_context_variable_name, encode_result=True):
command = u'{}.sql(u"""{} """).toJSON()'.format(sql_context_variable_name, self.query)
if self.samplemethod == u'sample':
command = u'{}.sample(False, {})'.format(command, self.samplefraction)
if self.maxrows >= 0:
command = u'{}.take({})'.format(command, self.maxrows)
else:
command = u'{}.collect()'.format(command)
# Unicode support has improved in Python 3 so we don't need to encode.
if encode_result:
print_command = '{}.encode("{}")'.format(constants.LONG_RANDOM_VARIABLE_NAME,
conf.pyspark_sql_encoding())
else:
print_command = constants.LONG_RANDOM_VARIABLE_NAME
command = u'for {} in {}: print({})'.format(constants.LONG_RANDOM_VARIABLE_NAME,
command,
print_command)
return Command(command)def _pyspark_command(self, sql_context_variable_name, encode_result=True):
command = u'{}.sql(u"""{} """).toJSON()'.format(sql_context_variable_name, self.query)
if self.samplemethod == u'sample':
command = u'{}.sample(False, {})'.format(command, self.samplefraction)
if self.maxrows >= 0:
command = u'{}.take({})'.format(command, self.maxrows)
else:
command = u'{}.collect()'.format(command)
# Unicode support has improved in Python 3 so we don't need to encode.
if encode_result:
print_command = '{}.encode("{}")'.format(constants.LONG_RANDOM_VARIABLE_NAME,
conf.pyspark_sql_encoding())
else:
print_command = constants.LONG_RANDOM_VARIABLE_NAME
command = u'for {} in {}: print({})'.format(constants.LONG_RANDOM_VARIABLE_NAME,
command,
print_command)
return Command(command)def _pyspark_command(self, spark_context_variable_name, encode_result=True):
command = u'{}.toJSON()'.format(spark_context_variable_name)
if self.samplemethod == u'sample':
command = u'{}.sample(False, {})'.format(command, self.samplefraction)
if self.maxrows >= 0:
command = u'{}.take({})'.format(command, self.maxrows)
else:
command = u'{}.collect()'.format(command)
# Unicode support has improved in Python 3 so we don't need to encode.
if encode_result:
print_command = '{}.encode("{}")'.format(constants.LONG_RANDOM_VARIABLE_NAME,
conf.pyspark_dataframe_encoding())
else:
print_command = constants.LONG_RANDOM_VARIABLE_NAME
command = u'for {} in {}: print({})'.format(constants.LONG_RANDOM_VARIABLE_NAME,
command,
print_command)
return Command(command)def _r_command(self, spark_context_variable_name):
command = spark_context_variable_name
if self.samplemethod == u'sample':
command = u'sample({}, FALSE, {})'.format(command,
self.samplefraction)
if self.maxrows >= 0:
command = u'take({},{})'.format(command, self.maxrows)
else:
command = u'collect({})'.format(command)
command = u'jsonlite::toJSON({})'.format(command)
command = u'for ({} in ({})) {{cat({})}}'.format(constants.LONG_RANDOM_VARIABLE_NAME,
command,
constants.LONG_RANDOM_VARIABLE_NAME)
return Command(command)def _pyspark_command(self, spark_context_variable_name, encode_result=True):
command = u'{}.toJSON()'.format(spark_context_variable_name)
if self.samplemethod == u'sample':
command = u'{}.sample(False, {})'.format(command, self.samplefraction)
if self.maxrows >= 0:
command = u'{}.take({})'.format(command, self.maxrows)
else:
command = u'{}.collect()'.format(command)
# Unicode support has improved in Python 3 so we don't need to encode.
if encode_result:
print_command = '{}.encode("{}")'.format(constants.LONG_RANDOM_VARIABLE_NAME,
conf.pyspark_dataframe_encoding())
else:
print_command = constants.LONG_RANDOM_VARIABLE_NAME
command = u'for {} in {}: print({})'.format(constants.LONG_RANDOM_VARIABLE_NAME,
command,
print_command)
return Command(command)def _pyspark_command(self, spark_context_variable_name, encode_result=True):
command = u'{}.toJSON()'.format(spark_context_variable_name)
if self.samplemethod == u'sample':
command = u'{}.sample(False, {})'.format(command, self.samplefraction)
if self.maxrows >= 0:
command = u'{}.take({})'.format(command, self.maxrows)
else:
command = u'{}.collect()'.format(command)
# Unicode support has improved in Python 3 so we don't need to encode.
if encode_result:
print_command = '{}.encode("{}")'.format(constants.LONG_RANDOM_VARIABLE_NAME,
conf.pyspark_dataframe_encoding())
else:
print_command = constants.LONG_RANDOM_VARIABLE_NAME
command = u'for {} in {}: print({})'.format(constants.LONG_RANDOM_VARIABLE_NAME,
command,
print_command)
return Command(command)def _pyspark_command(self, sql_context_variable_name, encode_result=True):
command = u'{}.sql(u"""{} """).toJSON()'.format(sql_context_variable_name, self.query)
if self.samplemethod == u'sample':
command = u'{}.sample(False, {})'.format(command, self.samplefraction)
if self.maxrows >= 0:
command = u'{}.take({})'.format(command, self.maxrows)
else:
command = u'{}.collect()'.format(command)
# Unicode support has improved in Python 3 so we don't need to encode.
if encode_result:
print_command = '{}.encode("{}")'.format(constants.LONG_RANDOM_VARIABLE_NAME,
conf.pyspark_dataframe_encoding())
else:
print_command = constants.LONG_RANDOM_VARIABLE_NAME
command = u'for {} in {}: print({})'.format(constants.LONG_RANDOM_VARIABLE_NAME,
command,
print_command)
return Command(command)def _r_command(self, spark_context_variable_name):
command = spark_context_variable_name
if self.samplemethod == u'sample':
command = u'sample({}, FALSE, {})'.format(command,
self.samplefraction)
if self.maxrows >= 0:
command = u'take({},{})'.format(command, self.maxrows)
else:
command = u'collect({})'.format(command)
command = u'jsonlite::toJSON({})'.format(command)
command = u'for ({} in ({})) {{cat({})}}'.format(constants.LONG_RANDOM_VARIABLE_NAME,
command,
constants.LONG_RANDOM_VARIABLE_NAME)
return Command(command)
Popular sparkmagic functions
- sparkmagic.controllerwidget.abstractmenuwidget.AbstractMenuWidget
- sparkmagic.livyclientlib.command.Command
- sparkmagic.livyclientlib.exceptions.BadUserConfigurationException
- sparkmagic.livyclientlib.exceptions.BadUserDataException
- sparkmagic.livyclientlib.exceptions.HttpClientException
- sparkmagic.livyclientlib.exceptions.SessionManagementException
- sparkmagic.sparkmagic.livyclientlib.command.Command
- sparkmagic.sparkmagic.livyclientlib.exceptions.BadUserDataException
- sparkmagic.sparkmagic.livyclientlib.exceptions.LivyClientLibException
- sparkmagic.sparkmagic.livyclientlib.livyreliablehttpclient.LivyReliableHttpClient.from_endpoint
- sparkmagic.sparkmagic.livyclientlib.livysession.LivySession
- sparkmagic.utils.configuration
- sparkmagic.utils.configuration.default_maxrows
- sparkmagic.utils.configuration.default_samplefraction
- sparkmagic.utils.configuration.livy_server_heartbeat_timeout_seconds
- sparkmagic.utils.configuration.pyspark_dataframe_encoding
- sparkmagic.utils.constants
- sparkmagic.utils.constants.LONG_RANDOM_VARIABLE_NAME
- sparkmagic.utils.sparkevents.SparkEvents
- sparkmagic.utils.sparklogger.SparkLog