How to use hdijupyterutils - 10 common examples

from . import configuration as conf
import sparkmagic.utils.constants as constants


def get_spark_events_handler():
    """
    Create an instance from the handler mentioned in the config file.
    """
    module, class_name = conf.events_handler_class().rsplit('.', 1)
    events_handler_module = importlib.import_module(module)
    events_handler = getattr(events_handler_module, class_name)
    handler = events_handler(constants.MAGICS_LOGGER_NAME, conf.logging_config())
    return handler


class SparkEvents(Events):
    def __init__(self):
        handler = get_spark_events_handler()
        super(SparkEvents, self).__init__(handler)
    

    def emit_library_loaded_event(self):
        event_name = constants.LIBRARY_LOADED_EVENT
        time_stamp = self.get_utc_date_time()

        kwargs_list = [(EVENT_NAME, event_name),
                       (TIMESTAMP, time_stamp)]

        self.send_to_handler(kwargs_list)

    def emit_cluster_change_event(self, cluster_dns_name, status_code, success, error_message):
        event_name = constants.CLUSTER_CHANGE_EVENT

def __init__(self, spark_controller, ipywidget_factory=None, ipython_display=None,
                 nested_widget_mode=False, testing=False, **kwargs):
        kwargs['orientation'] = 'vertical'

        if not testing:
            super(AbstractMenuWidget, self).__init__((), **kwargs)

        self.spark_controller = spark_controller

        if ipywidget_factory is None:
            ipywidget_factory = IpyWidgetFactory()
        self.ipywidget_factory = ipywidget_factory

        if ipython_display is None:
            ipython_display = IpythonDisplay()
        self.ipython_display = ipython_display

        self.children = []

        if not nested_widget_mode:
            self._repr_html_()

session_language, user_code_parser=None, **kwargs):
        # Required by Jupyter - Override
        self.implementation = implementation
        self.implementation_version = implementation_version
        self.language = language
        self.language_version = language_version
        self.language_info = language_info

        # Override
        self.session_language = session_language

        super(SparkKernelBase, self).__init__(**kwargs)

        self.logger = SparkLog(u"{}_jupyter_kernel".format(self.session_language))
        self._fatal_error = None
        self.ipython_display = IpythonDisplay()

        if user_code_parser is None:
            self.user_code_parser = UserCodeParser()
        else:
            self.user_code_parser = user_code_parser

        # Disable warnings for test env in HDI
        requests.packages.urllib3.disable_warnings()

        if not kwargs.get("testing", False):
            self._load_magics_extension()
            self._change_language()
            if conf.use_auto_viz():
                self._register_auto_viz()

self.df = self._convert_to_displayable_dataframe(df)

        if renderer is None:
            renderer = GraphRenderer()
        self.renderer = renderer

        if ipywidget_factory is None:
            ipywidget_factory = IpyWidgetFactory()
        self.ipywidget_factory = ipywidget_factory

        if encoding_widget is None:
            encoding_widget = EncodingWidget(self.df, encoding, self.on_render_viz)
        self.encoding_widget = encoding_widget

        if ipython_display is None:
            ipython_display = IpythonDisplay()
        self.ipython_display = ipython_display

        self.encoding = encoding

        # Widget that will become the only child of AutoVizWidget
        self.widget = self.ipywidget_factory.get_vbox()

        # Create output area
        self.to_display = self.ipywidget_factory.get_output()
        self.to_display.width = "800px"
        self.output = self.ipywidget_factory.get_hbox()
        self.output.children = [self.to_display]

        self.controls = self._create_controls_widget()

        if spark_events is None:

def __init__(self, display=None):
        if display is None:
            self.display = IpythonDisplay()
        else:
            self.display = display

def __init__(self, shell, data=None, spark_events=None):
        # You must call the parent constructor
        super(SparkMagicBase, self).__init__(shell)

        self.logger = SparkLog(u"SparkMagics")
        self.ipython_display = IpythonDisplay()
        self.spark_controller = SparkController(self.ipython_display)

        self.logger.debug("Initialized spark magics.")

        if spark_events is None:
            spark_events = SparkEvents()
        spark_events.emit_library_loaded_event()

import textwrap

from hdijupyterutils.guid import ObjectWithGuid

import sparkmagic.utils.configuration as conf
from sparkmagic.utils.sparklogger import SparkLog
from sparkmagic.utils.sparkevents import SparkEvents
from sparkmagic.utils.constants import MAGICS_LOGGER_NAME, FINAL_STATEMENT_STATUS
from .exceptions import LivyUnexpectedStatusException


class Command(ObjectWithGuid):
    def __init__(self, code, spark_events=None):
        super(Command, self).__init__()
        self.code = textwrap.dedent(code)
        self.logger = SparkLog(u"Command")
        if spark_events is None:
            spark_events = SparkEvents()
        self._spark_events = spark_events

    def __eq__(self, other):
        return self.code == other.code

    def __ne__(self, other):
        return not self == other

    def execute(self, session):
        self._spark_events.emit_statement_execution_start_event(session.guid, session.kind, session.id, self.guid)

from hdijupyterutils.guid import ObjectWithGuid

from sparkmagic.utils.utils import coerce_pandas_df_to_numeric_datetime, records_to_dataframe
import sparkmagic.utils.configuration as conf
import sparkmagic.utils.constants as constants
from sparkmagic.utils.sparkevents import SparkEvents
from .command import Command
from .exceptions import DataFrameParseException, BadUserDataException


class SQLQuery(ObjectWithGuid):
    def __init__(self, query, samplemethod=None, maxrows=None, samplefraction=None, spark_events=None, coerce=None):
        super(SQLQuery, self).__init__()
        
        if samplemethod is None:
            samplemethod = conf.default_samplemethod()
        if maxrows is None:
            maxrows = conf.default_maxrows()
        if samplefraction is None:
            samplefraction = conf.default_samplefraction()

        if samplemethod not in {u'take', u'sample'}:
            raise BadUserDataException(u'samplemethod (-m) must be one of (take, sample)')
        if not isinstance(maxrows, int):
            raise BadUserDataException(u'maxrows (-n) must be an integer')
        if not 0.0 <= samplefraction <= 1.0:
            raise BadUserDataException(u'samplefraction (-r) must be a float between 0.0 and 1.0')

import json
import pandas as pd
from collections import OrderedDict
from hdijupyterutils.guid import ObjectWithGuid

from sparkmagic.utils.utils import coerce_pandas_df_to_numeric_datetime
import sparkmagic.utils.configuration as conf
import sparkmagic.utils.constants as constants
from sparkmagic.utils.sparkevents import SparkEvents
from .command import Command
from .exceptions import DataFrameParseException, BadUserDataException


class SQLQuery(ObjectWithGuid):
    def __init__(self, query, samplemethod=None, maxrows=None, samplefraction=None, spark_events=None):
        super(SQLQuery, self).__init__()
        
        if samplemethod is None:
            samplemethod = conf.default_samplemethod()
        if maxrows is None:
            maxrows = conf.default_maxrows()
        if samplefraction is None:
            samplefraction = conf.default_samplefraction()

        if samplemethod not in {u'take', u'sample'}:
            raise BadUserDataException(u'samplemethod (-m) must be one of (take, sample)')
        if not isinstance(maxrows, int):
            raise BadUserDataException(u'maxrows (-n) must be an integer')
        if not 0.0 <= samplefraction <= 1.0:
            raise BadUserDataException(u'samplefraction (-r) must be a float between 0.0 and 1.0')

def __init__(self, df, encoding, change_hook, ipywidget_factory=None, testing=False, **kwargs):
        assert encoding is not None
        assert df is not None
        assert type(df) is pd.DataFrame

        kwargs['orientation'] = 'vertical'
        if not testing:
            super(EncodingWidget, self).__init__((), **kwargs)

        if ipywidget_factory is None:
            ipywidget_factory = IpyWidgetFactory()
        self.ipywidget_factory = ipywidget_factory

        self.df = df
        self.encoding = encoding
        self.change_hook = change_hook

        self.widget = self.ipywidget_factory.get_vbox()

        self.title = self.ipywidget_factory.get_html('Encoding:', width='148px', height='32px')

        # X view
        options_x_view = {text(i): text(i) for i in self.df.columns}
        options_x_view["-"] = None
        self.x_view = self.ipywidget_factory.get_dropdown(options=options_x_view,
                                                          description="X", value=self.encoding.x)
        self.x_view.on_trait_change(self._x_changed_callback, 'value')