How to use the omniduct.databases.base.DatabaseClient function in omniduct

To help you get started, we’ve selected a few omniduct examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github airbnb / omniduct / omniduct / databases / druid.py View on Github external
from __future__ import absolute_import

from interface_meta import override

from omniduct.utils.debug import logger

from .base import DatabaseClient


class DruidClient(DatabaseClient):
    """
    This Duct connects to a Druid server using the `pydruid` python library.
    """

    PROTOCOLS = ['druid']
    DEFAULT_PORT = 80
    NAMESPACE_NAMES = ['table']
    NAMESPACE_QUOTECHAR = '"'
    NAMESPACE_SEPARATOR = '.'

    @override
    def _init(self):
        self.__druid = None

    # Connection
    @override
github airbnb / omniduct / omniduct / databases / neo4j.py View on Github external
from __future__ import absolute_import

from interface_meta import override

from omniduct.utils.debug import logger

from .base import DatabaseClient


class Neo4jClient(DatabaseClient):
    """
    This Duct connects to a Neo4j graph database server using the `neo4j` python
    library.
    """

    PROTOCOLS = ['neo4j']
    DEFAULT_PORT = 7687
    DEFAULT_CURSOR_FORMATTER = 'raw'

    @override
    @classmethod
    def statement_cleanup(cls, statement):
        return statement  # base statement cleanup assumes SQL

    @override
    def _init(self):
github airbnb / omniduct / omniduct / databases / pyspark.py View on Github external
from interface_meta import override

from omniduct.databases.base import DatabaseClient
from omniduct.databases.hiveserver2 import HiveServer2Client


class PySparkClient(DatabaseClient):
    """
    This Duct connects to a local PySpark session using the `pyspark` library.
    """

    PROTOCOLS = ['pyspark']
    DEFAULT_PORT = None
    SUPPORTS_SESSION_PROPERTIES = True
    NAMESPACE_NAMES = ['schema', 'table']
    NAMESPACE_QUOTECHAR = '`'
    NAMESPACE_SEPARATOR = '.'

    @override
    def _init(self, app_name='omniduct', config=None, master=None, enable_hive_support=False):
        """
        Args:
            app_name (str): The application name of the SparkSession.
github airbnb / omniduct / omniduct / databases / stub.py View on Github external
from omniduct.databases.base import DatabaseClient


class StubDatabaseClient(DatabaseClient):

    PROTOCOLS = []
    DEFAULT_PORT = None

    def _init(self):
        pass

    # Connection management

    def _connect(self):
        raise NotImplementedError

    def _is_connected(self):
        raise NotImplementedError

    def _disconnect(self):
github airbnb / omniduct / omniduct / databases / sqlalchemy.py View on Github external
from __future__ import absolute_import

from interface_meta import override

from omniduct.utils.debug import logger

from .base import DatabaseClient
from ._schemas import SchemasMixin
from . import _pandas


class SQLAlchemyClient(DatabaseClient, SchemasMixin):
    """
    This Duct connects to several different databases using one of several
    SQLAlchemy drivers. In general, these are provided for their potential
    utility, but will be less functional than the specially crafted database
    clients.
    """

    PROTOCOLS = ['sqlalchemy', 'firebird', 'mssql', 'mysql', 'oracle', 'postgresql', 'sybase', 'snowflake']
    NAMESPACE_NAMES = ['database', 'table']
    NAMESPACE_QUOTECHAR = '"'  # TODO: Apply overrides depending on protocol?
    NAMESPACE_SEPARATOR = '.'

    @property
    @override
    def NAMESPACE_DEFAULT(self):
        return {
github airbnb / omniduct / omniduct / databases / hiveserver2.py View on Github external
import tempfile
import time

import pandas as pd
from interface_meta import override
from jinja2 import Template

from omniduct.utils.debug import logger
from omniduct.utils.processes import Timeout, run_in_subprocess

from .base import DatabaseClient
from ._schemas import SchemasMixin
from . import _pandas


class HiveServer2Client(DatabaseClient, SchemasMixin):
    """
    This Duct connects to an Apache HiveServer2 server instance using the
    `pyhive` or `impyla` libraries.

    Attributes:
        schema (str, None): The default schema to use for queries (will
            default to server-default if not specified).
        driver (str): One of 'pyhive' (default) or 'impyla', which specifies
            how the client communicates with Hive.
        auth_mechanism (str): The authorisation protocol to use for connections.
            Defaults to 'NOSASL'. Authorisation methods differ between drivers.
            Please refer to `pyhive` and `impyla` documentation for more details.
        push_using_hive_cli (bool): Whether the `.push()` operation should
            directly add files using `LOAD DATA LOCAL INPATH` rather than the
            `INSERT` operation via SQLAlchemy. Note that this requires the
            presence of the `hive` executable on the local PATH, or if
github airbnb / omniduct / omniduct / databases / presto.py View on Github external
import re
import sys

import pandas.io.sql
import six
from interface_meta import override
from future.utils import raise_with_traceback

from omniduct.utils.debug import logger

from .base import DatabaseClient
from ._schemas import SchemasMixin
from . import _pandas


class PrestoClient(DatabaseClient, SchemasMixin):
    """
    This Duct connects to a Facebook Presto server instance using the `pyhive`
    library.

    In addition to the standard `DatabaseClient` API, `PrestoClient` adds a
    `.schemas` descriptor attribute, which enables a tab completion driven
    exploration of a Presto database's schemas and tables.

    Attributes:
        catalog (str): The default catalog to use in database queries.
        schema (str): The default schema/database to use in database queries.
        connection_options (dict): Additional options to pass on to
            `pyhive.presto.connect(...)`.
    """

    PROTOCOLS = ['presto']