Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
from __future__ import absolute_import
from interface_meta import override
from omniduct.utils.debug import logger
from .base import DatabaseClient
class DruidClient(DatabaseClient):
"""
This Duct connects to a Druid server using the `pydruid` python library.
"""
PROTOCOLS = ['druid']
DEFAULT_PORT = 80
NAMESPACE_NAMES = ['table']
NAMESPACE_QUOTECHAR = '"'
NAMESPACE_SEPARATOR = '.'
@override
def _init(self):
self.__druid = None
# Connection
@override
from __future__ import absolute_import
from interface_meta import override
from omniduct.utils.debug import logger
from .base import DatabaseClient
class Neo4jClient(DatabaseClient):
"""
This Duct connects to a Neo4j graph database server using the `neo4j` python
library.
"""
PROTOCOLS = ['neo4j']
DEFAULT_PORT = 7687
DEFAULT_CURSOR_FORMATTER = 'raw'
@override
@classmethod
def statement_cleanup(cls, statement):
return statement # base statement cleanup assumes SQL
@override
def _init(self):
from interface_meta import override
from omniduct.databases.base import DatabaseClient
from omniduct.databases.hiveserver2 import HiveServer2Client
class PySparkClient(DatabaseClient):
"""
This Duct connects to a local PySpark session using the `pyspark` library.
"""
PROTOCOLS = ['pyspark']
DEFAULT_PORT = None
SUPPORTS_SESSION_PROPERTIES = True
NAMESPACE_NAMES = ['schema', 'table']
NAMESPACE_QUOTECHAR = '`'
NAMESPACE_SEPARATOR = '.'
@override
def _init(self, app_name='omniduct', config=None, master=None, enable_hive_support=False):
"""
Args:
app_name (str): The application name of the SparkSession.
from omniduct.databases.base import DatabaseClient
class StubDatabaseClient(DatabaseClient):
PROTOCOLS = []
DEFAULT_PORT = None
def _init(self):
pass
# Connection management
def _connect(self):
raise NotImplementedError
def _is_connected(self):
raise NotImplementedError
def _disconnect(self):
from __future__ import absolute_import
from interface_meta import override
from omniduct.utils.debug import logger
from .base import DatabaseClient
from ._schemas import SchemasMixin
from . import _pandas
class SQLAlchemyClient(DatabaseClient, SchemasMixin):
"""
This Duct connects to several different databases using one of several
SQLAlchemy drivers. In general, these are provided for their potential
utility, but will be less functional than the specially crafted database
clients.
"""
PROTOCOLS = ['sqlalchemy', 'firebird', 'mssql', 'mysql', 'oracle', 'postgresql', 'sybase', 'snowflake']
NAMESPACE_NAMES = ['database', 'table']
NAMESPACE_QUOTECHAR = '"' # TODO: Apply overrides depending on protocol?
NAMESPACE_SEPARATOR = '.'
@property
@override
def NAMESPACE_DEFAULT(self):
return {
import tempfile
import time
import pandas as pd
from interface_meta import override
from jinja2 import Template
from omniduct.utils.debug import logger
from omniduct.utils.processes import Timeout, run_in_subprocess
from .base import DatabaseClient
from ._schemas import SchemasMixin
from . import _pandas
class HiveServer2Client(DatabaseClient, SchemasMixin):
"""
This Duct connects to an Apache HiveServer2 server instance using the
`pyhive` or `impyla` libraries.
Attributes:
schema (str, None): The default schema to use for queries (will
default to server-default if not specified).
driver (str): One of 'pyhive' (default) or 'impyla', which specifies
how the client communicates with Hive.
auth_mechanism (str): The authorisation protocol to use for connections.
Defaults to 'NOSASL'. Authorisation methods differ between drivers.
Please refer to `pyhive` and `impyla` documentation for more details.
push_using_hive_cli (bool): Whether the `.push()` operation should
directly add files using `LOAD DATA LOCAL INPATH` rather than the
`INSERT` operation via SQLAlchemy. Note that this requires the
presence of the `hive` executable on the local PATH, or if
import re
import sys
import pandas.io.sql
import six
from interface_meta import override
from future.utils import raise_with_traceback
from omniduct.utils.debug import logger
from .base import DatabaseClient
from ._schemas import SchemasMixin
from . import _pandas
class PrestoClient(DatabaseClient, SchemasMixin):
"""
This Duct connects to a Facebook Presto server instance using the `pyhive`
library.
In addition to the standard `DatabaseClient` API, `PrestoClient` adds a
`.schemas` descriptor attribute, which enables a tab completion driven
exploration of a Presto database's schemas and tables.
Attributes:
catalog (str): The default catalog to use in database queries.
schema (str): The default schema/database to use in database queries.
connection_options (dict): Additional options to pass on to
`pyhive.presto.connect(...)`.
"""
PROTOCOLS = ['presto']