How to use the omniduct.utils.debug.logger function in omniduct

To help you get started, we’ve selected a few omniduct examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github airbnb / omniduct / omniduct / remotes / ssh_paramiko.py View on Github external
def _init(self):
        logger.warning("The Paramiko SSH client is still under development, \
                        and is not ready for use as a daily driver.")
github airbnb / omniduct / omniduct / caches / base.py View on Github external
logger.info("Unsetting key '{}' (namespace: '{}')...".format(row.key, row.namespace))
                self.unset(row.key, namespace=row.namespace)

        # Unset keys according to global constraints
        if total_bytes is not None or total_count is not None:
            if total_bytes is not None and not isinstance(total_bytes, int):
                raise ValueError("Invalid type specified for `total_bytes`: {}".format(total_bytes.__repr__()))
            if total_count is not None and not isinstance(total_count, int):
                raise ValueError("Invalid type specified for `total_count`: {}".format(total_bytes.__repr__()))
            usage = self.describe(namespaces=namespaces).assign(cum_bytes=lambda x: x.bytes.cumsum())

            unset_index = total_count if total_count is not None else len(usage)
            if total_bytes is not None:
                unset_index = min(unset_index, usage.cum_bytes.searchsorted(total_bytes, side='right'))
            for i, row in usage.loc[unset_index:].iterrows():
                logger.info("Unsetting key '{}' (namespace: '{}')...".format(row.key, row.namespace))
                self.unset(row.key, namespace=row.namespace)
github airbnb / omniduct / omniduct / databases / presto.py View on Github external
linenumber = message['errorLocation']['lineNumber'] - 1
                splt = statement.splitlines()
                splt[linenumber] += '   <--  {errorType} ({errorName}) occurred. {message} '.format(**message)
                context = '\n\n[Error Context]\n{}\n'.format('\n'.join([splt[l] for l in range(max(linenumber - 1, 0),
                                                                                               min(linenumber + 2, len(splt)))]))

                class ErrContext(object):

                    def __repr__(self):
                        return context

                # logged twice so that both notebook and console users see the error context
                exception_args.args = [exception_args, ErrContext()]
                logger.error(context)
            except:
                logger.warn(("Omniduct was unable to parse the database error messages. Refer to the "
                             "traceback below for full error details."))

            if isinstance(exception, type):
                exception = exception(exception_args)

            raise_with_traceback(exception, traceback)
github airbnb / omniduct / omniduct / utils / dependencies.py View on Github external
def check_dependencies(protocols, message=None):
    if protocols is None:
        return
    dependencies = []
    for protocol in protocols:
        dependencies.extend(__optional_dependencies__.get(protocol, []))
    missing_deps = []
    warning_deps = {}

    for dep in dependencies:
        m = re.match('^[a-z_][a-z0-9]*', dep)
        if not m:
            logger.warning('Invalid dependency requested: {}'.format(dep))

        package_name = m.group(0)
        accept_any_version = package_name == dep

        try:
            pkg_resources.get_distribution(dep)
        except VersionConflict:
            warning_deps[dep] = "{}=={}".format(package_name, pkg_resources.get_distribution(m.group(0)).version)
        except:
            # Some packages may be available, but not installed. If so, we
            # should accept them with warnings (if version specified in dep).
            try:
                importlib.import_module(package_name)
                if not accept_any_version:
                    warning_deps.append('{}=='.format(package_name))
            except:  # ImportError in python 2, ModuleNotFoundError in Python 3
github airbnb / omniduct / omniduct / databases / sqlalchemy.py View on Github external
def _connect(self):
        import sqlalchemy
        if self.protocol not in ['mysql']:
            logger.warning("While querying and executing should work as "
                           "expected, some operations on this database client "
                           "(such as listing tables, querying to tables, etc) "
                           "may not function as expected due to the backend "
                           "not supporting ANSI SQL.")

        self.engine = sqlalchemy.create_engine(self.db_uri, **self.engine_opts)
        self._sqlalchemy_metadata = sqlalchemy.MetaData(self.engine)
github airbnb / omniduct / omniduct / databases / _cursor_formatters.py View on Github external
def _format_dump(self, data):
        import pandas as pd

        df = pd.DataFrame(data=data, columns=self.column_names)

        if self.date_fields is not None:
            try:
                df = pd.io.sql._parse_date_columns(df, self.date_fields)
            except Exception as e:
                logger.warning('Unable to parse date columns. Perhaps your version of pandas is outdated.'
                               'Original error message was: {}: {}'.format(e.__class__.__name__, str(e)))

        if self.index_fields is not None:
            df.set_index(self.index_fields, inplace=True)

        return df