How to use the omniduct.utils.config.config function in omniduct

To help you get started, we’ve selected a few omniduct examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github airbnb / omniduct / omniduct / utils / debug.py View on Github external
def _scope_exit(self, success=True):
        if self._progress_bar is not None:
            self.progress(100, complete=True)
        props = self.__scopes[-1]
        if 'time' in props:
            logger.warning(
                "{} after {} on {}.".format(
                    'Complete' if success else 'Failed',
                    self.__get_time(time.time() - props['time']),
                    time.strftime('%Y-%m-%d')
                ) + (' CAVEATS: {}.'.format('; '.join(props['caveats'])) if props['caveats'] else '')
            )
        scope = self.__scopes.pop()
        if config.logging_level < logging.INFO:
            print("\t" * len(self.__scopes) + "Exited manual scope: {}".format(scope['name']), file=sys.stderr)
        elif 'has_logged' in scope:
            if len(self.__scopes) != 0:
                self.current_scope_props['has_logged'] = self.current_scope_props.get('has_logged') or props['has_logged']
github airbnb / omniduct / omniduct / caches / base.py View on Github external
_key,
                value=value,
                namespace=_namespace,
                serializer=_serializer,
                metadata=_metadata
            )
            # Return from cache every time, just in case serialization operation was
            # destructive (e.g. reading from cursors)
            return _cache.get(
                _key,
                namespace=_namespace,
                serializer=_serializer
            )
        except:
            logger.warning("Failed to save results to cache. If needed, please save them manually.")
            if config.cache_fail_hard:
                six.reraise(*sys.exc_info())
            return value  # As a last resort, return value object (which could be mutated by serialization).
github airbnb / omniduct / omniduct / session.py View on Github external
from .utils.config import config

__all__ = ['config', 'registry']


# The default console width is too wide for most notebooks, leading to ugly logging message / progress bars. We set this
# to a more reasonable value for Jupyter Notebooks.
ip = IPython.get_ipython()
if ip and ip.__class__.__name__ == "ZMQInteractiveShell":
    os.environ['COLUMNS'] = "80"


OMNIDUCT_CONFIG = os.environ.get('OMNIDUCT_CONFIG', None) or os.path.expanduser('~/.omniduct/config')

config.register("ducts", "The ducts to register with the system.")

config._config_path = OMNIDUCT_CONFIG

registry = DuctRegistry(getattr(config, 'ducts', {}))
registry.populate_namespace(globals())
github airbnb / omniduct / omniduct / utils / debug.py View on Github external
def progress(self, progress=None, complete=False, indeterminate=False):
        """
        Set the current progress to `progress`, and if not already showing, display
        a progress bar. If `complete` evaluates to True, then finish displaying the progress.
        """
        complete = complete or (self.current_scope_props is None)  # Only leave progress bar open if within a scope
        if config.logging_level <= logging.INFO:
            self.__get_progress_bar(indeterminate=indeterminate).update(progress)
            if complete:
                self.__get_progress_bar().finish(end=None)
                self._progress_bar = None
github airbnb / omniduct / omniduct / utils / processes.py View on Github external
cmd : string
        Command to be executed in subprocess.
    kwargs : keywords
        Options to pass to subprocess.Popen.

    Returns
    -------
    proc : Popen subprocess
        Subprocess used to run command.
    """

    logger.debug('Executing command: {0}'.format(cmd))
    config = DEFAULT_SUBPROCESS_CONFIG.copy()
    config.update(kwargs)
    if not check_output:
        if omniduct_config.logging_level < 20:
            config['stdout'] = None
            config['stderr'] = None
        else:
            config['stdout'] = open(os.devnull, 'w')
            config['stderr'] = open(os.devnull, 'w')
    timeout = config.pop('timeout', None)

    process = subprocess.Popen(cmd, **config)
    try:
        stdout, stderr = process.communicate(None, timeout=timeout)
    except subprocess.TimeoutExpired:
        os.killpg(os.getpgid(process.pid), signal.SIGINT)  # send signal to the process group, recursively killing all children
        output, unused_err = process.communicate()
        raise subprocess.TimeoutExpired(process.args, timeout, output=output)
    return SubprocessResults(returncode=process.returncode, stdout=stdout or b'', stderr=stderr or b'')
github airbnb / omniduct / omniduct / session.py View on Github external
# The default console width is too wide for most notebooks, leading to ugly logging message / progress bars. We set this
# to a more reasonable value for Jupyter Notebooks.
ip = IPython.get_ipython()
if ip and ip.__class__.__name__ == "ZMQInteractiveShell":
    os.environ['COLUMNS'] = "80"


OMNIDUCT_CONFIG = os.environ.get('OMNIDUCT_CONFIG', None) or os.path.expanduser('~/.omniduct/config')

config.register("ducts", "The ducts to register with the system.")

config._config_path = OMNIDUCT_CONFIG

registry = DuctRegistry(getattr(config, 'ducts', {}))
registry.populate_namespace(globals())
github airbnb / omniduct / omniduct / caches / base.py View on Github external
import dateutil
import pandas
import six
import yaml
from decorator import decorator
from interface_meta import quirk_docs

from omniduct.duct import Duct
from omniduct.utils.config import config
from omniduct.utils.debug import logger
from omniduct.utils.decorators import function_args_as_kwargs, require_connection

from ._serializers import PickleSerializer

config.register(
    'cache_fail_hard',
    description='Raise an exception if a cache fails to save (otherwise errors are logged and suppressed).',
    default=False
)


def cached_method(
        key,
        namespace=lambda self, kwargs: (
            self.cache_namespace or "{}.{}".format(self.__class__.__name__, self.name)
        ),
        cache=lambda self, kwargs: self.cache,
        use_cache=lambda self, kwargs: kwargs.pop('use_cache', True),
        renew=lambda self, kwargs: kwargs.pop('renew', False),
        serializer=lambda self, kwargs: PickleSerializer(),
        metadata=lambda self, kwargs: None
github airbnb / omniduct / omniduct / utils / debug.py View on Github external
from __future__ import print_function

import inspect
import logging
import sys
import time

import progressbar
import six
from decorator import decorate
from future.utils import raise_with_traceback

from .config import config

config.register('logging_level',
                description='Set the default logging level.',
                default=logging.INFO,
                onchange=lambda level: logger.setLevel(level, context='omniduct'))


class StatusLogger(object):
    """
    StatusLogger is a wrapper around `logging.Logger` that allows for consistent
    treatment of logging messages. While not strictly required,
    it simplifies and abstracts the usage of the logging module within omniduct.
    It also adds support for displaying progress bars.

    Instances of StatusLogger are proxies for `logging.Logger` objects, and thus
    inherit the methods and properties of Logger. For example, to log a warning, use:

    >>> StatusLogger().warn('test')