Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def _scope_exit(self, success=True):
if self._progress_bar is not None:
self.progress(100, complete=True)
props = self.__scopes[-1]
if 'time' in props:
logger.warning(
"{} after {} on {}.".format(
'Complete' if success else 'Failed',
self.__get_time(time.time() - props['time']),
time.strftime('%Y-%m-%d')
) + (' CAVEATS: {}.'.format('; '.join(props['caveats'])) if props['caveats'] else '')
)
scope = self.__scopes.pop()
if config.logging_level < logging.INFO:
print("\t" * len(self.__scopes) + "Exited manual scope: {}".format(scope['name']), file=sys.stderr)
elif 'has_logged' in scope:
if len(self.__scopes) != 0:
self.current_scope_props['has_logged'] = self.current_scope_props.get('has_logged') or props['has_logged']
_key,
value=value,
namespace=_namespace,
serializer=_serializer,
metadata=_metadata
)
# Return from cache every time, just in case serialization operation was
# destructive (e.g. reading from cursors)
return _cache.get(
_key,
namespace=_namespace,
serializer=_serializer
)
except:
logger.warning("Failed to save results to cache. If needed, please save them manually.")
if config.cache_fail_hard:
six.reraise(*sys.exc_info())
return value # As a last resort, return value object (which could be mutated by serialization).
from .utils.config import config
__all__ = ['config', 'registry']
# The default console width is too wide for most notebooks, leading to ugly logging message / progress bars. We set this
# to a more reasonable value for Jupyter Notebooks.
ip = IPython.get_ipython()
if ip and ip.__class__.__name__ == "ZMQInteractiveShell":
os.environ['COLUMNS'] = "80"
OMNIDUCT_CONFIG = os.environ.get('OMNIDUCT_CONFIG', None) or os.path.expanduser('~/.omniduct/config')
config.register("ducts", "The ducts to register with the system.")
config._config_path = OMNIDUCT_CONFIG
registry = DuctRegistry(getattr(config, 'ducts', {}))
registry.populate_namespace(globals())
def progress(self, progress=None, complete=False, indeterminate=False):
"""
Set the current progress to `progress`, and if not already showing, display
a progress bar. If `complete` evaluates to True, then finish displaying the progress.
"""
complete = complete or (self.current_scope_props is None) # Only leave progress bar open if within a scope
if config.logging_level <= logging.INFO:
self.__get_progress_bar(indeterminate=indeterminate).update(progress)
if complete:
self.__get_progress_bar().finish(end=None)
self._progress_bar = None
cmd : string
Command to be executed in subprocess.
kwargs : keywords
Options to pass to subprocess.Popen.
Returns
-------
proc : Popen subprocess
Subprocess used to run command.
"""
logger.debug('Executing command: {0}'.format(cmd))
config = DEFAULT_SUBPROCESS_CONFIG.copy()
config.update(kwargs)
if not check_output:
if omniduct_config.logging_level < 20:
config['stdout'] = None
config['stderr'] = None
else:
config['stdout'] = open(os.devnull, 'w')
config['stderr'] = open(os.devnull, 'w')
timeout = config.pop('timeout', None)
process = subprocess.Popen(cmd, **config)
try:
stdout, stderr = process.communicate(None, timeout=timeout)
except subprocess.TimeoutExpired:
os.killpg(os.getpgid(process.pid), signal.SIGINT) # send signal to the process group, recursively killing all children
output, unused_err = process.communicate()
raise subprocess.TimeoutExpired(process.args, timeout, output=output)
return SubprocessResults(returncode=process.returncode, stdout=stdout or b'', stderr=stderr or b'')
# The default console width is too wide for most notebooks, leading to ugly logging message / progress bars. We set this
# to a more reasonable value for Jupyter Notebooks.
ip = IPython.get_ipython()
if ip and ip.__class__.__name__ == "ZMQInteractiveShell":
os.environ['COLUMNS'] = "80"
OMNIDUCT_CONFIG = os.environ.get('OMNIDUCT_CONFIG', None) or os.path.expanduser('~/.omniduct/config')
config.register("ducts", "The ducts to register with the system.")
config._config_path = OMNIDUCT_CONFIG
registry = DuctRegistry(getattr(config, 'ducts', {}))
registry.populate_namespace(globals())
import dateutil
import pandas
import six
import yaml
from decorator import decorator
from interface_meta import quirk_docs
from omniduct.duct import Duct
from omniduct.utils.config import config
from omniduct.utils.debug import logger
from omniduct.utils.decorators import function_args_as_kwargs, require_connection
from ._serializers import PickleSerializer
config.register(
'cache_fail_hard',
description='Raise an exception if a cache fails to save (otherwise errors are logged and suppressed).',
default=False
)
def cached_method(
key,
namespace=lambda self, kwargs: (
self.cache_namespace or "{}.{}".format(self.__class__.__name__, self.name)
),
cache=lambda self, kwargs: self.cache,
use_cache=lambda self, kwargs: kwargs.pop('use_cache', True),
renew=lambda self, kwargs: kwargs.pop('renew', False),
serializer=lambda self, kwargs: PickleSerializer(),
metadata=lambda self, kwargs: None
from __future__ import print_function
import inspect
import logging
import sys
import time
import progressbar
import six
from decorator import decorate
from future.utils import raise_with_traceback
from .config import config
config.register('logging_level',
description='Set the default logging level.',
default=logging.INFO,
onchange=lambda level: logger.setLevel(level, context='omniduct'))
class StatusLogger(object):
"""
StatusLogger is a wrapper around `logging.Logger` that allows for consistent
treatment of logging messages. While not strictly required,
it simplifies and abstracts the usage of the logging module within omniduct.
It also adds support for displaying progress bars.
Instances of StatusLogger are proxies for `logging.Logger` objects, and thus
inherit the methods and properties of Logger. For example, to log a warning, use:
>>> StatusLogger().warn('test')