Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def check_filter_dates(self, dbname, url):
self.db.add(dbname)
df = self.dates[self.dates['date'] > '2018-02-01 01:00:00']
dff = gramex.data.filter(url=url, table='dates', args={'date>': ['2018-02-01 01:00:00']})
eqframe(dff, df)
def test_file(self):
self.check_filter(url=sales_file)
afe(
gramex.data.filter(url=sales_file, transform='2.1', sheet_name='dummy'),
gramex.cache.open(sales_file, 'excel', transform='2.2', sheet_name='dummy'),
)
self.check_filter(
url=sales_file,
transform=lambda d: d[d['sales'] > 100],
df=self.sales[self.sales['sales'] > 100],
)
with assert_raises(ValueError):
gramex.data.filter(url='', engine='nonexistent')
with assert_raises(OSError):
gramex.data.filter(url='nonexistent')
with assert_raises(TypeError):
gramex.data.filter(url=os.path.join(folder, 'test_cache_module.py'))
def eq(args, expected, **eqkwargs):
meta = {}
actual = gramex.data.filter(meta=meta, args=args, **kwargs)
eqframe(actual, expected, **eqkwargs)
return meta
def test_file(self):
self.check_filter(url=sales_file)
afe(
gramex.data.filter(url=sales_file, transform='2.1', sheet_name='dummy'),
gramex.cache.open(sales_file, 'excel', transform='2.2', sheet_name='dummy'),
)
self.check_filter(
url=sales_file,
transform=lambda d: d[d['sales'] > 100],
df=self.sales[self.sales['sales'] > 100],
)
with assert_raises(ValueError):
gramex.data.filter(url='', engine='nonexistent')
with assert_raises(OSError):
gramex.data.filter(url='nonexistent')
with assert_raises(TypeError):
gramex.data.filter(url=os.path.join(folder, 'test_cache_module.py'))
def schedule(handler, service):
if handler.request.method == 'GET':
data = get_schedule(service)
data = gramex.data.filter(data, handler.args)
# TODO: handle _format, _meta, _download, etc just like FormHandler
raise Return(gramex.data.download(data))
elif handler.request.method == 'POST':
key = handler.get_argument('name')
schedule = gramex.service[service][key]
results, kwargs = [], {}
# If ?mock is set, and it's an alert, capture the alert mails in result
if handler.get_argument('mock', False) and service == 'alert':
kwargs = {'callback': lambda **kwargs: results.append(kwargs)}
if schedule.thread:
args = yield schedule.function(**kwargs)
else:
args = yield gramex.service.threadpool.submit(schedule.function, **kwargs)
if service == 'alert' and isinstance(args, dict):
for arg in args.get('fail', []):
raise arg['error']
def load_data(data_config, handler=None):
'''
Loads data using gramex cache.
'''
if not isinstance(data_config, (dict, AttrDict,)):
raise ValueError('Data argument must be a dict like object.')
data = {}
for key, conf in data_config.items():
if isinstance(conf, (dict, AttrDict,)):
if 'function' in conf:
data[key] = build_transform(conf, vars={'handler': None})(handler=handler)[0]
elif conf.get('ext') in {'yaml', 'yml', 'json'}:
data[key] = gramex.cache.open(conf.pop('url'), conf.pop('ext'), **dict(conf))
elif 'url' in conf:
data[key] = gramex.data.filter(conf.pop('url'), **dict(conf))
else:
data[key] = conf
return data
def _train(self, model):
''' Looks for Model-Retrain in Request Headers,
trains a model and pickles it.
'''
# Update model parameters
model.update_params(self.request_body)
if 'Model-Retrain' in self.request.headers:
# Pass non model kwargs to gramex.data.filter
try:
data = gramex.data.filter(
model.url,
args=self.listify(['engine', 'url', 'ext', 'table', 'query', 'id']))
except AttributeError:
raise AttributeError('Model does not have a url')
# Train the model.
model.train(data)
model.trained = True
model.save(self.pickle_file_path)
return True
class FormHandler(BaseHandler):
'''
# Else there should be at least 1 key that has a url: sub-key. The data spec is at that level
# Data spec is (url, engine, table, ext, ...) which goes directly to filter
# It also has
# default: which is interpreted as argument defaults
# keys: defines the primary key columns
'''
# FormHandler function kwargs and the parameters they accept:
function_vars = {
'modify': {'data': None, 'key': None, 'handler': None},
'prepare': {'args': None, 'key': None, 'handler': None},
'queryfunction': {'args': None, 'key': None, 'handler': None},
}
data_filter_method = staticmethod(gramex.data.filter)
@classmethod
def setup(cls, **kwargs):
super(FormHandler, cls).setup(**kwargs)
conf_kwargs = merge(AttrDict(cls.conf.kwargs),
objectpath(gramex_conf, 'handlers.FormHandler', {}),
'setdefault')
cls.headers = conf_kwargs.pop('headers', {})
# Top level formats: key is special. Don't treat it as data
cls.formats = conf_kwargs.pop('formats', {})
default_config = conf_kwargs.pop('default', None)
# Remove other known special keys from dataset configuration
cls.clear_special_keys(conf_kwargs)
# If top level has url: then data spec is at top level. Else it's a set of sub-keys
if 'url' in conf_kwargs:
cls.datasets = {'data': conf_kwargs}