How to use the gramex.data.filter function in gramex

To help you get started, we’ve selected a few gramex examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github gramener / gramex / testlib / test_data.py View on Github external
def check_filter_dates(self, dbname, url):
        self.db.add(dbname)
        df = self.dates[self.dates['date'] > '2018-02-01 01:00:00']
        dff = gramex.data.filter(url=url, table='dates', args={'date>': ['2018-02-01 01:00:00']})
        eqframe(dff, df)
github gramener / gramex / testlib / test_data.py View on Github external
def test_file(self):
        self.check_filter(url=sales_file)
        afe(
            gramex.data.filter(url=sales_file, transform='2.1', sheet_name='dummy'),
            gramex.cache.open(sales_file, 'excel', transform='2.2', sheet_name='dummy'),
        )
        self.check_filter(
            url=sales_file,
            transform=lambda d: d[d['sales'] > 100],
            df=self.sales[self.sales['sales'] > 100],
        )
        with assert_raises(ValueError):
            gramex.data.filter(url='', engine='nonexistent')
        with assert_raises(OSError):
            gramex.data.filter(url='nonexistent')
        with assert_raises(TypeError):
            gramex.data.filter(url=os.path.join(folder, 'test_cache_module.py'))
github gramener / gramex / testlib / test_data.py View on Github external
def eq(args, expected, **eqkwargs):
            meta = {}
            actual = gramex.data.filter(meta=meta, args=args, **kwargs)
            eqframe(actual, expected, **eqkwargs)
            return meta
github gramener / gramex / testlib / test_data.py View on Github external
def test_file(self):
        self.check_filter(url=sales_file)
        afe(
            gramex.data.filter(url=sales_file, transform='2.1', sheet_name='dummy'),
            gramex.cache.open(sales_file, 'excel', transform='2.2', sheet_name='dummy'),
        )
        self.check_filter(
            url=sales_file,
            transform=lambda d: d[d['sales'] > 100],
            df=self.sales[self.sales['sales'] > 100],
        )
        with assert_raises(ValueError):
            gramex.data.filter(url='', engine='nonexistent')
        with assert_raises(OSError):
            gramex.data.filter(url='nonexistent')
        with assert_raises(TypeError):
            gramex.data.filter(url=os.path.join(folder, 'test_cache_module.py'))
github gramener / gramex / gramex / apps / admin2 / gramexadmin.py View on Github external
def schedule(handler, service):
    if handler.request.method == 'GET':
        data = get_schedule(service)
        data = gramex.data.filter(data, handler.args)
        # TODO: handle _format, _meta, _download, etc just like FormHandler
        raise Return(gramex.data.download(data))
    elif handler.request.method == 'POST':
        key = handler.get_argument('name')
        schedule = gramex.service[service][key]
        results, kwargs = [], {}
        # If ?mock is set, and it's an alert, capture the alert mails in result
        if handler.get_argument('mock', False) and service == 'alert':
            kwargs = {'callback': lambda **kwargs: results.append(kwargs)}
        if schedule.thread:
            args = yield schedule.function(**kwargs)
        else:
            args = yield gramex.service.threadpool.submit(schedule.function, **kwargs)
        if service == 'alert' and isinstance(args, dict):
            for arg in args.get('fail', []):
                raise arg['error']
github gramener / gramex / gramex / pptgen / __init__.py View on Github external
def load_data(data_config, handler=None):
    '''
    Loads data using gramex cache.
    '''
    if not isinstance(data_config, (dict, AttrDict,)):
        raise ValueError('Data argument must be a dict like object.')

    data = {}
    for key, conf in data_config.items():
        if isinstance(conf, (dict, AttrDict,)):
            if 'function' in conf:
                data[key] = build_transform(conf, vars={'handler': None})(handler=handler)[0]
            elif conf.get('ext') in {'yaml', 'yml', 'json'}:
                data[key] = gramex.cache.open(conf.pop('url'), conf.pop('ext'), **dict(conf))
            elif 'url' in conf:
                data[key] = gramex.data.filter(conf.pop('url'), **dict(conf))
        else:
            data[key] = conf
    return data
github gramener / gramex / gramex / handlers / modelhandler.py View on Github external
def _train(self, model):
        ''' Looks for Model-Retrain in Request Headers,
        trains a model and pickles it.
        '''
        # Update model parameters
        model.update_params(self.request_body)
        if 'Model-Retrain' in self.request.headers:
            # Pass non model kwargs to gramex.data.filter
            try:
                data = gramex.data.filter(
                    model.url,
                    args=self.listify(['engine', 'url', 'ext', 'table', 'query', 'id']))
            except AttributeError:
                raise AttributeError('Model does not have a url')
            # Train the model.
            model.train(data)
            model.trained = True
            model.save(self.pickle_file_path)
            return True
github gramener / gramex / gramex / handlers / formhandler.py View on Github external
class FormHandler(BaseHandler):
    '''
    # Else there should be at least 1 key that has a url: sub-key. The data spec is at that level
    # Data spec is (url, engine, table, ext, ...) which goes directly to filter
    # It also has
    #   default: which is interpreted as argument defaults
    #   keys: defines the primary key columns
    '''
    # FormHandler function kwargs and the parameters they accept:
    function_vars = {
        'modify': {'data': None, 'key': None, 'handler': None},
        'prepare': {'args': None, 'key': None, 'handler': None},
        'queryfunction': {'args': None, 'key': None, 'handler': None},
    }
    data_filter_method = staticmethod(gramex.data.filter)

    @classmethod
    def setup(cls, **kwargs):
        super(FormHandler, cls).setup(**kwargs)
        conf_kwargs = merge(AttrDict(cls.conf.kwargs),
                            objectpath(gramex_conf, 'handlers.FormHandler', {}),
                            'setdefault')
        cls.headers = conf_kwargs.pop('headers', {})
        # Top level formats: key is special. Don't treat it as data
        cls.formats = conf_kwargs.pop('formats', {})
        default_config = conf_kwargs.pop('default', None)
        # Remove other known special keys from dataset configuration
        cls.clear_special_keys(conf_kwargs)
        # If top level has url: then data spec is at top level. Else it's a set of sub-keys
        if 'url' in conf_kwargs:
            cls.datasets = {'data': conf_kwargs}