How to use the dataflows.update_resource function in dataflows

To help you get started, we’ve selected a few dataflows examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github datopian / ckan-cloud-operator / ckan_cloud_operator / dataflows / resources.py View on Github external
def main_flow(prefix, operator):
    return Flow(
        cluster_info(operator),
        update_resource(['res_1'], name='cluster-info', path='cluster-info.csv'),
        checkpoint(f'{prefix}-cluster-info'),
        ckan_cloud_instances(operator),
        update_resource(['res_2'], name='ckan-cloud-instances', path='ckan-cloud-instances.csv'),
    )
github datopian / ckan-cloud-operator / ckan_cloud_operator / dataflows / resources.py View on Github external
def main_flow(prefix, operator):
    return Flow(
        cluster_info(operator),
        update_resource(['res_1'], name='cluster-info', path='cluster-info.csv'),
        checkpoint(f'{prefix}-cluster-info'),
        ckan_cloud_instances(operator),
        update_resource(['res_2'], name='ckan-cloud-instances', path='ckan-cloud-instances.csv'),
    )
github frictionlessdata / datapackage-pipelines / datapackage_pipelines / lib / concatenate.py View on Github external
def flow(parameters):
    return Flow(
        concatenate(
            parameters.get('fields', {}),
            parameters.get('target', {}),
            parameters.get('resources')
        ),
        update_resource(
            parameters.get('target', {}).get('name', 'concat'),
            **{
                PROP_STREAMING: True
            }
github datasets / airport-codes / airport-codes-flow.py View on Github external
"path": "http://ourairports.com/data/",
              "title": "Our Airports"
            }
        ],
        readme=readme()
    ),
    add_computed_field([{
        "operation": "format",
        "target": "coordinates",
        "with": "{latitude_deg}, {longitude_deg}"
    }]),
    delete_fields(fields=[
        "id","longitude_deg","latitude_deg",
        "scheduled_service","home_link","wikipedia_link","keywords"
    ]),
    update_resource('airport-codes', **{'path':'data/airport-codes.csv'}),
    validate(),
    dump_to_path()
)


def flow(parameters, datapackage, resources, stats):
    return dialing_info_cldr


if __name__ == '__main__':
    dialing_info_cldr.process()
github frictionlessdata / datapackage-pipelines / datapackage_pipelines / lib / join.py View on Github external
def flow(parameters):
    source = parameters['source']
    target = parameters['target']
    return Flow(
        load_lazy_json(source['name']),
        join(
            source['name'],
            source['key'],
            target['name'],
            target['key'],
            parameters['fields'],
            parameters.get('full', True),
            source.get('delete', False)
        ),
        update_resource(
            target['name'],
            **{
                PROP_STREAMING: True
            }
github datasets / natural-gas / natural_gas_flow.py View on Github external
headers=['Date', 'Price'],
        name='daily'
    ),
    load(
        load_source='http://www.eia.gov/dnav/ng/hist_xls/RNGWHHDm.xls',
        format='xls',
        sheet=2,
        skip_rows=[1,2,3,-1],
        headers=['Month', 'Price'],
        name='monthly'
    ),
    format_date,
    set_type('Date', resources='daily', type='date'),
    set_type('Month',resources='monthly', type='yearmonth'),
    update_resource('daily', **{'path':'data/daily.csv', 'dpp:streaming': True}),
    update_resource('monthly', **{'path':'data/monthly.csv', 'dpp:streaming': True}),
    validate()
)


def flow(parameters, datapackage, resources, stats):
    return natural_gas


if __name__ == '__main__':
    natural_gas.process()
github datasets / finance-vix / finance_vix_flow.py View on Github external
{
              "name": "graph",
              "title": "VIX - CBOE Volatility Index",
              "specType": "simple",
              "spec": {"type": "line","group": "Date","series": ["VIX Close"]}
            }
        ],
        readme=readme()
    ),
    load(
        load_source='http://www.cboe.com/publish/ScheduledTask/MktData/datahouse/vixcurrent.csv',
        headers=2,
        name='vix-daily'
    ),
    set_type('Date', type='date', format='any'),
    update_resource('vix-daily', **{'title': 'VIX Daily', 'path':'data/vix-daily.csv', 'dpp:streaming': True}),
    validate()
)


def flow(parameters, datapackage, resources, stats):
    return finance_vix


if __name__ == '__main__':
    finance_vix.process()
github frictionlessdata / datapackage-pipelines / datapackage_pipelines / lib / update_resource.py View on Github external
def flow(parameters):
    resources = parameters.get('resources', [])
    metadata = parameters.pop('metadata', {})
    return Flow(
        update_resource(resources, **metadata),
    )