How to use the @terascope/job-components.getOpConfig function in @terascope/job-components

To help you get started, we’ve selected a few @terascope/job-components examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github terascope / teraslice / packages / teraslice / lib / processors / elasticsearch_bulk.js View on Github external
function crossValidation(job, sysconfig) {
    const opConfig = getOpConfig(job, 'elasticsearch_bulk');
    const elasticConnectors = sysconfig.terafoundation.connectors.elasticsearch;

    // check to verify if connection map provided is
    // consistent with sysconfig.terafoundation.connectors
    if (opConfig.multisend) {
        _.forOwn(opConfig.connection_map, (value) => {
            if (!elasticConnectors[value]) {
                throw new Error(`elasticsearch_bulk connection_map specifies a connection for [${value}] but is not found in the system configuration [terafoundation.connectors.elasticsearch]`);
            }
        });
    }
}
github terascope / teraslice / packages / teraslice / lib / readers / elasticsearch_reader / index.js View on Github external
function crossValidation(job) {
    if (job.lifecycle === 'persistent') {
        const opConfig = getOpConfig(job, 'elasticsearch_reader');
        if (opConfig.interval === 'auto') {
            throw new Error('interval for reader must be manually set while job is in persistent mode');
        }
    }
}
github terascope / teraslice / packages / teraslice / lib / readers / elasticsearch_data_generator.js View on Github external
function crossValidation(job) {
    const opConfig = getOpConfig(job, 'elasticsearch_data_generator');

    if (opConfig.set_id) {
        const indexSelectorConfig = getOpConfig(job, 'elasticsearch_index_selector');

        if (!indexSelectorConfig.id_field) {
            throw new Error('elasticsearch_data_generator is mis-configured, set_id must be used in tandem with id_field which is set in elasticsearch_index_selector');
        }

        if (indexSelectorConfig.id_field !== 'id') {
            throw new Error('id_field set in elasticsearch_index_selector must be set to "id" when elasticsearch_data_generator is creating ids');
        }
    }
}
github terascope / teraslice / packages / teraslice / lib / readers / elasticsearch_reader / index.js View on Github external
function newSlicer(context, executionContext, retryData, logger) {
    const opConfig = getOpConfig(executionContext.config, 'elasticsearch_reader');
    const client = getClient(context, opConfig, 'elasticsearch');
    return require('./elasticsearch_date_range/slicer.js')(context, opConfig, executionContext, retryData, logger, client);
}
github terascope / teraslice / packages / teraslice / lib / readers / elasticsearch_data_generator.js View on Github external
function crossValidation(job) {
    const opConfig = getOpConfig(job, 'elasticsearch_data_generator');

    if (opConfig.set_id) {
        const indexSelectorConfig = getOpConfig(job, 'elasticsearch_index_selector');

        if (!indexSelectorConfig.id_field) {
            throw new Error('elasticsearch_data_generator is mis-configured, set_id must be used in tandem with id_field which is set in elasticsearch_index_selector');
        }

        if (indexSelectorConfig.id_field !== 'id') {
            throw new Error('id_field set in elasticsearch_index_selector must be set to "id" when elasticsearch_data_generator is creating ids');
        }
    }
}
github terascope / teraslice / packages / teraslice / lib / readers / elasticsearch_data_generator.js View on Github external
function newSlicer(context, executionContext) {
    const executionConfig = executionContext.config;
    const opConfig = getOpConfig(executionConfig, 'elasticsearch_data_generator');
    const slicers = [];

    if (executionConfig.lifecycle === 'once') {
        slicers.push(onceGenerator(context, opConfig, executionConfig));
    } else {
        slicers.push(persistentGenerator(context, opConfig, executionConfig));
    }

    return Promise.resolve(slicers);
}