How to use teraslice - 3 common examples

To help you get started, we’ve selected a few teraslice examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github terascope / teraslice / packages / teraslice-worker / lib / execution-context / assets.js View on Github external
async load() {
        const {
            context,
            assets,
            assetsDirectory
        } = this;
        const { logger } = context;

        // no need to load assets
        if (isEmpty(assets)) return;

        this.assetStore = await makeAssetStore(context);

        let idArray;

        try {
            idArray = await this.assetStore.parseAssetsArray(assets);
        } catch (err) {
            throw new WrapError(err);
        }

        await Promise.map(idArray, async (assetIdentifier) => {
            const downloaded = await fs.pathExists(path.join(assetsDirectory, assetIdentifier));
            // need to return the id to the assets array sent back
            if (downloaded) return { id: assetIdentifier };

            const assetRecord = await this.assetStore.get(assetIdentifier);
            logger.info(`loading assets: ${assetIdentifier}`);
github terascope / teraslice / packages / teraslice-worker / lib / execution-context / assets.js View on Github external
await Promise.map(idArray, async (assetIdentifier) => {
            const downloaded = await fs.pathExists(path.join(assetsDirectory, assetIdentifier));
            // need to return the id to the assets array sent back
            if (downloaded) return { id: assetIdentifier };

            const assetRecord = await this.assetStore.get(assetIdentifier);
            logger.info(`loading assets: ${assetIdentifier}`);
            const buff = Buffer.from(assetRecord.blob, 'base64');
            return saveAsset(logger, assetsDirectory, assetIdentifier, buff);
        });
github terascope / teraslice / packages / teraslice-worker / lib / execution-context / execution-runner.js View on Github external
constructor(context, executionContext) {
        this._runner = makeExecutionRunner(context, {
            execution: executionContext.job,
            processAssignment: executionContext.assignment,
        });

        this._assets = new Assets(context, executionContext);
        this._logger = makeLogger(context, executionContext, 'execution_runner');
        this._assignment = executionContext.assignment;

        Object.assign(this, executionContext);

        this.config = executionContext.job;
        this.queue = [];
        this.reader = null;
        this.slicer = null;
        this.reporter = null;
        this.queueLength = 10000;

teraslice

Distributed computing platform for processing JSON data

Apache-2.0
Latest version published 29 days ago

Package Health Score

67 / 100
Full package analysis