How to use skale - 10 common examples

const t = require('tape');
const sc = require('skale').context();

t.test('collect', function (t) {
  t.plan(1);

  sc.range(6)
    .collect(function (err, res) {
      t.deepEqual(res, [0, 1, 2, 3, 4, 5]);
      sc.end();
    });
});

const t = require('tape');
const sc = require('skale').context();

t.test('union', function (t) {
  t.plan(1);

  sc.parallelize([1, 2, 3, 4])
    .union(sc.parallelize([5, 6, 7, 8]))
    .collect(function (err, res) {
      t.deepEqual(res, [1, 2, 3, 4, 5, 6, 7, 8]);
      sc.end();
    });
});

const fs = require('fs');
const t = require('tape');
const rimraf = require('rimraf');
const sc = require('skale').context();

const savedir = '/tmp/skale-test/save';

t.test('save csv', function (t) {
  t.plan(4);

  rimraf(savedir, function (err) {
    t.ok(!err, 'delete previous saved data');
    sc.range(10)
      .map(a => [a, a, a])
      .save(savedir, {stream: true, csv: true}, function (err) {
        t.ok(!err, 'save returns no error');
        t.ok(fs.existsSync(savedir + '/0.csv'), 'saved filename is correct');
        sc.textFile(savedir + '/')
          .collect(function (err, res) {
            t.deepEqual(res, [

const t = require('tape');
const azure = require('azure-storage');
const sc = require('skale').context();

//const skip = process.env.CI || (process.env.AZURE_STORAGE_CONNECTION_STRING ? false : true);
const skip = true;
const retry = new azure.ExponentialRetryPolicyFilter();
const az = skip ? null : azure.createBlobService().withFilter(retry);
const savedir = 'wasb://skalejs/save';

t.test('save azure', {skip: skip}, function (t) {
  t.plan(3);

  deleteAzureDir('save', '', function (err) {
    t.ok(!err, 'delete previous saved test data');
    sc.range(10)
      .save(savedir, function (err) {
        t.ok(!err, 'save returns no error');
        sc.textFile(savedir + '/')

const fs = require('fs');
const t = require('tape');
const rimraf = require('rimraf');
const sc = require('skale').context();

const savedir = '/tmp/skale-test/save';

t.test('save', function (t) {
  t.plan(4);

  rimraf(savedir, function (err) {
    t.ok(!err, 'delete previous saved data');
    sc.range(10)
      .save(savedir, function (err) {
        t.ok(!err, 'save returns no error');
        t.ok(fs.existsSync(savedir + '/0'), 'saved filename is correct');
        sc.textFile(savedir + '/')
          .map(a => JSON.parse(a))
          .collect(function (err, res) {
            t.deepEqual(res, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9], 'saved content is correct');

const t = require('tape');
const aws = require('aws-sdk');
const sc = require('skale').context();

const skip = process.env.CI || (process.env.AWS_ACCESS_KEY_ID ? false : true);
const s3 = skip ? null : new aws.S3({httpOptions: {timeout: 3600000}, signatureVersion: 'v4'});
const savedir = 's3://skale-test-eu-west-1/test/save';

t.test('save s3', {skip: skip}, function (t) {
  t.plan(3);

  deleteS3Dir('skale-test-eu-west-1', 'test/save/', function (err) {
    t.ok(!err, 'delete S3 previous saved test data');
    sc.range(10)
      .save(savedir, function (err) {
        t.ok(!err, 'save returns no error');
        sc.textFile(savedir + '/')
          .map(a => JSON.parse(a))
          .collect(function (err, res) {

#!/usr/bin/env node

const sc = require('skale').context();

const file = process.argv[2] || '/etc/hosts';

sc.textFile(file)
  .flatMap(line => line.split(' '))
  .map(word => [word, 1])
  .reduceByKey((a, b) => a + b, 0)
  .count()
  .then(function (res) {
    console.log(res);
    sc.end();
  });

#!/usr/bin/env node

const fs = require('fs');
const sc = require('skale').context();

const s1 = sc.lineStream(fs.createReadStream(__dirname + '/kv.data')).map(line => line.split(' '));
const s2 = sc.lineStream(fs.createReadStream(__dirname + '/kv2.data')).map(line =>line.split(' '));

s1.coGroup(s2).collect(function(err, res) {
  console.log(res[0]);
  console.log(res[1]);  
  console.log(res[2]);
  console.log(res[3]);    
  sc.end();
});

#!/usr/bin/env node

const sc = require('skale').context();

sc.range(6).map((a) => a*a).reduce((a,b) => a+b, 0)
  .then(function (res) {
    console.log(res);
    console.assert(res === 55);
    sc.end();
  });

#!/usr/bin/env node

const sc = require('skale').context();

//sc.range(900).save('/tmp/truc', {gzip: true}, (err, res) => {
//sc.range(900).save('/tmp/truc', {stream: true}, (err, res) => {
//sc.range(900).save('s3://skale-demo/test/s1', {gzip: false, stream: true}, (err, res) => {
sc.range(900).save('/tmp/truc', {gzip: true, stream: true}, (err, res) => {
  console.log(res);
  sc.end();
});