How to use the wink-nlp-utils.tokens function in wink-nlp-utils

/* eslint no-console: 0 */
// Load wink-bm25-text-search
var bm25 = require( 'wink-bm25-text-search' );
// Create search engine's instance
var engine = bm25();
// Load NLP utilities
var nlp = require( 'wink-nlp-utils' );
// Load sample data (load any other JSON data instead of sample)
var docs = require( 'wink-bm25-text-search/sample-data/demo-data-for-wink-bm25.json' );

// Define preparatory task pipe!
var pipe = [
  nlp.string.lowerCase,
  nlp.string.tokenize0,
  nlp.tokens.removeWords,
  nlp.tokens.stem,
  nlp.tokens.propagateNegations
];
// Contains search query.
var query;

// Step I: Define config
// Only field weights are required in this example.
engine.defineConfig( { fldWeights: { title: 1, body: 2 } } );
// Step II: Define PrepTasks pipe.
// Set up 'default' preparatory tasks i.e. for everything else
engine.definePrepTasks( pipe );

// Step III: Add Docs
// Add documents now...
docs.forEach( function ( doc, i ) {

// Load wink-bm25-text-search
var bm25 = require( 'wink-bm25-text-search' );
// Create search engine's instance
var engine = bm25();
// Load NLP utilities
var nlp = require( 'wink-nlp-utils' );
// Load sample data (load any other JSON data instead of sample)
var docs = require( 'wink-bm25-text-search/sample-data/demo-data-for-wink-bm25.json' );

// Define preparatory task pipe!
var pipe = [
  nlp.string.lowerCase,
  nlp.string.tokenize0,
  nlp.tokens.removeWords,
  nlp.tokens.stem,
  nlp.tokens.propagateNegations
];
// Contains search query.
var query;

// Step I: Define config
// Only field weights are required in this example.
engine.defineConfig( { fldWeights: { title: 1, body: 2 } } );
// Step II: Define PrepTasks pipe.
// Set up 'default' preparatory tasks i.e. for everything else
engine.definePrepTasks( pipe );

// Step III: Add Docs
// Add documents now...
docs.forEach( function ( doc, i ) {
  // Note, 'i' becomes the unique id for 'doc'
  engine.addDoc( doc, i );

describe( 'complete clean workflow test', function () {
  var bts = bm25();
  var prepTasks = [
    {
      whenInputIs: [ [
      prepare.string.lowerCase,
      prepare.string.removeExtraSpaces,
      prepare.string.tokenize0,
      prepare.tokens.stem,
      prepare.tokens.propagateNegations ] ],
      expectedOutputIs: 5
    },
    {
      whenInputIs: [ [
      prepare.string.lowerCase,
      prepare.string.removeExtraSpaces,
      prepare.string.tokenize0,
      prepare.tokens.propagateNegations,
      prepare.tokens.removeWords,
      prepare.tokens.stem ], 'body' ],
      expectedOutputIs: 6
    }
  ];

  it( 'defineConfig should return true when proper config is passed', function () {

{
      whenInputIs: [ [
      prepare.string.lowerCase,
      prepare.string.removeExtraSpaces,
      prepare.string.tokenize0,
      prepare.tokens.stem,
      prepare.tokens.propagateNegations ] ],
      expectedOutputIs: 5
    },
    {
      whenInputIs: [ [
      prepare.string.lowerCase,
      prepare.string.removeExtraSpaces,
      prepare.string.tokenize0,
      prepare.tokens.propagateNegations,
      prepare.tokens.removeWords,
      prepare.tokens.stem ], 'body' ],
      expectedOutputIs: 6
    }
  ];

  it( 'defineConfig should return true when proper config is passed', function () {
    var config = {
      fldWeights: {
         title: 4,
         body: 1,
         tags: 2
       },
       bm25Params: {
          k1: 1.2,
          k: 1,
          b: 0.75