How to use the @nlpjs/lang-es/src.TokenizerEs function in @nlpjs/lang-es

To help you get started, we’ve selected a few @nlpjs/lang-es examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github hubtype / botonic / packages / botonic-plugin-contentful / src / nlp / tokens.ts View on Github external
return this.trim(normalized.split(/[^a-zA-Zá-úÁ-ÚñÑüÜ]+/))
  }

  private trim(arr: string[]): string[] {
    while (arr[arr.length - 1] === '') {
      arr.pop()
    }
    while (arr[0] === '') {
      arr.shift()
    }
    return arr
  }
}

const tokenizers: { [locale: string]: Tokenizer } = {
  es: new TokenizerEs(),
  en: new TokenizerEn(),
  ca: new TokenizerCa(),
  pl: new Tokenizer(),
  pt: new TokenizerPt(),
}

export function tokenizerPerLocale(locale: Locale): Tokenizer {
  return tokenizers[locale]
}
export const DEFAULT_SEPARATORS = ';,./()!?" '
export const DEFAULT_SEPARATORS_REGEX = new RegExp(
  '[' + DEFAULT_SEPARATORS + ']',
  'g'
)
export const DEFAULT_NOT_SEPARATORS_REGEX = new RegExp(
  '[^' + DEFAULT_SEPARATORS + ']',

@nlpjs/lang-es

Core

MIT
Latest version published 2 years ago

Package Health Score

62 / 100
Full package analysis

Similar packages