How to use the @nlpjs/core-loader.Tokenizer function in @nlpjs/core-loader
To help you get started, we’ve selected a few @nlpjs/core-loader examples, based on popular ways it is used in public projects.
Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
axa-group / nlp.js / packages / node-nlp / src / nlp / nlp-util.js View on Github 
static getTokenizer(locale) {
if (!locale) {
return new Tokenizer();
}
const name = `Tokenizer${locale.slice(0, 1).toUpperCase()}${locale.slice(
1
)}`;
const TokenizerClass = LangAll[name];
return TokenizerClass
? new TokenizerClass(undefined, true)
: new Tokenizer(undefined, true);
}static getTokenizer(locale) {
if (!locale) {
return new Tokenizer();
}
const name = `Tokenizer${locale.slice(0, 1).toUpperCase()}${locale.slice(
1
)}`;
const TokenizerClass = LangAll[name];
return TokenizerClass
? new TokenizerClass(undefined, true)
: new Tokenizer(undefined, true);
}
