Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
chars.forEach(c => {
let codePoint = c.codePointAt(0) || c.charAt(0);
if (codePoint > 0xFFFF) {
// Character is in a Supplementary Unicode Plane. This is where emoji live so
// we're going to just break each character in this range out as its own token.
tokens.push(c);
if (!StringUtility.isNullOrWhitespace(token)) {
tokens.push(token);
token = '';
}
}
else if (!(this.config.tokenRegex.test(c) || StringUtility.isWhitespace(c))) {
token = token.concat(c);
}
else if (!StringUtility.isNullOrWhitespace(token)) {
tokens.push(token);
token = '';
}
});
let suffix = afterStr.substr(match.index + match.length).trim();
let endingMatch = RegExpUtility.getMatches(this.config.generalEndingRegex, suffix).pop();
if (endingMatch) {
tokens.push(new Token(er.start || 0, er.start + er.length + match.index + match.length || 0));
}
}
}
}
let beforeStr = source.substr(0, er.start);
match = RegExpUtility.getMatches(this.config.periodTimeOfDayWithDateRegex, beforeStr).pop();
if (match) {
if (StringUtility.isNullOrWhitespace(beforeStr.substr(match.index + match.length))) {
let middleStr = source.substr(match.index + match.length, er.start - match.index - match.length);
if (StringUtility.isWhitespace(middleStr)) {
tokens.push(new Token(match.index, er.start + er.length));
}
}
else {
let pauseMatch = RegExpUtility.getMatches(this.config.middlePauseRegex, beforeStr.substr(match.index + match.length)).pop();
if (pauseMatch) {
// TODO: should use trimStart() instead?
let suffix = source.substr(er.start + er.length || 0).trim();
let endingMatch = RegExpUtility.getMatches(this.config.generalEndingRegex, suffix).pop();
if (endingMatch) {
tokens.push(new Token(match.index, er.start + er.length || 0));
}
}