How to use js-tokens - 10 common examples
To help you get started, we’ve selected a few js-tokens examples, based on popular ways it is used in public projects.
Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
erzu / porter / packages / porter / lib / matchRequire.js View on Github 
exports.findAll = function(content) {
const parts = content.match(jsTokens)
const deps = []
let i = 0
let part
function next() {
part = parts[i++]
}
function space() {
do {
next()
} while (rSpace.test(part))
}
function findRequire() {
// to rule out module.require()function replace(src, envs) {
if (!processEnvRe.test(src)) {
return src;
}
var out = [];
var purge = envs.some(function(env) {
return env._ && env._.indexOf('purge') !== -1;
});
jsTokens.lastIndex = 0
var parts = src.match(jsTokens);
for (var i = 0; i < parts.length; i++) {
if (parts[i ] === 'process' &&
parts[i + 1] === '.' &&
parts[i + 2] === 'env' &&
parts[i + 3] === '.') {
var prevCodeToken = getAdjacentCodeToken(-1, parts, i);
var nextCodeToken = getAdjacentCodeToken(1, parts, i + 4);
var replacement = getReplacementString(envs, parts[i + 4], purge);
if (prevCodeToken !== '.' &&
nextCodeToken !== '.' &&
nextCodeToken !== '=' &&
typeof replacement === 'string') {
out.push(replacement);
i += 4;
continue;function replace(src, envs) {
if (!processEnvRe.test(src)) {
return src;
}
var out = [];
var purge = envs.some(function(env) {
return env._ && env._.indexOf('purge') !== -1;
});
jsTokens.lastIndex = 0
var parts = src.match(jsTokens);
for (var i = 0; i < parts.length; i++) {
if (parts[i ] === 'process' &&
parts[i + 1] === '.' &&
parts[i + 2] === 'env' &&
parts[i + 3] === '.') {
var prevCodeToken = getAdjacentCodeToken(-1, parts, i);
var nextCodeToken = getAdjacentCodeToken(1, parts, i + 4);
var replacement = getReplacementString(envs, parts[i + 4], purge);
if (prevCodeToken !== '.' &&
nextCodeToken !== '.' &&
nextCodeToken !== '=' &&
typeof replacement === 'string') {
out.push(replacement);
i += 4;
continue;function replace(src, envs) {
if (!processEnvRe.test(src)) {
return src;
}
var out = [];
var purge = envs.some(function(env) {
return env._ && env._.indexOf('purge') !== -1;
});
jsTokens.lastIndex = 0
var parts = src.match(jsTokens);
for (var i = 0; i < parts.length; i++) {
if (parts[i ] === 'process' &&
parts[i + 1] === '.' &&
parts[i + 2] === 'env' &&
parts[i + 3] === '.') {
var prevCodeToken = getAdjacentCodeToken(-1, parts, i);
var nextCodeToken = getAdjacentCodeToken(1, parts, i + 4);
var replacement = getReplacementString(envs, parts[i + 4], purge);
if (prevCodeToken !== '.' &&
nextCodeToken !== '.' &&
nextCodeToken !== '=' &&
typeof replacement === 'string') {
out.push(replacement);
i += 4;function replace(src, envs) {
if (!processEnvRe.test(src)) {
return src;
}
var out = [];
var purge = envs.some(function(env) {
return env._ && env._.indexOf('purge') !== -1;
});
jsTokens.lastIndex = 0
var parts = src.match(jsTokens);
for (var i = 0; i < parts.length; i++) {
if (parts[i ] === 'process' &&
parts[i + 1] === '.' &&
parts[i + 2] === 'env' &&
parts[i + 3] === '.') {
var prevCodeToken = getAdjacentCodeToken(-1, parts, i);
var nextCodeToken = getAdjacentCodeToken(1, parts, i + 4);
var replacement = getReplacementString(envs, parts[i + 4], purge);
if (prevCodeToken !== '.' &&
nextCodeToken !== '.' &&
nextCodeToken !== '=' &&
typeof replacement === 'string') {
out.push(replacement);
i += 4;function replace(src, envs) {
if (!processEnvRe.test(src)) {
return src;
}
var out = '';
var purge = envs.some(function(env) {
return env._ && env._.indexOf('purge') !== -1;
});
jsTokens.lastIndex = 0;
var parts = src.match(jsTokens);
for (var i = 0; i < parts.length; i++) {
if (parts[i] === 'process' && parts[i + 1] === '.' && parts[i + 2] === 'env' && parts[i + 3] === '.') {
var prevCodeToken = getAdjacentCodeToken(-1, parts, i);
var nextCodeToken = getAdjacentCodeToken(1, parts, i + 4);
var replacement = getReplacementString(envs, parts[i + 4], purge);
if (prevCodeToken !== '.' && nextCodeToken !== '.' && nextCodeToken !== '=' && typeof replacement === 'string') {
out += replacement;
i += 4;
continue;
}
}
out += parts[i];
}
return out;
}function replace(src, envs) {
if (!processEnvRe.test(src)) {
return src;
}
var out = '';
var purge = envs.some(function(env) {
return env._ && env._.indexOf('purge') !== -1;
});
jsTokens.lastIndex = 0
var parts = src.match(jsTokens);
for (var i = 0; i < parts.length; i++) {
if (parts[i ] === 'process' &&
parts[i + 1] === '.' &&
parts[i + 2] === 'env' &&
parts[i + 3] === '.') {
var prevCodeToken = getAdjacentCodeToken(-1, parts, i);
var nextCodeToken = getAdjacentCodeToken(1, parts, i + 4);
var replacement = getReplacementString(envs, parts[i + 4], purge);
if (prevCodeToken !== '.' &&
nextCodeToken !== '.' &&
nextCodeToken !== '=' &&
typeof replacement === 'string') {
out += replacement;
i += 4;const esTokenizer = code => {
const tokens = code
.match(jsTokens)
.map(value => {
jsTokens.lastIndex = 0;
return matchToToken(jsTokens.exec(value));
})
.map(token => {
if (token.type === 'name' && isKeyword(token.value)) {
token.type = 'keyword';
}
return token;
});
return tokens;
};const tokenize = data => {
return data.match(jst).filter(chr => !BAD_TOKENS.includes(chr))
}var esTokenizer = function esTokenizer(code) {
var tokens = code.match(jsTokens).map(function (value) {
jsTokens.lastIndex = 0;
return matchToToken(jsTokens.exec(value));
}).map(function (token) {
if (token.type === 'name' && isKeyword(token.value)) {
token.type = 'keyword';
}
return token;
});
return tokens;
};
