How to use the espree.tokenize function in espree
To help you get started, we’ve selected a few espree examples, based on popular ways it is used in public projects.
Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
eslint / eslint / tests / lib / rules / utils / ast-utils.js View on Github 
it(`should return ${expectedResults[key]} for ${key}`, () => {
assert.strictEqual(astUtils.isDecimalIntegerNumericToken(espree.tokenize(key)[0]), expectedResults[key]);
});
});return payloads.map(payload => {
let count = 0;
count += espree.tokenize(payload, { loc: true, range: true }).length;
count += espree.parse(payload, { loc: true, range: true }).body.length;
return count;
});
}function createJsIdentitySourcemap(
sourceUrl: string,
sourceContent: string,
lineOffset: number,
firstLineCharOffset: number) {
const generator = new SourceMapGenerator();
const tokens = espree.tokenize(
sourceContent, {loc: true, ecmaVersion: 2017, sourceType: 'module'});
tokens.forEach((token) => {
if (!token.loc) {
return null;
}
const mapping: Mapping = {
original: {
line: token.loc.start.line + lineOffset,
column: token.loc.start.column +
(token.loc.start.line === 1 ? firstLineCharOffset : 0)
},
generated: token.loc.start,
source: sourceUrl
};
if (token.type === 'Identifier') {tokenize(js, opts) {
return espree.tokenize(js, {...opts, ...ESPREE_OPTS});
},
};canTokensBeAdjacent(leftValue, rightValue) {
let leftToken;
if (typeof leftValue === "string") {
const leftTokens = espree.tokenize(leftValue, { ecmaVersion: 2015 });
leftToken = leftTokens[leftTokens.length - 1];
} else {
leftToken = leftValue;
}
const rightToken = typeof rightValue === "string" ? espree.tokenize(rightValue, { ecmaVersion: 2015 })[0] : rightValue;
if (leftToken.type === "Punctuator" || rightToken.type === "Punctuator") {
if (leftToken.type === "Punctuator" && rightToken.type === "Punctuator") {
const PLUS_TOKENS = new Set(["+", "++"]);
const MINUS_TOKENS = new Set(["-", "--"]);
return !(
PLUS_TOKENS.has(leftToken.value) && PLUS_TOKENS.has(rightToken.value) ||
MINUS_TOKENS.has(leftToken.value) && MINUS_TOKENS.has(rightToken.value)
);
}
return true;
}
if (
leftToken.type === "String" || rightToken.type === "String" ||node.properties.forEach(function(property) {
var key = property.key,
tokens;
if (!key || property.method || property.computed || property.shorthand) {
return;
}
if (key.type === "Literal" && typeof key.value === "string") {
quotes = true;
if (checkQuotesRedundancy) {
try {
tokens = espree.tokenize(key.value);
} catch (e) {
necessaryQuotes = true;
return;
}
necessaryQuotes = necessaryQuotes || !areQuotesRedundant(tokens) || KEYWORDS && isKeyword(tokens[0].value);
}
} else if (KEYWORDS && checkQuotesRedundancy && key.type === "Identifier" && isKeyword(key.name)) {
necessaryQuotes = true;
context.report(node, "Properties should be quoted as `{{property}}` is a reserved word.", {property: key.name});
} else {
lackOfQuotes = true;
}
if (quotes && lackOfQuotes) {
context.report(node, "Inconsistently quoted property `{{key}}` found.", {function checkUnnecessaryQuotes(node) {
var key = node.key,
isKeywordToken,
tokens;
if (node.method || node.computed || node.shorthand) {
return;
}
if (key.type === "Literal" && typeof key.value === "string") {
try {
tokens = espree.tokenize(key.value);
} catch (e) {
return;
}
if (tokens.length !== 1) {
return;
}
isKeywordToken = isKeyword(tokens[0].value);
if (isKeywordToken && KEYWORDS) {
return;
}
if (CHECK_UNNECESSARY && areQuotesRedundant(tokens, NUMBERS)) {
context.report(node, MESSAGE_UNNECESSARY, {property: key.value});canTokensBeAdjacent(leftValue, rightValue) {
let leftToken;
if (typeof leftValue === "string") {
const leftTokens = espree.tokenize(leftValue, { ecmaVersion: 2015 });
leftToken = leftTokens[leftTokens.length - 1];
} else {
leftToken = leftValue;
}
const rightToken = typeof rightValue === "string" ? espree.tokenize(rightValue, { ecmaVersion: 2015 })[0] : rightValue;
if (leftToken.type === "Punctuator" || rightToken.type === "Punctuator") {
if (leftToken.type === "Punctuator" && rightToken.type === "Punctuator") {
const PLUS_TOKENS = new Set(["+", "++"]);
const MINUS_TOKENS = new Set(["-", "--"]);
return !(
PLUS_TOKENS.has(leftToken.value) && PLUS_TOKENS.has(rightToken.value) ||
MINUS_TOKENS.has(leftToken.value) && MINUS_TOKENS.has(rightToken.value)function createJsIdentitySourcemap(
sourceUrl: string,
sourceContent: string,
originalFileContent: string,
lineOffset: number,
firstLineCharOffset: number) {
const generator = new SourceMapGenerator();
const tokens = espree.tokenize(
sourceContent, {loc: true, ecmaVersion: 2017, sourceType: 'module'});
tokens.forEach(token => {
if (!token.loc) {
return null;
}
let mapping: any = {
original: {
line: token.loc.start.line + lineOffset,
column: token.loc.start.column +
(token.loc.start.line === 1 ? firstLineCharOffset : 0)
},
generated: token.loc.start,
source: sourceUrl
};
if (token.type === 'Identifier') {function checkUnnecessaryQuotes(node) {
var key = node.key,
isKeywordToken,
tokens;
if (node.method || node.computed || node.shorthand) {
return;
}
if (key.type === "Literal" && typeof key.value === "string") {
try {
tokens = espree.tokenize(key.value);
} catch (e) {
return;
}
if (tokens.length !== 1) {
return;
}
isKeywordToken = isKeyword(tokens[0].value);
if (isKeywordToken && KEYWORDS) {
return;
}
if (CHECK_UNNECESSARY && areQuotesRedundant(tokens, NUMBERS)) {
context.report(node, MESSAGE_UNNECESSARY, {property: key.value});espree
An Esprima-compatible JavaScript parser built on Acorn
