How to use the parse5/lib/tokenizer.CHARACTER_TOKEN function in parse5

To help you get started, we’ve selected a few parse5 examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github inikulin / parse5 / packages / parse5-sax-parser / lib / index.js View on Github external
_runParsingLoop() {
        let token = null;

        do {
            token = this.parserFeedbackSimulator.getNextToken();

            if (token.type === Tokenizer.HIBERNATION_TOKEN) {
                break;
            }

            if (
                token.type === Tokenizer.CHARACTER_TOKEN ||
                token.type === Tokenizer.WHITESPACE_CHARACTER_TOKEN ||
                token.type === Tokenizer.NULL_CHARACTER_TOKEN
            ) {
                if (this.pendingText === null) {
                    token.type = Tokenizer.CHARACTER_TOKEN;
                    this.pendingText = token;
                } else {
                    this.pendingText.chars += token.chars;

                    if (this.options.sourceCodeLocationInfo) {
                        const { endLine, endCol, endOffset } = token.location;
                        Object.assign(this.pendingText.location, {
                            endLine,
                            endCol,
                            endOffset
                        });
github inikulin / parse5 / packages / parse5-sax-parser / lib / index.js View on Github external
reshapeToken: origToken => ({ tagName: origToken.tagName, sourceCodeLocation: origToken.location })
    },
    [Tokenizer.COMMENT_TOKEN]: {
        eventName: 'comment',
        reshapeToken: origToken => ({ text: origToken.data, sourceCodeLocation: origToken.location })
    },
    [Tokenizer.DOCTYPE_TOKEN]: {
        eventName: 'doctype',
        reshapeToken: origToken => ({
            name: origToken.name,
            publicId: origToken.publicId,
            systemId: origToken.systemId,
            sourceCodeLocation: origToken.location
        })
    },
    [Tokenizer.CHARACTER_TOKEN]: {
        eventName: 'text',
        reshapeToken: origToken => ({ text: origToken.chars, sourceCodeLocation: origToken.location })
    }
};

module.exports = SAXParser;
github inikulin / parse5 / packages / parse5-sax-parser / lib / parser-feedback-simulator.js View on Github external
getNextToken() {
        const token = this.tokenizer.getNextToken();

        if (token.type === Tokenizer.START_TAG_TOKEN) {
            this._handleStartTagToken(token);
        } else if (token.type === Tokenizer.END_TAG_TOKEN) {
            this._handleEndTagToken(token);
        } else if (token.type === Tokenizer.NULL_CHARACTER_TOKEN && this.inForeignContent) {
            token.type = Tokenizer.CHARACTER_TOKEN;
            token.chars = unicode.REPLACEMENT_CHARACTER;
        } else if (this.skipNextNewLine) {
            if (token.type !== Tokenizer.HIBERNATION_TOKEN) {
                this.skipNextNewLine = false;
            }

            if (token.type === Tokenizer.WHITESPACE_CHARACTER_TOKEN && token.chars[0] === '\n') {
                if (token.chars.length === 1) {
                    return this.getNextToken();
                }

                token.chars = token.chars.substr(1);
            }
        }

        return token;
github inikulin / parse5 / packages / parse5-sax-parser / lib / index.js View on Github external
let token = null;

        do {
            token = this.parserFeedbackSimulator.getNextToken();

            if (token.type === Tokenizer.HIBERNATION_TOKEN) {
                break;
            }

            if (
                token.type === Tokenizer.CHARACTER_TOKEN ||
                token.type === Tokenizer.WHITESPACE_CHARACTER_TOKEN ||
                token.type === Tokenizer.NULL_CHARACTER_TOKEN
            ) {
                if (this.pendingText === null) {
                    token.type = Tokenizer.CHARACTER_TOKEN;
                    this.pendingText = token;
                } else {
                    this.pendingText.chars += token.chars;

                    if (this.options.sourceCodeLocationInfo) {
                        const { endLine, endCol, endOffset } = token.location;
                        Object.assign(this.pendingText.location, {
                            endLine,
                            endCol,
                            endOffset
                        });
                    }
                }
            } else {
                this._emitPendingText();
                this._handleToken(token);