How to use the react-diff-view/tokenize.tokenize function in react-diff-view

const enhancers = [
            editsType === 'none' ? null : markEdits(hunks, {type: editsType}),
            markWord('\r', 'carriage-return', '␍'),
            markWord('\t', 'tab', '→'),
        ];

        const options = {
            highlight: language !== 'text',
            refractor: refractor,
            language: language,
            oldSource: oldSource,
            enhancers: compact(enhancers),
        };

        try {
            const tokens = tokenize(hunks, options);
            const payload = {
                success: true,
                tokens: tokens,
            };
            self.postMessage({id, payload});
        }
        catch (ex) {
            const payload = {
                success: false,
                reason: ex.message,
            };
            self.postMessage({id, payload});
        }
    }
);