markdown-tokens/mod.js

export const
    /**
     * @param {string} markdown
     */
    tokenize = markdown => {
        const lines = markdown.split('\n');
        let inCodeBlock = false;
        let inQuote = false;

        return lines.map((line) => {
            const tokens = [];
            if (line.trim() === '') {
                inQuote = false;
                return tokens;
            }

            if (inCodeBlock) {
                if (line.startsWith('```')) {
                    inCodeBlock = false;
                    tokens.push({
                        startIndex: 0,
                        endIndex: 3,
                        content: '```',
                        formattingEnd: 'codeblock'
                    });
                } else {
                    tokens.push({
                        startIndex: 0,
                        endIndex: line.length,
                        content: line
                    });
                }
                return tokens;
            }

            if (line.startsWith('```')) {
                inCodeBlock = true;
                tokens.push({
                    startIndex: 0,
                    endIndex: 3,
                    content: '```',
                    formattingStart: 'codeblock'
                });
                const lang = line.slice(3);
                if (lang) {
                    tokens.push({
                        startIndex: 3,
                        endIndex: line.length,
                        content: lang,
                        formattingMetadata: 'codeblockLanguage'
                    });
                }
                return tokens;
            }

            let currentIndex = 0;

            const headingMatch = line.match(/^(#{1,6})\s/);
            const quoteMatch = line.match(/^>\s?/);
            const listItemMatch = line.match(/^- /);
            const orderedListItemMatch = line.match(/^(\d+)\. /);

            let isContinuationQuote = inQuote && !quoteMatch;

            if (headingMatch) {
                const hLevel = headingMatch[1].length;
                tokens.push({
                    startIndex: 0,
                    endIndex: hLevel,
                    content: headingMatch[1],
                    formattingStart: 'heading',
                    formattingMetadata: `h${hLevel}`
                });
                currentIndex = hLevel;
                inQuote = false; // Headings end quotes usually
            } else if (quoteMatch) {
                inQuote = true;
                tokens.push({
                    startIndex: 0,
                    endIndex: 1,
                    content: '>',
                    formattingStart: 'quote'
                });
                currentIndex = 1;
            } else if (listItemMatch) {
                tokens.push({
                    startIndex: 0,
                    endIndex: 2,
                    content: '- ',
                    formattingStart: 'listItem'
                });
                currentIndex = 2;
                inQuote = false;
            } else if (orderedListItemMatch) {
                tokens.push({
                    startIndex: 0,
                    endIndex: orderedListItemMatch[0].length,
                    content: orderedListItemMatch[0],
                    formattingStart: 'orderedListItem'
                });
                currentIndex = orderedListItemMatch[0].length;
                inQuote = false;
            }

            const remainingText = line.slice(currentIndex);
            const inlineMarkers = /(\*\*|__|\*|_|~~|`|!\[|\[|\]|\(|\))/g;
            let lastIndex = 0;
            let match;
            const activeFormats = [];

            while ((match = inlineMarkers.exec(remainingText)) !== null) {
                const marker = match[0];
                const markerIndex = match.index;

                if (markerIndex > lastIndex) {
                    tokens.push({
                        startIndex: currentIndex + lastIndex,
                        endIndex: currentIndex + markerIndex,
                        content: remainingText.slice(lastIndex, markerIndex)
                    });
                }

                let fStart = null;
                let fEnd = null;

                if (marker === '**' || marker === '__') {
                    const existingIndex = activeFormats.findLastIndex(f => f.type === 'bold' && f.marker === marker);
                    if (existingIndex !== -1) {
                        fEnd = 'bold';
                        activeFormats.splice(existingIndex, 1);
                    } else {
                        fStart = 'bold';
                        activeFormats.push({
                            type: 'bold',
                            marker
                        });
                    }
                } else if (marker === '*' || marker === '_') {
                    const existingIndex = activeFormats.findLastIndex(f => f.type === 'italic' && f.marker === marker);
                    if (existingIndex !== -1) {
                        fEnd = 'italic';
                        activeFormats.splice(existingIndex, 1);
                    } else {
                        fStart = 'italic';
                        activeFormats.push({
                            type: 'italic',
                            marker
                        });
                    }
                } else if (marker === '~~') {
                    const existingIndex = activeFormats.findLastIndex(f => f.type === 'strikethrough');
                    if (existingIndex !== -1) {
                        fEnd = 'strikethrough';
                        activeFormats.splice(existingIndex, 1);
                    } else {
                        fStart = 'strikethrough';
                        activeFormats.push({
                            type: 'strikethrough'
                        });
                    }
                } else if (marker === '`') {
                    const existingIndex = activeFormats.findLastIndex(f => f.type === 'code');
                    if (existingIndex !== -1) {
                        fEnd = 'code';
                        activeFormats.splice(existingIndex, 1);
                    } else {
                        fStart = 'code';
                        activeFormats.push({
                            type: 'code'
                        });
                    }
                } else if (marker === '![') {
                    fStart = 'imageText';
                    activeFormats.push({
                        type: 'imageText'
                    });
                } else if (marker === '[') {
                    fStart = 'hyperlinkText';
                    activeFormats.push({
                        type: 'hyperlinkText'
                    });
                } else if (marker === ']') {
                    const imgIndex = activeFormats.findLastIndex(f => f.type === 'imageText');
                    const linkIndex = activeFormats.findLastIndex(f => f.type === 'hyperlinkText');
                    if (imgIndex !== -1) {
                        fEnd = 'imageText';
                        activeFormats.splice(imgIndex, 1);
                    } else if (linkIndex !== -1) {
                        fEnd = 'hyperlinkText';
                        activeFormats.splice(linkIndex, 1);
                    }
                } else if (marker === '(') {
                    const lastToken = tokens[tokens.length - 1];
                    if (lastToken && lastToken.formattingEnd === 'imageText') {
                        fStart = 'imageAddress';
                        activeFormats.push({
                            type: 'imageAddress'
                        });
                    } else if (lastToken && lastToken.formattingEnd === 'hyperlinkText') {
                        fStart = 'hyperlinkAddress';
                        activeFormats.push({
                            type: 'hyperlinkAddress'
                        });
                    }
                } else if (marker === ')') {
                    const imgIndex = activeFormats.findLastIndex(f => f.type === 'imageAddress');
                    const linkIndex = activeFormats.findLastIndex(f => f.type === 'hyperlinkAddress');
                    if (imgIndex !== -1) {
                        fEnd = 'imageAddress';
                        activeFormats.splice(imgIndex, 1);
                    } else if (linkIndex !== -1) {
                        fEnd = 'hyperlinkAddress';
                        activeFormats.splice(linkIndex, 1);
                    }
                }

                const token = {
                    startIndex: currentIndex + markerIndex,
                    endIndex: currentIndex + markerIndex + marker.length,
                    content: marker
                };
                if (fStart) token.formattingStart = fStart;
                if (fEnd) token.formattingEnd = fEnd;
                tokens.push(token);

                lastIndex = markerIndex + marker.length;
            }

            if (lastIndex < remainingText.length) {
                tokens.push({
                    startIndex: currentIndex + lastIndex,
                    endIndex: currentIndex + remainingText.length,
                    content: remainingText.slice(lastIndex)
                });
            }

            if (isContinuationQuote && tokens.length > 0) {
                tokens[0].formattingStart = 'quote';
            }

            return tokens;
        });
    };