markdown-tokens/mod.js

export const
    /**
     * @param {string} markdown
     */
    tokenize = markdown => {
        const lines = markdown.split('\n');
        const result = [];
        let inCodeblock = false;

        for (const line of lines) {
            if (line.trim() === '') {
                result.push([]);
                continue;
            }

            const tokens = [];
            let currentIndex = 0;

            // Helper function to add a token
            const addToken = (content, formattingStart = undefined, formattingEnd = undefined, formattingMetadata = undefined) => {
                const token = {
                    startIndex: currentIndex,
                    endIndex: currentIndex + content.length,
                    content
                };

                if (formattingStart) token.formattingStart = formattingStart;
                if (formattingEnd) token.formattingEnd = formattingEnd;
                if (formattingMetadata) token.formattingMetadata = formattingMetadata;

                tokens.push(token);
                currentIndex += content.length;
            };

            // Check if this line starts or ends a codeblock
            if (line.startsWith('```')) {
                if (!inCodeblock) {
                    inCodeblock = true;
                    addToken('```', 'codeblock');
                    // Check for language after ```
                    const languageMatch = line.slice(3).match(/^([a-zA-Z0-9]+)/);
                    if (languageMatch) {
                        addToken(languageMatch[1], undefined, undefined, 'codeblockLanguage');
                    }
                } else {
                    inCodeblock = false;
                    addToken('```', undefined, 'codeblock');
                }
                result.push(tokens);
                continue;
            }

            // If we're inside a codeblock, treat everything as plain text
            if (inCodeblock) {
                addToken(line);
                result.push(tokens);
                continue;
            }

            let i = 0;
            while (i < line.length) {
                const remaining = line.slice(i);

                // Check for ordered list
                const orderedListMatch = remaining.match(/^(\d+\. )/);
                if (orderedListMatch && i === 0) {
                    addToken(orderedListMatch[1], 'orderedListItem');
                    i += orderedListMatch[1].length;
                    continue;
                }

                // Check for unordered list
                if (remaining.startsWith('- ') && i === 0) {
                    addToken('- ', 'listItem');
                    i += 2;
                    continue;
                }

                // Check for image
                if (remaining.startsWith('![')) {
                    addToken('![', 'imageText');
                    i += 2;
                    continue;
                }

                // Check for hyperlink start (but not if we already started image)
                if (remaining.startsWith('[') && !tokens.some(t => t.formattingStart === 'imageText' && !tokens.some(t2 => t2.formattingEnd === 'imageText' && t2.startIndex > t.startIndex))) {
                    addToken('[', 'hyperlinkText');
                    i += 1;
                    continue;
                }

                // Check for text end and address start
                if (remaining.startsWith('](')) {
                    // Check if we're in an image or hyperlink
                    const openImageText = tokens.some(t => t.formattingStart === 'imageText' && !tokens.some(t2 => t2.formattingEnd === 'imageText' && t2.startIndex > t.startIndex));
                    const openHyperlinkText = tokens.some(t => t.formattingStart === 'hyperlinkText' && !tokens.some(t2 => t2.formattingEnd === 'hyperlinkText' && t2.startIndex > t.startIndex));

                    if (openImageText) {
                        addToken(']', undefined, 'imageText');
                        i += 1;
                        addToken('(', 'imageAddress');
                        i += 1;
                    } else if (openHyperlinkText) {
                        addToken(']', undefined, 'hyperlinkText');
                        i += 1;
                        addToken('(', 'hyperlinkAddress');
                        i += 1;
                    } else {
                        addToken('](');
                        i += 2;
                    }
                    continue;
                }

                // Check for address end
                if (remaining.startsWith(')')) {
                    const openImageAddress = tokens.some(t => t.formattingStart === 'imageAddress' && !tokens.some(t2 => t2.formattingEnd === 'imageAddress' && t2.startIndex > t.startIndex));
                    const openHyperlinkAddress = tokens.some(t => t.formattingStart === 'hyperlinkAddress' && !tokens.some(t2 => t2.formattingEnd === 'hyperlinkAddress' && t2.startIndex > t.startIndex));

                    if (openImageAddress) {
                        addToken(')', undefined, 'imageAddress');
                        i += 1;
                    } else if (openHyperlinkAddress) {
                        addToken(')', undefined, 'hyperlinkAddress');
                        i += 1;
                    } else {
                        addToken(')');
                        i += 1;
                    }
                    continue;
                }

                // Check for bold
                if (remaining.startsWith('**')) {
                    const openBold = tokens.some(t => t.formattingStart === 'bold' && !tokens.some(t2 => t2.formattingEnd === 'bold' && t2.startIndex > t.startIndex));
                    if (openBold) {
                        addToken('**', undefined, 'bold');
                    } else {
                        addToken('**', 'bold');
                    }
                    i += 2;
                    continue;
                }

                // Check for italic (both * and _)
                if (remaining.startsWith('*') || remaining.startsWith('_')) {
                    const char = remaining[0];
                    const openItalic = tokens.some(t => t.formattingStart === 'italic' && t.content === char && !tokens.some(t2 => t2.formattingEnd === 'italic' && t2.content === char && t2.startIndex > t.startIndex));
                    if (openItalic) {
                        addToken(char, undefined, 'italic');
                    } else {
                        addToken(char, 'italic');
                    }
                    i += 1;
                    continue;
                }

                // Check for strikethrough
                if (remaining.startsWith('~~')) {
                    const openStrike = tokens.some(t => t.formattingStart === 'strikethrough' && !tokens.some(t2 => t2.formattingEnd === 'strikethrough' && t2.startIndex > t.startIndex));
                    if (openStrike) {
                        addToken('~~', undefined, 'strikethrough');
                    } else {
                        addToken('~~', 'strikethrough');
                    }
                    i += 2;
                    continue;
                }

                // Check for inline code
                if (remaining.startsWith('`')) {
                    const openCode = tokens.some(t => t.formattingStart === 'code' && !tokens.some(t2 => t2.formattingEnd === 'code' && t2.startIndex > t.startIndex));
                    if (openCode) {
                        addToken('`', undefined, 'code');
                    } else {
                        addToken('`', 'code');
                    }
                    i += 1;
                    continue;
                }

                // Regular text - find the next special character or end of line
                let textEnd = i + 1;
                while (textEnd < line.length) {
                    const nextChar = line.slice(textEnd);
                    if (nextChar.startsWith('**') || nextChar.startsWith('~~') ||
                        nextChar.startsWith('![') || nextChar.startsWith('[') ||
                        nextChar.startsWith('](') || nextChar.startsWith(')') ||
                        nextChar.startsWith('*') || nextChar.startsWith('_') ||
                        nextChar.startsWith('`')) {
                        break;
                    }
                    textEnd++;
                }

                const textContent = line.slice(i, textEnd);
                addToken(textContent);
                i = textEnd;
            }

            result.push(tokens);
        }

        return result;
    };