markdown-tokens/mod.js

export const
    /**
     * Tokenize markdown into an array of lines, each line an array of token objects.
     * Tokens have: startIndex, endIndex (exclusive), content, and optionally
     * formattingStart | formattingEnd | formattingMetadata.
     * Only implements the subset required by tests (bold, italic, strikethrough, inline code,
     * hyperlinks, images, unordered + ordered list items, fenced code blocks with language).
     * @param {string} markdown
     * @returns {Array<Array<object>>}
     */
    tokenize = markdown => {
        const lines = markdown.split('\n');
        const result = [];
        let inCodeblock = false;
        for (const line of lines) {
            // Codeblock handling first
            if (inCodeblock) {
                if (line === '```') {
                    result.push([
                        {
                            startIndex: 0,
                            endIndex: 3,
                            content: '```',
                            formattingEnd: 'codeblock'
                        }
                    ]);
                    inCodeblock = false;
                } else {
                    result.push(line.length === 0 ? [] : [
                        {
                            startIndex: 0,
                            endIndex: line.length,
                            content: line
                        }
                    ]);
                }
                continue;
            }
            // Opening codeblock
            if (line.startsWith('```')) {
                const tokens = [
                    {
                        startIndex: 0,
                        endIndex: 3,
                        content: '```',
                        formattingStart: 'codeblock'
                    }
                ];
                if (line.length > 3) {
                    tokens.push({
                        startIndex: 3,
                        endIndex: line.length,
                        content: line.slice(3),
                        formattingMetadata: 'codeblockLanguage'
                    });
                }
                result.push(tokens);
                inCodeblock = true;
                continue;
            }
            // Blank line
            if (line.trim() === '') {
                result.push([]);
                continue;
            }
            const tokens = [];
            let i = 0;
            // List detection (unordered / ordered) at line start
            if (line.startsWith('- ')) {
                tokens.push({
                    startIndex: 0,
                    endIndex: 2,
                    content: '- ',
                    formattingStart: 'listItem'
                });
                i = 2;
            } else {
                const orderedMatch = line.match(/^(\d+\. )/);
                if (orderedMatch) {
                    tokens.push({
                        startIndex: 0,
                        endIndex: orderedMatch[1].length,
                        content: orderedMatch[1],
                        formattingStart: 'orderedListItem'
                    });
                    i = orderedMatch[1].length;
                }
            }

            // Stack for formatting markers
            const stack = [];
            // Track last closed bracket type for hyperlink/image address start detection
            let lastClosedBracketType = null; // 'hyperlink' | 'image'

            let buffer = '';
            let bufferStart = i;

            const flushBuffer = (currentIndex) => {
                if (buffer.length) {
                    tokens.push({
                        startIndex: bufferStart,
                        endIndex: currentIndex,
                        content: buffer
                    });
                    buffer = '';
                }
            };

            const openOrClose = (type, marker, length) => {
                // Determine if closing (top of stack is same type) else opening
                const isClosing = stack.length && stack[stack.length - 1].type === type;
                flushBuffer(i);
                if (isClosing) {
                    tokens.push({
                        startIndex: i,
                        endIndex: i + length,
                        content: marker,
                        formattingEnd: type
                    });
                    stack.pop();
                } else {
                    tokens.push({
                        startIndex: i,
                        endIndex: i + length,
                        content: marker,
                        formattingStart: type
                    });
                    stack.push({ type, marker });
                }
                i += length;
                bufferStart = i;
            };

            while (i < line.length) {
                // If inside inline code, only look for closing backtick
                if (stack.length && stack[stack.length - 1].type === 'code') {
                    if (line[i] === '`') {
                        openOrClose('code', '`', 1);
                        continue;
                    }
                    // Treat rest as plain text char
                    buffer += line[i];
                    i++;
                    continue;
                }

                const twoChars = line.slice(i, i + 2);
                // Image start '!['
                if (twoChars === '![') {
                    flushBuffer(i);
                    tokens.push({
                        startIndex: i,
                        endIndex: i + 2,
                        content: '![' ,
                        formattingStart: 'imageText'
                    });
                    stack.push({ type: 'imageText', marker: '![' });
                    i += 2;
                    bufferStart = i;
                    continue;
                }
                // Bold '**'
                if (twoChars === '**') {
                    openOrClose('bold', '**', 2);
                    continue;
                }
                // Strikethrough '~~'
                if (twoChars === '~~') {
                    openOrClose('strikethrough', '~~', 2);
                    continue;
                }
                // Hyperlink or image text start '['
                if (line[i] === '[') {
                    flushBuffer(i);
                    tokens.push({
                        startIndex: i,
                        endIndex: i + 1,
                        content: '[',
                        formattingStart: 'hyperlinkText'
                    });
                    stack.push({ type: 'hyperlinkText', marker: '[' });
                    i += 1;
                    bufferStart = i;
                    continue;
                }
                // Closing bracket ']' for hyperlink / image text
                if (line[i] === ']') {
                    // Find last imageText or hyperlinkText on stack (should be top ideally)
                    const top = stack[stack.length - 1];
                    if (top && (top.type === 'hyperlinkText' || top.type === 'imageText')) {
                        flushBuffer(i);
                        tokens.push({
                            startIndex: i,
                            endIndex: i + 1,
                            content: ']',
                            formattingEnd: top.type
                        });
                        stack.pop();
                        lastClosedBracketType = top.type === 'imageText' ? 'image' : 'hyperlink';
                        i += 1;
                        bufferStart = i;
                        continue;
                    }
                }
                // Opening parenthesis for address right after closing bracket
                if (line[i] === '(' && (lastClosedBracketType === 'hyperlink' || lastClosedBracketType === 'image')) {
                    flushBuffer(i);
                    const type = lastClosedBracketType === 'image' ? 'imageAddress' : 'hyperlinkAddress';
                    tokens.push({
                        startIndex: i,
                        endIndex: i + 1,
                        content: '(',
                        formattingStart: type
                    });
                    stack.push({ type, marker: '(' });
                    i += 1;
                    bufferStart = i;
                    lastClosedBracketType = null;
                    continue;
                }
                // Closing parenthesis for address
                if (line[i] === ')') {
                    const top = stack[stack.length - 1];
                    if (top && (top.type === 'hyperlinkAddress' || top.type === 'imageAddress')) {
                        flushBuffer(i);
                        tokens.push({
                            startIndex: i,
                            endIndex: i + 1,
                            content: ')',
                            formattingEnd: top.type
                        });
                        stack.pop();
                        i += 1;
                        bufferStart = i;
                        continue;
                    }
                }
                // Inline code '`'
                if (line[i] === '`') {
                    openOrClose('code', '`', 1);
                    continue;
                }
                // Italic '*' or '_' (treat both as same type 'italic')
                if (line[i] === '*' || line[i] === '_') {
                    openOrClose('italic', line[i], 1);
                    continue;
                }
                // Default accumulate char
                buffer += line[i];
                i++;
            }
            // Flush remaining plain text
            flushBuffer(i);
            result.push(tokens);
        }
        return result;
    };