markdown-tokens/mod.js

export const
    /**
     * @param {string} markdown
     */
    tokenize = markdown => {
        const lines = markdown.split('\n');
        const result = [];
        
        // Track formatting state across the document
        let inCodeBlock = false;
        
        for (const line of lines) {
            if (line.trim() === '') {
                result.push([]);
                continue;
            }
            
            const tokens = [];
            let i = 0;
            
            // Track formatting state within the line
            const formatStack = {
                bold: [],
                italic: [],
                strikethrough: [],
                code: []
            };
            
            while (i < line.length) {
                // Code block start/end
                if (line.substring(i, i + 3) === '```') {
                    if (!inCodeBlock) {
                        // Starting code block
                        const codeBlockMatch = line.substring(i).match(/^```(\w*)/);
                        tokens.push({
                            startIndex: i,
                            endIndex: i + 3,
                            content: '```',
                            formattingStart: 'codeblock'
                        });
                        i += 3;
                        
                        if (codeBlockMatch && codeBlockMatch[1]) {
                            tokens.push({
                                startIndex: i,
                                endIndex: i + codeBlockMatch[1].length,
                                content: codeBlockMatch[1],
                                formattingMetadata: 'codeblockLanguage'
                            });
                            i += codeBlockMatch[1].length;
                        }
                        inCodeBlock = true;
                        continue;
                    } else {
                        // Ending code block
                        tokens.push({
                            startIndex: i,
                            endIndex: i + 3,
                            content: '```',
                            formattingEnd: 'codeblock'
                        });
                        i += 3;
                        inCodeBlock = false;
                        continue;
                    }
                }
                
                // If we're in a code block, treat everything as plain text
                if (inCodeBlock) {
                    tokens.push({
                        startIndex: i,
                        endIndex: line.length,
                        content: line.substring(i)
                    });
                    break;
                }
                
                // Ordered list items
                const orderedListMatch = line.substring(i).match(/^(\d+\. )/);
                if (orderedListMatch && i === 0) {
                    tokens.push({
                        startIndex: i,
                        endIndex: i + orderedListMatch[1].length,
                        content: orderedListMatch[1],
                        formattingStart: 'orderedListItem'
                    });
                    i += orderedListMatch[1].length;
                    continue;
                }
                
                // Unordered list items
                if (line.substring(i, i + 2) === '- ' && i === 0) {
                    tokens.push({
                        startIndex: i,
                        endIndex: i + 2,
                        content: '- ',
                        formattingStart: 'listItem'
                    });
                    i += 2;
                    continue;
                }
                
                // Images
                if (line.substring(i, i + 2) === '![') {
                    const imageMatch = line.substring(i).match(/^!\[([^\]]*)\]\(([^)]*)\)/);
                    if (imageMatch) {
                        tokens.push({
                            startIndex: i,
                            endIndex: i + 2,
                            content: '![',
                            formattingStart: 'imageText'
                        });
                        i += 2;
                        
                        tokens.push({
                            startIndex: i,
                            endIndex: i + imageMatch[1].length,
                            content: imageMatch[1]
                        });
                        i += imageMatch[1].length;
                        
                        tokens.push({
                            startIndex: i,
                            endIndex: i + 1,
                            content: ']',
                            formattingEnd: 'imageText'
                        });
                        i += 1;
                        
                        tokens.push({
                            startIndex: i,
                            endIndex: i + 1,
                            content: '(',
                            formattingStart: 'imageAddress'
                        });
                        i += 1;
                        
                        tokens.push({
                            startIndex: i,
                            endIndex: i + imageMatch[2].length,
                            content: imageMatch[2]
                        });
                        i += imageMatch[2].length;
                        
                        tokens.push({
                            startIndex: i,
                            endIndex: i + 1,
                            content: ')',
                            formattingEnd: 'imageAddress'
                        });
                        i += 1;
                        continue;
                    }
                }
                
                // Links
                if (line[i] === '[') {
                    const linkMatch = line.substring(i).match(/^\[([^\]]*)\]\(([^)]*)\)/);
                    if (linkMatch) {
                        tokens.push({
                            startIndex: i,
                            endIndex: i + 1,
                            content: '[',
                            formattingStart: 'hyperlinkText'
                        });
                        i += 1;
                        
                        tokens.push({
                            startIndex: i,
                            endIndex: i + linkMatch[1].length,
                            content: linkMatch[1]
                        });
                        i += linkMatch[1].length;
                        
                        tokens.push({
                            startIndex: i,
                            endIndex: i + 1,
                            content: ']',
                            formattingEnd: 'hyperlinkText'
                        });
                        i += 1;
                        
                        tokens.push({
                            startIndex: i,
                            endIndex: i + 1,
                            content: '(',
                            formattingStart: 'hyperlinkAddress'
                        });
                        i += 1;
                        
                        tokens.push({
                            startIndex: i,
                            endIndex: i + linkMatch[2].length,
                            content: linkMatch[2]
                        });
                        i += linkMatch[2].length;
                        
                        tokens.push({
                            startIndex: i,
                            endIndex: i + 1,
                            content: ')',
                            formattingEnd: 'hyperlinkAddress'
                        });
                        i += 1;
                        continue;
                    }
                }
                
                // Bold text
                if (line.substring(i, i + 2) === '**') {
                    if (formatStack.bold.length === 0) {
                        tokens.push({
                            startIndex: i,
                            endIndex: i + 2,
                            content: '**',
                            formattingStart: 'bold'
                        });
                        formatStack.bold.push(i);
                    } else {
                        tokens.push({
                            startIndex: i,
                            endIndex: i + 2,
                            content: '**',
                            formattingEnd: 'bold'
                        });
                        formatStack.bold.pop();
                    }
                    i += 2;
                    continue;
                }
                
                // Strikethrough
                if (line.substring(i, i + 2) === '~~') {
                    if (formatStack.strikethrough.length === 0) {
                        tokens.push({
                            startIndex: i,
                            endIndex: i + 2,
                            content: '~~',
                            formattingStart: 'strikethrough'
                        });
                        formatStack.strikethrough.push(i);
                    } else {
                        tokens.push({
                            startIndex: i,
                            endIndex: i + 2,
                            content: '~~',
                            formattingEnd: 'strikethrough'
                        });
                        formatStack.strikethrough.pop();
                    }
                    i += 2;
                    continue;
                }
                
                // Inline code
                if (line[i] === '`') {
                    if (formatStack.code.length === 0) {
                        tokens.push({
                            startIndex: i,
                            endIndex: i + 1,
                            content: '`',
                            formattingStart: 'code'
                        });
                        formatStack.code.push(i);
                    } else {
                        tokens.push({
                            startIndex: i,
                            endIndex: i + 1,
                            content: '`',
                            formattingEnd: 'code'
                        });
                        formatStack.code.pop();
                    }
                    i += 1;
                    continue;
                }
                
                // Italic text (single asterisk or underscore)
                if (line[i] === '*' || line[i] === '_') {
                    if (formatStack.italic.length === 0) {
                        tokens.push({
                            startIndex: i,
                            endIndex: i + 1,
                            content: line[i],
                            formattingStart: 'italic'
                        });
                        formatStack.italic.push(i);
                    } else {
                        tokens.push({
                            startIndex: i,
                            endIndex: i + 1,
                            content: line[i],
                            formattingEnd: 'italic'
                        });
                        formatStack.italic.pop();
                    }
                    i += 1;
                    continue;
                }
                
                // Regular text - find the next formatting character
                let nextFormatIndex = i + 1;
                while (nextFormatIndex < line.length) {
                    const char = line[nextFormatIndex];
                    const twoChar = line.substring(nextFormatIndex, nextFormatIndex + 2);
                    const threeChar = line.substring(nextFormatIndex, nextFormatIndex + 3);
                    
                    if (char === '*' || char === '_' || char === '`' || char === '[' || 
                        twoChar === '**' || twoChar === '~~' || twoChar === '![' ||
                        threeChar === '```') {
                        break;
                    }
                    nextFormatIndex++;
                }
                
                if (nextFormatIndex > i) {
                    tokens.push({
                        startIndex: i,
                        endIndex: nextFormatIndex,
                        content: line.substring(i, nextFormatIndex)
                    });
                    i = nextFormatIndex;
                } else {
                    i++;
                }
            }
            
            result.push(tokens);
        }
        
        return result;
    };