394 lines
No EOL
15 KiB
JavaScript
394 lines
No EOL
15 KiB
JavaScript
export const
|
|
/**
|
|
* @param {string} markdown
|
|
*/
|
|
tokenize = markdown => {
|
|
const lines = markdown.split('\n');
|
|
const result = [];
|
|
let inCodeBlock = false;
|
|
|
|
for (let lineIndex = 0; lineIndex < lines.length; lineIndex++) {
|
|
const line = lines[lineIndex];
|
|
const tokens = [];
|
|
|
|
if (line.trim() === '') {
|
|
result.push([]);
|
|
continue;
|
|
}
|
|
|
|
// Handle code block boundaries
|
|
if (line.startsWith('```')) {
|
|
if (!inCodeBlock) {
|
|
// Starting code block
|
|
tokens.push({
|
|
startIndex: 0,
|
|
endIndex: 3,
|
|
content: '```',
|
|
formattingStart: 'codeblock'
|
|
});
|
|
|
|
// Check for language specification
|
|
if (line.length > 3) {
|
|
const language = line.substring(3);
|
|
tokens.push({
|
|
startIndex: 3,
|
|
endIndex: line.length,
|
|
content: language,
|
|
formattingMetadata: 'codeblockLanguage'
|
|
});
|
|
}
|
|
inCodeBlock = true;
|
|
} else {
|
|
// Ending code block
|
|
tokens.push({
|
|
startIndex: 0,
|
|
endIndex: 3,
|
|
content: '```',
|
|
formattingEnd: 'codeblock'
|
|
});
|
|
inCodeBlock = false;
|
|
}
|
|
result.push(tokens);
|
|
continue;
|
|
}
|
|
|
|
// If we're inside a code block, treat everything as plain text
|
|
if (inCodeBlock) {
|
|
tokens.push({
|
|
startIndex: 0,
|
|
endIndex: line.length,
|
|
content: line
|
|
});
|
|
result.push(tokens);
|
|
continue;
|
|
}
|
|
|
|
let i = 0;
|
|
const formattingStack = [];
|
|
|
|
while (i < line.length) {
|
|
const char = line[i];
|
|
const nextChar = line[i + 1];
|
|
|
|
// Handle ordered list items
|
|
if (i === 0 && /^\d+\. /.test(line)) {
|
|
const match = line.match(/^(\d+\. )/);
|
|
if (match) {
|
|
tokens.push({
|
|
startIndex: 0,
|
|
endIndex: match[1].length,
|
|
content: match[1],
|
|
formattingStart: 'orderedListItem'
|
|
});
|
|
i = match[1].length;
|
|
continue;
|
|
}
|
|
}
|
|
|
|
// Handle unordered list items
|
|
if (i === 0 && line.startsWith('- ')) {
|
|
tokens.push({
|
|
startIndex: 0,
|
|
endIndex: 2,
|
|
content: '- ',
|
|
formattingStart: 'listItem'
|
|
});
|
|
i = 2;
|
|
continue;
|
|
}
|
|
|
|
// Handle bold text (**)
|
|
if (char === '*' && nextChar === '*') {
|
|
const boldIndex = formattingStack.findIndex(f => f.type === 'bold');
|
|
if (boldIndex === -1) {
|
|
// Start bold
|
|
tokens.push({
|
|
startIndex: i,
|
|
endIndex: i + 2,
|
|
content: '**',
|
|
formattingStart: 'bold'
|
|
});
|
|
formattingStack.push({ type: 'bold', index: tokens.length - 1 });
|
|
} else {
|
|
// End bold
|
|
tokens.push({
|
|
startIndex: i,
|
|
endIndex: i + 2,
|
|
content: '**',
|
|
formattingEnd: 'bold'
|
|
});
|
|
formattingStack.splice(boldIndex, 1);
|
|
}
|
|
i += 2;
|
|
continue;
|
|
}
|
|
|
|
// Handle italic text (*) - only if not part of **
|
|
if (char === '*' && nextChar !== '*' && (i === 0 || line[i - 1] !== '*')) {
|
|
const italicIndex = formattingStack.findIndex(f => f.type === 'italic' && f.marker === '*');
|
|
if (italicIndex === -1) {
|
|
// Start italic
|
|
tokens.push({
|
|
startIndex: i,
|
|
endIndex: i + 1,
|
|
content: '*',
|
|
formattingStart: 'italic'
|
|
});
|
|
formattingStack.push({ type: 'italic', marker: '*', index: tokens.length - 1 });
|
|
} else {
|
|
// End italic
|
|
tokens.push({
|
|
startIndex: i,
|
|
endIndex: i + 1,
|
|
content: '*',
|
|
formattingEnd: 'italic'
|
|
});
|
|
formattingStack.splice(italicIndex, 1);
|
|
}
|
|
i++;
|
|
continue;
|
|
}
|
|
|
|
// Handle italic text (_)
|
|
if (char === '_') {
|
|
const italicIndex = formattingStack.findIndex(f => f.type === 'italic' && f.marker === '_');
|
|
if (italicIndex === -1) {
|
|
// Start italic
|
|
tokens.push({
|
|
startIndex: i,
|
|
endIndex: i + 1,
|
|
content: '_',
|
|
formattingStart: 'italic'
|
|
});
|
|
formattingStack.push({ type: 'italic', marker: '_', index: tokens.length - 1 });
|
|
} else {
|
|
// End italic
|
|
tokens.push({
|
|
startIndex: i,
|
|
endIndex: i + 1,
|
|
content: '_',
|
|
formattingEnd: 'italic'
|
|
});
|
|
formattingStack.splice(italicIndex, 1);
|
|
}
|
|
i++;
|
|
continue;
|
|
}
|
|
|
|
// Handle strikethrough text (~~)
|
|
if (char === '~' && nextChar === '~') {
|
|
const strikeIndex = formattingStack.findIndex(f => f.type === 'strikethrough');
|
|
if (strikeIndex === -1) {
|
|
// Start strikethrough
|
|
tokens.push({
|
|
startIndex: i,
|
|
endIndex: i + 2,
|
|
content: '~~',
|
|
formattingStart: 'strikethrough'
|
|
});
|
|
formattingStack.push({ type: 'strikethrough', index: tokens.length - 1 });
|
|
} else {
|
|
// End strikethrough
|
|
tokens.push({
|
|
startIndex: i,
|
|
endIndex: i + 2,
|
|
content: '~~',
|
|
formattingEnd: 'strikethrough'
|
|
});
|
|
formattingStack.splice(strikeIndex, 1);
|
|
}
|
|
i += 2;
|
|
continue;
|
|
}
|
|
|
|
// Handle inline code (`)
|
|
if (char === '`') {
|
|
const codeIndex = formattingStack.findIndex(f => f.type === 'code');
|
|
if (codeIndex === -1) {
|
|
// Start code
|
|
tokens.push({
|
|
startIndex: i,
|
|
endIndex: i + 1,
|
|
content: '`',
|
|
formattingStart: 'code'
|
|
});
|
|
formattingStack.push({ type: 'code', index: tokens.length - 1 });
|
|
} else {
|
|
// End code
|
|
tokens.push({
|
|
startIndex: i,
|
|
endIndex: i + 1,
|
|
content: '`',
|
|
formattingEnd: 'code'
|
|
});
|
|
formattingStack.splice(codeIndex, 1);
|
|
}
|
|
i++;
|
|
continue;
|
|
}
|
|
|
|
// Handle images (![...])
|
|
if (char === '!' && nextChar === '[') {
|
|
tokens.push({
|
|
startIndex: i,
|
|
endIndex: i + 2,
|
|
content: '![',
|
|
formattingStart: 'imageText'
|
|
});
|
|
formattingStack.push({ type: 'imageText', index: tokens.length - 1 });
|
|
i += 2;
|
|
continue;
|
|
}
|
|
|
|
// Handle hyperlinks ([...])
|
|
if (char === '[') {
|
|
tokens.push({
|
|
startIndex: i,
|
|
endIndex: i + 1,
|
|
content: '[',
|
|
formattingStart: 'hyperlinkText'
|
|
});
|
|
formattingStack.push({ type: 'hyperlinkText', index: tokens.length - 1 });
|
|
i++;
|
|
continue;
|
|
}
|
|
|
|
// Handle closing brackets
|
|
if (char === ']') {
|
|
const imageIndex = formattingStack.findIndex(f => f.type === 'imageText');
|
|
const hyperlinkIndex = formattingStack.findIndex(f => f.type === 'hyperlinkText');
|
|
|
|
if (imageIndex !== -1) {
|
|
tokens.push({
|
|
startIndex: i,
|
|
endIndex: i + 1,
|
|
content: ']',
|
|
formattingEnd: 'imageText'
|
|
});
|
|
formattingStack.splice(imageIndex, 1);
|
|
} else if (hyperlinkIndex !== -1) {
|
|
tokens.push({
|
|
startIndex: i,
|
|
endIndex: i + 1,
|
|
content: ']',
|
|
formattingEnd: 'hyperlinkText'
|
|
});
|
|
formattingStack.splice(hyperlinkIndex, 1);
|
|
}
|
|
i++;
|
|
continue;
|
|
}
|
|
|
|
// Handle opening parentheses for addresses
|
|
if (char === '(') {
|
|
// Check if we just closed an image or hyperlink text
|
|
const lastToken = tokens[tokens.length - 1];
|
|
if (lastToken && lastToken.formattingEnd === 'imageText') {
|
|
tokens.push({
|
|
startIndex: i,
|
|
endIndex: i + 1,
|
|
content: '(',
|
|
formattingStart: 'imageAddress'
|
|
});
|
|
formattingStack.push({ type: 'imageAddress', index: tokens.length - 1 });
|
|
} else if (lastToken && lastToken.formattingEnd === 'hyperlinkText') {
|
|
tokens.push({
|
|
startIndex: i,
|
|
endIndex: i + 1,
|
|
content: '(',
|
|
formattingStart: 'hyperlinkAddress'
|
|
});
|
|
formattingStack.push({ type: 'hyperlinkAddress', index: tokens.length - 1 });
|
|
} else {
|
|
// Regular text
|
|
const textStart = i;
|
|
i++;
|
|
while (i < line.length && !isFormattingChar(line[i], line[i + 1])) {
|
|
i++;
|
|
}
|
|
tokens.push({
|
|
startIndex: textStart,
|
|
endIndex: i,
|
|
content: line.substring(textStart, i)
|
|
});
|
|
continue;
|
|
}
|
|
i++;
|
|
continue;
|
|
}
|
|
|
|
// Handle closing parentheses
|
|
if (char === ')') {
|
|
const imageAddressIndex = formattingStack.findIndex(f => f.type === 'imageAddress');
|
|
const hyperlinkAddressIndex = formattingStack.findIndex(f => f.type === 'hyperlinkAddress');
|
|
|
|
if (imageAddressIndex !== -1) {
|
|
tokens.push({
|
|
startIndex: i,
|
|
endIndex: i + 1,
|
|
content: ')',
|
|
formattingEnd: 'imageAddress'
|
|
});
|
|
formattingStack.splice(imageAddressIndex, 1);
|
|
} else if (hyperlinkAddressIndex !== -1) {
|
|
tokens.push({
|
|
startIndex: i,
|
|
endIndex: i + 1,
|
|
content: ')',
|
|
formattingEnd: 'hyperlinkAddress'
|
|
});
|
|
formattingStack.splice(hyperlinkAddressIndex, 1);
|
|
} else {
|
|
// Regular text
|
|
const textStart = i;
|
|
i++;
|
|
while (i < line.length && !isFormattingChar(line[i], line[i + 1])) {
|
|
i++;
|
|
}
|
|
tokens.push({
|
|
startIndex: textStart,
|
|
endIndex: i,
|
|
content: line.substring(textStart, i)
|
|
});
|
|
continue;
|
|
}
|
|
i++;
|
|
continue;
|
|
}
|
|
|
|
// Handle regular text
|
|
const textStart = i;
|
|
while (i < line.length && !isFormattingChar(line[i], line[i + 1])) {
|
|
i++;
|
|
}
|
|
|
|
if (i > textStart) {
|
|
tokens.push({
|
|
startIndex: textStart,
|
|
endIndex: i,
|
|
content: line.substring(textStart, i)
|
|
});
|
|
}
|
|
}
|
|
|
|
result.push(tokens);
|
|
}
|
|
|
|
return result;
|
|
};
|
|
|
|
function isFormattingChar(char, nextChar) {
|
|
if (!char) return false;
|
|
|
|
return (
|
|
char === '*' ||
|
|
char === '_' ||
|
|
char === '~' ||
|
|
char === '`' ||
|
|
char === '[' ||
|
|
char === ']' ||
|
|
char === '(' ||
|
|
char === ')' ||
|
|
(char === '!' && nextChar === '[')
|
|
);
|
|
} |