markdown-tokens/mod.test.js
2025-12-24 01:06:12 +01:00

1105 lines
No EOL
36 KiB
JavaScript

import {
describe,
test,
expect
} from 'bun:test';
import outdent from 'outdent';
import Joi from 'joi';
import {
tokenize
} from './mod.js';
describe(
'tokenize comprehensive markdown sample',
() => {
const markdown = outdent `
Plain text with **bold text** and *italic text* and **_bold italic text_** and ~~strikethrough text~~.
Some more text with __bold text__ and _italic text_ and __*bold italic text*__.
Another paragraph with \`inline code\` and a [hyperlink](https://example.com).
![image](image.jpg)
Unordered list:
- First item
- Second item with **bold**
- Third item
Ordered list:
1. First numbered item
2. Second numbered item with *italic*
3. Third numbered item
\`\`\`js
function example() {
return "code block";
}
\`\`\`
# Level 1 heading
## Level 2 heading
### Level 3 heading
#### Level 4 heading
##### Level 5 heading
###### Level 6 heading
> Quote
Second line of quote paragraph
Normal paragraph
> Quote with **bold**, *italic*, ~~strikethrough~~, __more bold__, _more italic_, \`inline code\` and a [hyperlink](https://example.com).
# Heading with **bold**, *italic*, ~~strikethrough~~, __more bold__, _more italic_, \`inline code\` and a [hyperlink](https://example.com).
`;
const result = tokenize(markdown);
test(
'outputs array of 44 lines',
() => {
expect(Array.isArray(result)).toBe(true);
expect(result.length).toBe(44);
}
);
test(
'validates schema',
() => {
const
formattingTypes = [
'bold',
'italic',
'strikethrough',
'code',
'hyperlinkText',
'hyperlinkAddress',
'imageText',
'imageAddress',
'listItem',
'orderedListItem',
'codeblock',
'heading',
'quote'
],
schema = Joi.array().items(
Joi.array().items(
Joi
.object({
startIndex: Joi.number().integer().required(),
endIndex: Joi.number().integer().required(),
content: Joi.string().required(),
formattingStart: Joi.valid(...formattingTypes),
formattingEnd: Joi.valid(...formattingTypes),
formattingMetadata: Joi.valid('codeblockLanguage', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6')
})
.oxor('formattingStart', 'formattingEnd')
)
);
expect(schema.validate(result).error).toBeUndefined();
}
);
test(
'parses empty lines',
() => {
[1, 3, 5, 7, 12, 17, 23, 25, 27, 29, 31, 33, 35, 38, 40, 42].forEach(index => {
expect(Array.isArray(result[index])).toBe(true);
expect(result[index].length).toBe(0);
});
}
);
test(
'parses paragraph 1',
() => {
expect(result[0]).toEqual([
{
startIndex: 0,
endIndex: 16,
content: 'Plain text with '
},
{
startIndex: 16,
endIndex: 18,
content: '**',
formattingStart: 'bold'
},
{
startIndex: 18,
endIndex: 27,
content: 'bold text'
},
{
startIndex: 27,
endIndex: 29,
content: '**',
formattingEnd: 'bold'
},
{
startIndex: 29,
endIndex: 34,
content: ' and '
},
{
startIndex: 34,
endIndex: 35,
content: '*',
formattingStart: 'italic'
},
{
startIndex: 35,
endIndex: 46,
content: 'italic text'
},
{
startIndex: 46,
endIndex: 47,
content: '*',
formattingEnd: 'italic'
},
{
startIndex: 47,
endIndex: 52,
content: ' and '
},
{
startIndex: 52,
endIndex: 54,
content: '**',
formattingStart: 'bold'
},
{
startIndex: 54,
endIndex: 55,
content: '_',
formattingStart: 'italic'
},
{
startIndex: 55,
endIndex: 71,
content: 'bold italic text'
},
{
startIndex: 71,
endIndex: 72,
content: '_',
formattingEnd: 'italic'
},
{
startIndex: 72,
endIndex: 74,
content: '**',
formattingEnd: 'bold'
},
{
startIndex: 74,
endIndex: 79,
content: ' and '
},
{
startIndex: 79,
endIndex: 81,
content: '~~',
formattingStart: 'strikethrough'
},
{
startIndex: 81,
endIndex: 99,
content: 'strikethrough text'
},
{
startIndex: 99,
endIndex: 101,
content: '~~',
formattingEnd: 'strikethrough'
},
{
startIndex: 101,
endIndex: 102,
content: '.'
}
]);
}
);
test(
'parses paragraph 2',
() => {
expect(result[2]).toEqual([
{
startIndex: 0,
endIndex: 20,
content: 'Some more text with '
},
{
startIndex: 20,
endIndex: 22,
content: '__',
formattingStart: 'bold'
},
{
startIndex: 22,
endIndex: 31,
content: 'bold text'
},
{
startIndex: 31,
endIndex: 33,
content: '__',
formattingEnd: 'bold'
},
{
startIndex: 33,
endIndex: 38,
content: ' and '
},
{
startIndex: 38,
endIndex: 39,
content: '_',
formattingStart: 'italic'
},
{
startIndex: 39,
endIndex: 50,
content: 'italic text'
},
{
startIndex: 50,
endIndex: 51,
content: '_',
formattingEnd: 'italic'
},
{
startIndex: 51,
endIndex: 56,
content: ' and '
},
{
startIndex: 56,
endIndex: 58,
content: '__',
formattingStart: 'bold'
},
{
startIndex: 58,
endIndex: 59,
content: '*',
formattingStart: 'italic'
},
{
startIndex: 59,
endIndex: 75,
content: 'bold italic text'
},
{
startIndex: 75,
endIndex: 76,
content: '*',
formattingEnd: 'italic'
},
{
startIndex: 76,
endIndex: 78,
content: '__',
formattingEnd: 'bold'
},
{
startIndex: 78,
endIndex: 79,
content: '.'
}
]);
}
);
test(
'parses paragraph 3',
() => {
expect(result[4]).toEqual([
{
startIndex: 0,
endIndex: 23,
content: 'Another paragraph with '
},
{
startIndex: 23,
endIndex: 24,
content: '`',
formattingStart: 'code',
},
{
startIndex: 24,
endIndex: 35,
content: 'inline code'
},
{
startIndex: 35,
endIndex: 36,
content: '`',
formattingEnd: 'code'
},
{
startIndex: 36,
endIndex: 43,
content: ' and a '
},
{
startIndex: 43,
endIndex: 44,
content: '[',
formattingStart: 'hyperlinkText'
},
{
startIndex: 44,
endIndex: 53,
content: 'hyperlink'
},
{
startIndex: 53,
endIndex: 54,
content: ']',
formattingEnd: 'hyperlinkText'
},
{
startIndex: 54,
endIndex: 55,
content: '(',
formattingStart: 'hyperlinkAddress'
},
{
startIndex: 55,
endIndex: 74,
content: 'https://example.com'
},
{
startIndex: 74,
endIndex: 75,
content: ')',
formattingEnd: 'hyperlinkAddress'
},
{
startIndex: 75,
endIndex: 76,
content: '.'
}
]);
}
);
test(
'parses paragraph 4',
() => {
expect(result[6]).toEqual([
{
startIndex: 0,
endIndex: 2,
content: '![',
formattingStart: 'imageText'
},
{
startIndex: 2,
endIndex: 7,
content: 'image'
},
{
startIndex: 7,
endIndex: 8,
content: ']',
formattingEnd: 'imageText'
},
{
startIndex: 8,
endIndex: 9,
content: '(',
formattingStart: 'imageAddress'
},
{
startIndex: 9,
endIndex: 18,
content: 'image.jpg'
},
{
startIndex: 18,
endIndex: 19,
content: ')',
formattingEnd: 'imageAddress'
}
]);
}
);
test(
'parses paragraph 5',
() => {
expect(result[8]).toEqual([
{
startIndex: 0,
endIndex: 15,
content: 'Unordered list:'
}
]);
expect(result[9]).toEqual([
{
startIndex: 0,
endIndex: 2,
content: '- ',
formattingStart: 'listItem'
},
{
startIndex: 2,
endIndex: 12,
content: 'First item'
}
]);
expect(result[10]).toEqual([
{
startIndex: 0,
endIndex: 2,
content: '- ',
formattingStart: 'listItem'
},
{
startIndex: 2,
endIndex: 19,
content: 'Second item with '
},
{
startIndex: 19,
endIndex: 21,
content: '**',
formattingStart: 'bold'
},
{
startIndex: 21,
endIndex: 25,
content: 'bold'
},
{
startIndex: 25,
endIndex: 27,
content: '**',
formattingEnd: 'bold'
}
]);
expect(result[11]).toEqual([
{
startIndex: 0,
endIndex: 2,
content: '- ',
formattingStart: 'listItem'
},
{
startIndex: 2,
endIndex: 12,
content: 'Third item'
}
]);
}
);
test(
'parses paragraph 6',
() => {
expect(result[13]).toEqual([
{
startIndex: 0,
endIndex: 13,
content: 'Ordered list:'
}
]);
expect(result[14]).toEqual([
{
startIndex: 0,
endIndex: 3,
content: '1. ',
formattingStart: 'orderedListItem'
},
{
startIndex: 3,
endIndex: 22,
content: 'First numbered item'
}
]);
expect(result[15]).toEqual([
{
startIndex: 0,
endIndex: 3,
content: '2. ',
formattingStart: 'orderedListItem'
},
{
startIndex: 3,
endIndex: 29,
content: 'Second numbered item with '
},
{
startIndex: 29,
endIndex: 30,
content: '*',
formattingStart: 'italic'
},
{
startIndex: 30,
endIndex: 36,
content: 'italic'
},
{
startIndex: 36,
endIndex: 37,
content: '*',
formattingEnd: 'italic'
}
]);
expect(result[16]).toEqual([
{
startIndex: 0,
endIndex: 3,
content: '3. ',
formattingStart: 'orderedListItem'
},
{
startIndex: 3,
endIndex: 22,
content: 'Third numbered item'
}
]);
}
);
test(
'parses paragraph 7',
() => {
expect(result[18]).toEqual([
{
startIndex: 0,
endIndex: 3,
content: '```',
formattingStart: 'codeblock'
},
{
startIndex: 3,
endIndex: 5,
content: 'js',
formattingMetadata: 'codeblockLanguage'
}
]);
expect(result[19]).toEqual([{
startIndex: 0,
endIndex: 20,
content: 'function example() {'
}]);
expect(result[20]).toEqual([{
startIndex: 0,
endIndex: 24,
content: ' return "code block";'
}]);
expect(result[21]).toEqual([{
startIndex: 0,
endIndex: 1,
content: '}'
}]);
expect(result[22]).toEqual([{
startIndex: 0,
endIndex: 3,
content: '```',
formattingEnd: 'codeblock'
}]);
}
);
test(
'parses headings',
() => {
expect(result[24]).toEqual([
{
startIndex: 0,
endIndex: 1,
content: '#',
formattingStart: 'heading',
formattingMetadata: 'h1'
},
{
startIndex: 1,
endIndex: 17,
content: ' Level 1 heading'
}
]);
expect(result[26]).toEqual([
{
startIndex: 0,
endIndex: 2,
content: '##',
formattingStart: 'heading',
formattingMetadata: 'h2'
},
{
startIndex: 2,
endIndex: 18,
content: ' Level 2 heading'
}
]);
expect(result[28]).toEqual([
{
startIndex: 0,
endIndex: 3,
content: '###',
formattingStart: 'heading',
formattingMetadata: 'h3'
},
{
startIndex: 3,
endIndex: 19,
content: ' Level 3 heading'
}
]);
expect(result[30]).toEqual([
{
startIndex: 0,
endIndex: 4,
content: '####',
formattingStart: 'heading',
formattingMetadata: 'h4'
},
{
startIndex: 4,
endIndex: 20,
content: ' Level 4 heading'
}
]);
expect(result[32]).toEqual([
{
startIndex: 0,
endIndex: 5,
content: '#####',
formattingStart: 'heading',
formattingMetadata: 'h5'
},
{
startIndex: 5,
endIndex: 21,
content: ' Level 5 heading'
}
]);
expect(result[34]).toEqual([
{
startIndex: 0,
endIndex: 6,
content: '######',
formattingStart: 'heading',
formattingMetadata: 'h6'
},
{
startIndex: 6,
endIndex: 22,
content: ' Level 6 heading'
}
]);
}
);
test(
'parses quotes',
() => {
expect(result[36]).toEqual([
{
startIndex: 0,
endIndex: 1,
content: '>',
formattingStart: 'quote'
},
{
startIndex: 1,
endIndex: 7,
content: ' Quote'
}
]);
expect(result[37]).toEqual([
{
startIndex: 0,
endIndex: 30,
content: 'Second line of quote paragraph',
formattingStart: 'quote'
}
]);
expect(result[39]).toEqual([
{
startIndex: 0,
endIndex: 16,
content: 'Normal paragraph'
}
]);
}
);
test(
'parses complex quote and heading',
() => {
expect(result[41]).toEqual([
{
startIndex: 0,
endIndex: 1,
content: '>',
formattingStart: 'quote'
},
{
startIndex: 1,
endIndex: 13,
content: ' Quote with '
},
{
startIndex: 13,
endIndex: 15,
content: '**',
formattingStart: 'bold'
},
{
startIndex: 15,
endIndex: 19,
content: 'bold'
},
{
startIndex: 19,
endIndex: 21,
content: '**',
formattingEnd: 'bold'
},
{
startIndex: 21,
endIndex: 23,
content: ', '
},
{
startIndex: 23,
endIndex: 24,
content: '*',
formattingStart: 'italic'
},
{
startIndex: 24,
endIndex: 30,
content: 'italic'
},
{
startIndex: 30,
endIndex: 31,
content: '*',
formattingEnd: 'italic'
},
{
startIndex: 31,
endIndex: 33,
content: ', '
},
{
startIndex: 33,
endIndex: 35,
content: '~~',
formattingStart: 'strikethrough'
},
{
startIndex: 35,
endIndex: 48,
content: 'strikethrough'
},
{
startIndex: 48,
endIndex: 50,
content: '~~',
formattingEnd: 'strikethrough'
},
{
startIndex: 50,
endIndex: 52,
content: ', '
},
{
startIndex: 52,
endIndex: 54,
content: '__',
formattingStart: 'bold'
},
{
startIndex: 54,
endIndex: 63,
content: 'more bold'
},
{
startIndex: 63,
endIndex: 65,
content: '__',
formattingEnd: 'bold'
},
{
startIndex: 65,
endIndex: 67,
content: ', '
},
{
startIndex: 67,
endIndex: 68,
content: '_',
formattingStart: 'italic'
},
{
startIndex: 68,
endIndex: 79,
content: 'more italic'
},
{
startIndex: 79,
endIndex: 80,
content: '_',
formattingEnd: 'italic'
},
{
startIndex: 80,
endIndex: 82,
content: ', '
},
{
startIndex: 82,
endIndex: 83,
content: '`',
formattingStart: 'code',
},
{
startIndex: 83,
endIndex: 94,
content: 'inline code'
},
{
startIndex: 94,
endIndex: 95,
content: '`',
formattingEnd: 'code'
},
{
startIndex: 95,
endIndex: 102,
content: ' and a '
},
{
startIndex: 102,
endIndex: 103,
content: '[',
formattingStart: 'hyperlinkText'
},
{
startIndex: 103,
endIndex: 112,
content: 'hyperlink'
},
{
startIndex: 112,
endIndex: 113,
content: ']',
formattingEnd: 'hyperlinkText'
},
{
startIndex: 113,
endIndex: 114,
content: '(',
formattingStart: 'hyperlinkAddress'
},
{
startIndex: 114,
endIndex: 133,
content: 'https://example.com'
},
{
startIndex: 133,
endIndex: 134,
content: ')',
formattingEnd: 'hyperlinkAddress'
},
{
startIndex: 134,
endIndex: 135,
content: '.'
}
]);
expect(result[43]).toEqual([
{
startIndex: 0,
endIndex: 1,
content: '#',
formattingStart: 'heading',
formattingMetadata: 'h1'
},
{
startIndex: 1,
endIndex: 15,
content: ' Heading with '
},
{
startIndex: 15,
endIndex: 17,
content: '**',
formattingStart: 'bold'
},
{
startIndex: 17,
endIndex: 21,
content: 'bold'
},
{
startIndex: 21,
endIndex: 23,
content: '**',
formattingEnd: 'bold'
},
{
startIndex: 23,
endIndex: 25,
content: ', '
},
{
startIndex: 25,
endIndex: 26,
content: '*',
formattingStart: 'italic'
},
{
startIndex: 26,
endIndex: 32,
content: 'italic'
},
{
startIndex: 32,
endIndex: 33,
content: '*',
formattingEnd: 'italic'
},
{
startIndex: 33,
endIndex: 35,
content: ', '
},
{
startIndex: 35,
endIndex: 37,
content: '~~',
formattingStart: 'strikethrough'
},
{
startIndex: 37,
endIndex: 50,
content: 'strikethrough'
},
{
startIndex: 50,
endIndex: 52,
content: '~~',
formattingEnd: 'strikethrough'
},
{
startIndex: 52,
endIndex: 54,
content: ', '
},
{
startIndex: 54,
endIndex: 56,
content: '__',
formattingStart: 'bold'
},
{
startIndex: 56,
endIndex: 65,
content: 'more bold'
},
{
startIndex: 65,
endIndex: 67,
content: '__',
formattingEnd: 'bold'
},
{
startIndex: 67,
endIndex: 69,
content: ', '
},
{
startIndex: 69,
endIndex: 70,
content: '_',
formattingStart: 'italic'
},
{
startIndex: 70,
endIndex: 81,
content: 'more italic'
},
{
startIndex: 81,
endIndex: 82,
content: '_',
formattingEnd: 'italic'
},
{
startIndex: 82,
endIndex: 84,
content: ', '
},
{
startIndex: 84,
endIndex: 85,
content: '`',
formattingStart: 'code',
},
{
startIndex: 85,
endIndex: 96,
content: 'inline code'
},
{
startIndex: 96,
endIndex: 97,
content: '`',
formattingEnd: 'code'
},
{
startIndex: 97,
endIndex: 104,
content: ' and a '
},
{
startIndex: 104,
endIndex: 105,
content: '[',
formattingStart: 'hyperlinkText'
},
{
startIndex: 105,
endIndex: 114,
content: 'hyperlink'
},
{
startIndex: 114,
endIndex: 115,
content: ']',
formattingEnd: 'hyperlinkText'
},
{
startIndex: 115,
endIndex: 116,
content: '(',
formattingStart: 'hyperlinkAddress'
},
{
startIndex: 116,
endIndex: 135,
content: 'https://example.com'
},
{
startIndex: 135,
endIndex: 136,
content: ')',
formattingEnd: 'hyperlinkAddress'
},
{
startIndex: 136,
endIndex: 137,
content: '.'
}
]);
}
);
}
);