498 lines
No EOL
16 KiB
JavaScript
498 lines
No EOL
16 KiB
JavaScript
import {
|
|
describe,
|
|
test,
|
|
expect
|
|
} from 'bun:test';
|
|
|
|
import outdent from 'outdent';
|
|
import Joi from 'joi';
|
|
|
|
import {
|
|
tokenize
|
|
} from './mod.js';
|
|
|
|
describe(
|
|
'tokenize comprehensive markdown sample',
|
|
() => {
|
|
const markdown = outdent `
|
|
This is plain text with **bold text** and *italic text* and **_bold italic text_** and ~~strikethrough text~~.
|
|
|
|
Another paragraph with \`inline code\` and a [hyperlink](https://example.com).
|
|
|
|
|
|
|
|
Unordered list:
|
|
- First item
|
|
- Second item with **bold**
|
|
- Third item
|
|
|
|
Ordered list:
|
|
1. First numbered item
|
|
2. Second numbered item with *italic*
|
|
3. Third numbered item
|
|
|
|
\`\`\`js
|
|
function example() {
|
|
return "code block";
|
|
}
|
|
\`\`\`
|
|
`;
|
|
const result = tokenize(markdown);
|
|
test(
|
|
'outputs array of 21 lines',
|
|
() => {
|
|
expect(Array.isArray(result)).toBe(true);
|
|
expect(result.length).toBe(21);
|
|
}
|
|
);
|
|
test(
|
|
'validates schema',
|
|
() => {
|
|
const
|
|
formattingTypes = [
|
|
'bold',
|
|
'italic',
|
|
'strikethrough',
|
|
'code',
|
|
'hyperlinkText',
|
|
'hyperlinkAddress',
|
|
'imageText',
|
|
'imageAddress',
|
|
'listItem',
|
|
'orderedListItem',
|
|
'codeblock'
|
|
],
|
|
schema = Joi.array().items(
|
|
Joi.array().items(
|
|
Joi
|
|
.object({
|
|
startIndex: Joi.number().integer().required(),
|
|
endIndex: Joi.number().integer().required(),
|
|
content: Joi.string().required(),
|
|
formattingStart: Joi.valid(...formattingTypes),
|
|
formattingEnd: Joi.valid(...formattingTypes),
|
|
formattingMetadata: Joi.valid('codeblockLanguage')
|
|
})
|
|
.oxor(
|
|
'formattingStart',
|
|
'formattingEnd',
|
|
'formattingMetadata'
|
|
)
|
|
)
|
|
);
|
|
expect(schema.validate(result).error).toBeUndefined();
|
|
}
|
|
);
|
|
test(
|
|
'parses empty lines',
|
|
() => {
|
|
[1, 3, 5, 10, 15].forEach(index => {
|
|
expect(Array.isArray(result[index])).toBe(true);
|
|
expect(result[index].length).toBe(0);
|
|
});
|
|
}
|
|
);
|
|
test(
|
|
'parses paragraph 1',
|
|
() => {
|
|
expect(result[0]).toEqual([
|
|
{
|
|
startIndex: 0,
|
|
endIndex: 24,
|
|
content: 'This is plain text with '
|
|
},
|
|
{
|
|
startIndex: 24,
|
|
endIndex: 26,
|
|
content: '**',
|
|
formattingStart: 'bold'
|
|
},
|
|
{
|
|
startIndex: 26,
|
|
endIndex: 35,
|
|
content: 'bold text'
|
|
},
|
|
{
|
|
startIndex: 35,
|
|
endIndex: 37,
|
|
content: '**',
|
|
formattingEnd: 'bold'
|
|
},
|
|
{
|
|
startIndex: 37,
|
|
endIndex: 42,
|
|
content: ' and '
|
|
},
|
|
{
|
|
startIndex: 42,
|
|
endIndex: 43,
|
|
content: '*',
|
|
formattingStart: 'italic'
|
|
},
|
|
{
|
|
startIndex: 43,
|
|
endIndex: 54,
|
|
content: 'italic text'
|
|
},
|
|
{
|
|
startIndex: 54,
|
|
endIndex: 55,
|
|
content: '*',
|
|
formattingEnd: 'italic'
|
|
},
|
|
{
|
|
startIndex: 55,
|
|
endIndex: 60,
|
|
content: ' and '
|
|
},
|
|
{
|
|
startIndex: 60,
|
|
endIndex: 62,
|
|
content: '**',
|
|
formattingStart: 'bold'
|
|
},
|
|
{
|
|
startIndex: 62,
|
|
endIndex: 63,
|
|
content: '_',
|
|
formattingStart: 'italic'
|
|
},
|
|
{
|
|
startIndex: 63,
|
|
endIndex: 79,
|
|
content: 'bold italic text'
|
|
},
|
|
{
|
|
startIndex: 79,
|
|
endIndex: 80,
|
|
content: '_',
|
|
formattingEnd: 'italic'
|
|
},
|
|
{
|
|
startIndex: 80,
|
|
endIndex: 82,
|
|
content: '**',
|
|
formattingEnd: 'bold'
|
|
},
|
|
{
|
|
startIndex: 82,
|
|
endIndex: 87,
|
|
content: ' and '
|
|
},
|
|
{
|
|
startIndex: 87,
|
|
endIndex: 89,
|
|
content: '~~',
|
|
formattingStart: 'strikethrough'
|
|
},
|
|
{
|
|
startIndex: 89,
|
|
endIndex: 107,
|
|
content: 'strikethrough text'
|
|
},
|
|
{
|
|
startIndex: 107,
|
|
endIndex: 109,
|
|
content: '~~',
|
|
formattingEnd: 'strikethrough'
|
|
},
|
|
{
|
|
startIndex: 109,
|
|
endIndex: 110,
|
|
content: '.'
|
|
}
|
|
]);
|
|
}
|
|
);
|
|
test(
|
|
'parses paragraph 2',
|
|
() => {
|
|
expect(result[2]).toEqual([
|
|
{
|
|
startIndex: 0,
|
|
endIndex: 23,
|
|
content: 'Another paragraph with '
|
|
},
|
|
{
|
|
startIndex: 23,
|
|
endIndex: 24,
|
|
content: '`',
|
|
formattingStart: 'code',
|
|
},
|
|
{
|
|
startIndex: 24,
|
|
endIndex: 35,
|
|
content: 'inline code'
|
|
},
|
|
{
|
|
startIndex: 35,
|
|
endIndex: 36,
|
|
content: '`',
|
|
formattingEnd: 'code'
|
|
},
|
|
{
|
|
startIndex: 36,
|
|
endIndex: 43,
|
|
content: ' and a '
|
|
},
|
|
{
|
|
startIndex: 43,
|
|
endIndex: 44,
|
|
content: '[',
|
|
formattingStart: 'hyperlinkText'
|
|
},
|
|
{
|
|
startIndex: 44,
|
|
endIndex: 53,
|
|
content: 'hyperlink'
|
|
},
|
|
{
|
|
startIndex: 53,
|
|
endIndex: 54,
|
|
content: ']',
|
|
formattingEnd: 'hyperlinkText'
|
|
},
|
|
{
|
|
startIndex: 54,
|
|
endIndex: 55,
|
|
content: '(',
|
|
formattingStart: 'hyperlinkAddress'
|
|
},
|
|
{
|
|
startIndex: 55,
|
|
endIndex: 74,
|
|
content: 'https://example.com'
|
|
},
|
|
{
|
|
startIndex: 74,
|
|
endIndex: 75,
|
|
content: ')',
|
|
formattingEnd: 'hyperlinkAddress'
|
|
},
|
|
{
|
|
startIndex: 75,
|
|
endIndex: 76,
|
|
content: '.'
|
|
}
|
|
]);
|
|
}
|
|
);
|
|
test(
|
|
'parses paragraph 3',
|
|
() => {
|
|
expect(result[4]).toEqual([
|
|
{
|
|
startIndex: 0,
|
|
endIndex: 2,
|
|
content: '![',
|
|
formattingStart: 'imageText'
|
|
},
|
|
{
|
|
startIndex: 2,
|
|
endIndex: 7,
|
|
content: 'image'
|
|
},
|
|
{
|
|
startIndex: 7,
|
|
endIndex: 8,
|
|
content: ']',
|
|
formattingEnd: 'imageText'
|
|
},
|
|
{
|
|
startIndex: 8,
|
|
endIndex: 9,
|
|
content: '(',
|
|
formattingStart: 'imageAddress'
|
|
},
|
|
{
|
|
startIndex: 9,
|
|
endIndex: 18,
|
|
content: 'image.jpg'
|
|
},
|
|
{
|
|
startIndex: 18,
|
|
endIndex: 19,
|
|
content: ')',
|
|
formattingEnd: 'imageAddress'
|
|
}
|
|
]);
|
|
}
|
|
);
|
|
test(
|
|
'parses paragraph 4',
|
|
() => {
|
|
expect(result[6]).toEqual([
|
|
{
|
|
startIndex: 0,
|
|
endIndex: 15,
|
|
content: 'Unordered list:'
|
|
}
|
|
]);
|
|
expect(result[7]).toEqual([
|
|
{
|
|
startIndex: 0,
|
|
endIndex: 2,
|
|
content: '- ',
|
|
formattingStart: 'listItem'
|
|
},
|
|
{
|
|
startIndex: 2,
|
|
endIndex: 12,
|
|
content: 'First item'
|
|
}
|
|
]);
|
|
expect(result[8]).toEqual([
|
|
{
|
|
startIndex: 0,
|
|
endIndex: 2,
|
|
content: '- ',
|
|
formattingStart: 'listItem'
|
|
},
|
|
{
|
|
startIndex: 2,
|
|
endIndex: 19,
|
|
content: 'Second item with '
|
|
},
|
|
{
|
|
startIndex: 19,
|
|
endIndex: 21,
|
|
content: '**',
|
|
formattingStart: 'bold'
|
|
},
|
|
{
|
|
startIndex: 21,
|
|
endIndex: 25,
|
|
content: 'bold'
|
|
},
|
|
{
|
|
startIndex: 25,
|
|
endIndex: 27,
|
|
content: '**',
|
|
formattingEnd: 'bold'
|
|
}
|
|
]);
|
|
expect(result[9]).toEqual([
|
|
{
|
|
startIndex: 0,
|
|
endIndex: 2,
|
|
content: '- ',
|
|
formattingStart: 'listItem'
|
|
},
|
|
{
|
|
startIndex: 2,
|
|
endIndex: 12,
|
|
content: 'Third item'
|
|
}
|
|
]);
|
|
}
|
|
);
|
|
test(
|
|
'parses paragraph 5',
|
|
() => {
|
|
expect(result[11]).toEqual([
|
|
{
|
|
startIndex: 0,
|
|
endIndex: 13,
|
|
content: 'Ordered list:'
|
|
}
|
|
]);
|
|
expect(result[12]).toEqual([
|
|
{
|
|
startIndex: 0,
|
|
endIndex: 3,
|
|
content: '1. ',
|
|
formattingStart: 'orderedListItem'
|
|
},
|
|
{
|
|
startIndex: 3,
|
|
endIndex: 22,
|
|
content: 'First numbered item'
|
|
}
|
|
]);
|
|
expect(result[13]).toEqual([
|
|
{
|
|
startIndex: 0,
|
|
endIndex: 3,
|
|
content: '2. ',
|
|
formattingStart: 'orderedListItem'
|
|
},
|
|
{
|
|
startIndex: 3,
|
|
endIndex: 29,
|
|
content: 'Second numbered item with '
|
|
},
|
|
{
|
|
startIndex: 29,
|
|
endIndex: 30,
|
|
content: '*',
|
|
formattingStart: 'italic'
|
|
},
|
|
{
|
|
startIndex: 30,
|
|
endIndex: 36,
|
|
content: 'italic'
|
|
},
|
|
{
|
|
startIndex: 36,
|
|
endIndex: 37,
|
|
content: '*',
|
|
formattingEnd: 'italic'
|
|
}
|
|
]);
|
|
expect(result[14]).toEqual([
|
|
{
|
|
startIndex: 0,
|
|
endIndex: 3,
|
|
content: '3. ',
|
|
formattingStart: 'orderedListItem'
|
|
},
|
|
{
|
|
startIndex: 3,
|
|
endIndex: 22,
|
|
content: 'Third numbered item'
|
|
}
|
|
]);
|
|
}
|
|
);
|
|
test(
|
|
'parses paragraph 6',
|
|
() => {
|
|
expect(result[16]).toEqual([
|
|
{
|
|
startIndex: 0,
|
|
endIndex: 3,
|
|
content: '```',
|
|
formattingStart: 'codeblock'
|
|
},
|
|
{
|
|
startIndex: 3,
|
|
endIndex: 5,
|
|
content: 'js',
|
|
formattingMetadata: 'codeblockLanguage'
|
|
}
|
|
]);
|
|
expect(result[17]).toEqual([{
|
|
startIndex: 0,
|
|
endIndex: 20,
|
|
content: 'function example() {'
|
|
}]);
|
|
expect(result[18]).toEqual([{
|
|
startIndex: 0,
|
|
endIndex: 24,
|
|
content: ' return "code block";'
|
|
}]);
|
|
expect(result[19]).toEqual([{
|
|
startIndex: 0,
|
|
endIndex: 1,
|
|
content: '}'
|
|
}]);
|
|
expect(result[20]).toEqual([{
|
|
startIndex: 0,
|
|
endIndex: 3,
|
|
content: '```',
|
|
formattingEnd: 'codeblock'
|
|
}]);
|
|
}
|
|
);
|
|
}
|
|
); |