extern/heynote
1
0
Fork 0
mirror of https://github.com/heyman/heynote.git synced 2025-06-20 17:47:42 +02:00
Code Issues Packages Projects Releases Wiki Activity

Optimize the Lezer external tokenizer for block content

Browse Source
This commit is contained in:
Jonatan Heyman 2023-01-14 13:08:24 +01:00
parent ab1a80199e
commit 52f1ac71da
1 changed files with 14 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
src/editor/lang-heynote/external-tokens.js
View File

@ -3,6 +3,9 @@ import { NoteContent } from "./parser.terms.js"
const EOF = -1; const EOF = -1;
const FIRST_TOKEN_CHAR = "\n".charCodeAt(0)
const SECOND_TOKEN_CHAR = "∞".charCodeAt(0)
export const noteContent = new ExternalTokenizer((input) => { export const noteContent = new ExternalTokenizer((input) => {
let current = input.peek(0); let current = input.peek(0);
let next = input.peek(1); let next = input.peek(1);
@ -12,6 +15,9 @@ export const noteContent = new ExternalTokenizer((input) => {
} }
while (true) { while (true) {
// unless the first two characters are a newline and a "∞" character, we don't have a note content token
// so we don't need to check for the rest of the token
if (current === FIRST_TOKEN_CHAR && next === SECOND_TOKEN_CHAR) {
let potentialLang = ""; let potentialLang = "";
for (let i=0; i<18; i++) { for (let i=0; i<18; i++) {
potentialLang += String.fromCharCode(input.peek(i)); potentialLang += String.fromCharCode(input.peek(i));
@ -20,6 +26,7 @@ export const noteContent = new ExternalTokenizer((input) => {
input.acceptToken(NoteContent); input.acceptToken(NoteContent);
return; return;
} }
}
if (next === EOF) { if (next === EOF) {
input.acceptToken(NoteContent, 1); input.acceptToken(NoteContent, 1);
return; return;