Add CSS support

Add Language class and move language related settings into editor/language.js

In order to add an additional language now, one only has to add in to the LANGUAGES list, and add the token to heynote.grammar

src/components/StatusBar.vue

@@ -1,16 +1,6 @@
 <script>
-    const LANGUAGE_NAMES = {
+    import { LANGUAGES } from '../editor/languages.js'
-        "text": "Plain Text",
+    const LANGUAGE_NAMES = Object.fromEntries(LANGUAGES.map(l => [l.token, l.name]))
-        "javascript": "JavaScript",
-        "json": "JSON",
-        "python": "Python",
-        "html": "HTML",
-        "sql": "SQL",
-        "markdown": "Markdown",
-        "java": "Java",
-        "lezer": "Lezer",
-        "php": "PHP",
-    }
 
     export default {
         props: [

src/editor/block/commands.js

@@ -5,7 +5,6 @@ import {
     moveLineUp as defaultMoveLineUp,
 } from "@codemirror/commands"
 import { heynoteEvent, LANGUAGE_CHANGE } from "../annotation.js";
-import { HIGHLIGHTJS_TO_TOKEN } from "../languages"
 import { blockState, getActiveNoteBlock, getNoteBlockFromPos } from "./block"
 import { levenshtein_distance } from "../language-detection/levenshtein"
 

src/editor/lang-heynote/external-tokens.js

@@ -1,11 +1,15 @@
 import { ExternalTokenizer } from '@lezer/lr'
 import { NoteContent } from "./parser.terms.js"
+import { LANGUAGES } from '../languages.js';
 
 const EOF = -1;
 
 const FIRST_TOKEN_CHAR = "\n".charCodeAt(0)
 const SECOND_TOKEN_CHAR = "∞".charCodeAt(0)
 
+const languageTokensMatcher = LANGUAGES.map(l => l.token).join("|")
+const tokenRegEx = new RegExp(`^\\n∞∞∞(${languageTokensMatcher})(-a)?\\n`, "g")
+
 export const noteContent = new ExternalTokenizer((input) => {
     let current = input.peek(0);
     let next = input.peek(1);
@@ -22,7 +26,7 @@ export const noteContent = new ExternalTokenizer((input) => {
             for (let i=0; i<18; i++) {
                 potentialLang += String.fromCharCode(input.peek(i));
             }
-            if (potentialLang.match(/^\n∞∞∞(text|javascript|json|python|html|sql|markdown|java|lezer|php)(-a)?\n/g)) {
+            if (potentialLang.match(tokenRegEx)) {
                 input.acceptToken(NoteContent);
                 return;
             }

src/editor/lang-heynote/heynote.grammar

@@ -11,7 +11,7 @@ NoteDelimiter {
 
 @tokens {
     noteDelimiterMark { "∞∞∞" }
-    NoteLanguage { "text" | "javascript" | "json" | "python" | "html" | "sql" | "markdown" | "java" | "lezer" | "php" }
+    NoteLanguage { "text" | "javascript" | "json" | "python" | "html" | "sql" | "markdown" | "java" | "lezer" | "php" | "css" }
     Auto { "-a" }
     noteDelimiterEnter { "\n" }
     //NoteContent { String }

src/editor/lang-heynote/nested-parser.js

@@ -11,19 +11,9 @@ import { lezerLanguage } from "@codemirror/lang-lezer"
 import { phpLanguage } from "@codemirror/lang-php"
 
 import { NoteContent, NoteLanguage } from "./parser.terms.js"
+import { LANGUAGES } from "../languages.js"
 
-
+const languageMapping = Object.fromEntries(LANGUAGES.map(l => [l.token, l.parser]))
-const languageMapping = {
-    "json": jsonLanguage.parser,
-    "javascript": javascriptLanguage.parser,
-    "python": pythonLanguage.parser,
-    "html": htmlLanguage.parser,
-    "sql": StandardSQL.language.parser,
-    "markdown": markdownLanguage.parser,
-    "java": javaLanguage.parser,
-    "lezer": lezerLanguage.parser,
-    "php": phpLanguage.parser,
-}
 
 
 export function configureNesting() {
@@ -33,7 +23,7 @@ export function configureNesting() {
             let noteLang = node.node.parent.firstChild.getChildren(NoteLanguage)[0]
             let langName = input.read(noteLang?.from, noteLang?.to)
             //console.log("langName:", langName)
-            if (langName in languageMapping) {
+            if (langName in languageMapping && languageMapping[langName] !== null) {
                 //console.log("found parser for language:", langName)
                 return {
                     parser:languageMapping[langName],

src/editor/lang-heynote/parser.js

@@ -10,7 +10,7 @@ export const parser = LRParser.deserialize({
   maxTerm: 10,
   skippedNodes: [0],
   repeatNodeCount: 1,
-  tokenData: "&s~RYYZq}!Ov#[#]!R#^#_!j#`#a$T#a#b$m#d#e%c#g#h&O#h#i&U%&x%&y&b~vOX~~yP#T#U|~!ROU~~!UP#h#i!X~![P#a#b!_~!bP#`#a!e~!jOT~~!mQ#T#U!s#g#h#w~!vP#j#k!y~!|P#T#U#P~#UPT~#g#h#X~#[P#V#W#_~#bP#f#g#e~#hP#]#^#k~#nP#d#e#q~#tP#h#i!e~#zP#c#d#}~$QP#b#c!e~$WP#X#Y$Z~$^P#n#o$a~$dP#X#Y$g~$jP#f#g!e~$pP#T#U$s~$vP#f#g$y~$|P#_#`%P~%SP#W#X%V~%YP#c#d%]~%`P#k#l#}~%fQ#[#]%l#m#n%r~%oP#d#e!e~%uP#h#i%x~%{P#[#]#w~&RP#e#f!_~&XP#X#Y&[~&_P#l#m#q~&eP%&x%&y&h~&kP%&x%&y&n~&sOY~",
+  tokenData: "'S~RZYZt}!Oy#V#W!U#[#]!g#^#_!y#`#a$d#a#b$|#d#e%r#g#h&_#h#i&e%&x%&y&q~yOX~~|P#T#U!P~!UOU~~!XP#g#h![~!_P#g#h!b~!gOT~~!jP#h#i!m~!pP#a#b!s~!vP#`#a!b~!|Q#T#U#S#g#h$W~#VP#j#k#Y~#]P#T#U#`~#ePT~#g#h#h~#kP#V#W#n~#qP#f#g#t~#wP#]#^#z~#}P#d#e$Q~$TP#h#i!b~$ZP#c#d$^~$aP#b#c!b~$gP#X#Y$j~$mP#n#o$p~$sP#X#Y$v~$yP#f#g!b~%PP#T#U%S~%VP#f#g%Y~%]P#_#`%`~%cP#W#X%f~%iP#c#d%l~%oP#k#l$^~%uQ#[#]%{#m#n&R~&OP#d#e!b~&UP#h#i&X~&[P#[#]$W~&bP#e#f!s~&hP#X#Y&k~&nP#l#m$Q~&tP%&x%&y&w~&zP%&x%&y&}~'SOY~",
   tokenizers: [0, noteContent],
   topRules: {"Document":[0,2]},
   tokenPrec: 0

src/editor/language-detection/autodetect.js

@@ -3,10 +3,12 @@ import { EditorView } from "codemirror";
 import { redoDepth } from "@codemirror/commands";
 import { getActiveNoteBlock, blockState } from "../block/block";
 import { levenshtein_distance } from "./levenshtein";
-import { HIGHLIGHTJS_TO_TOKEN } from "../languages";
+import { LANGUAGES } from "../languages";
 import { changeLanguageTo } from "../block/commands";
 import { LANGUAGE_CHANGE } from "../annotation";
 
+const HIGHLIGHTJS_TO_TOKEN = Object.fromEntries(LANGUAGES.map(l => [l.highlightjs,l.token]))
+
 
 export function languageDetection(getView) {
     const previousBlockContent = []

src/editor/languages.js

@@ -1,17 +1,35 @@
-export const LANGUAGE_TOKENS = [
+import { jsonLanguage } from "@codemirror/lang-json"
-    "text",
+import { pythonLanguage } from "@codemirror/lang-python"
-    "javascript",
+import { javascriptLanguage } from "@codemirror/lang-javascript"
-    "json",
+import { htmlLanguage } from "@codemirror/lang-html"
-    "python",
+import { StandardSQL } from "@codemirror/lang-sql"
-    "html",
+import { markdownLanguage } from "@codemirror/lang-markdown"
-    "sql",
+import { javaLanguage } from "@codemirror/lang-java"
-    "markdown",
+import { lezerLanguage } from "@codemirror/lang-lezer"
-    "java",
+import { phpLanguage } from "@codemirror/lang-php"
-    "lezer",
+import { cssLanguage } from "@codemirror/lang-css"
-    "php",
+
+
+class Language {
+    constructor(token, name, parser, highlightjs) {
+        this.token = token
+        this.name = name
+        this.parser = parser
+        this.highlightjs = highlightjs
+    }
+}
+
+export const LANGUAGES = [
+    new Language("text", "Plain Text", null, "plaintext"),
+    new Language("javascript", "JavaScript", javascriptLanguage.parser, "javascript"),
+    new Language("json", "JSON", jsonLanguage.parser, "json"),
+    new Language("python", "Python", pythonLanguage.parser, "python"),
+    new Language("html", "HTML", htmlLanguage.parser, "html"),
+    new Language("sql", "SQL", StandardSQL.language.parser, "sql"),
+    new Language("markdown", "Markdown", markdownLanguage.parser, "markdown"),
+    new Language("java", "Java", javaLanguage.parser, "java"),
+    new Language("lezer", "Lezer", lezerLanguage.parser, "lezer"),
+    new Language("php", "PHP", phpLanguage.parser, "php"),
+    new Language("css", "CSS", cssLanguage.parser, "css"),
 ]
 
-export const HIGHLIGHTJS_TO_TOKEN = Object.fromEntries(LANGUAGE_TOKENS.map(l => [l,l]))
-HIGHLIGHTJS_TO_TOKEN["plaintext"] = "text"
-
-