Add CSS support

Add Language class and move language related settings into editor/language.js

In order to add an additional language now, one only has to add in to the LANGUAGES list, and add the token to heynote.grammar
This commit is contained in:
Jonatan Heyman 2023-01-16 20:36:07 +01:00
parent 4660f42460
commit 41573820ae
8 changed files with 48 additions and 45 deletions

View File

@ -1,16 +1,6 @@
<script>
const LANGUAGE_NAMES = {
"text": "Plain Text",
"javascript": "JavaScript",
"json": "JSON",
"python": "Python",
"html": "HTML",
"sql": "SQL",
"markdown": "Markdown",
"java": "Java",
"lezer": "Lezer",
"php": "PHP",
}
import { LANGUAGES } from '../editor/languages.js'
const LANGUAGE_NAMES = Object.fromEntries(LANGUAGES.map(l => [l.token, l.name]))
export default {
props: [

View File

@ -5,7 +5,6 @@ import {
moveLineUp as defaultMoveLineUp,
} from "@codemirror/commands"
import { heynoteEvent, LANGUAGE_CHANGE } from "../annotation.js";
import { HIGHLIGHTJS_TO_TOKEN } from "../languages"
import { blockState, getActiveNoteBlock, getNoteBlockFromPos } from "./block"
import { levenshtein_distance } from "../language-detection/levenshtein"

View File

@ -1,11 +1,15 @@
import { ExternalTokenizer } from '@lezer/lr'
import { NoteContent } from "./parser.terms.js"
import { LANGUAGES } from '../languages.js';
const EOF = -1;
const FIRST_TOKEN_CHAR = "\n".charCodeAt(0)
const SECOND_TOKEN_CHAR = "∞".charCodeAt(0)
const languageTokensMatcher = LANGUAGES.map(l => l.token).join("|")
const tokenRegEx = new RegExp(`^\\n∞∞∞(${languageTokensMatcher})(-a)?\\n`, "g")
export const noteContent = new ExternalTokenizer((input) => {
let current = input.peek(0);
let next = input.peek(1);
@ -22,7 +26,7 @@ export const noteContent = new ExternalTokenizer((input) => {
for (let i=0; i<18; i++) {
potentialLang += String.fromCharCode(input.peek(i));
}
if (potentialLang.match(/^\n∞∞∞(text|javascript|json|python|html|sql|markdown|java|lezer|php)(-a)?\n/g)) {
if (potentialLang.match(tokenRegEx)) {
input.acceptToken(NoteContent);
return;
}

View File

@ -11,7 +11,7 @@ NoteDelimiter {
@tokens {
noteDelimiterMark { "∞∞∞" }
NoteLanguage { "text" | "javascript" | "json" | "python" | "html" | "sql" | "markdown" | "java" | "lezer" | "php" }
NoteLanguage { "text" | "javascript" | "json" | "python" | "html" | "sql" | "markdown" | "java" | "lezer" | "php" | "css" }
Auto { "-a" }
noteDelimiterEnter { "\n" }
//NoteContent { String }

View File

@ -11,19 +11,9 @@ import { lezerLanguage } from "@codemirror/lang-lezer"
import { phpLanguage } from "@codemirror/lang-php"
import { NoteContent, NoteLanguage } from "./parser.terms.js"
import { LANGUAGES } from "../languages.js"
const languageMapping = {
"json": jsonLanguage.parser,
"javascript": javascriptLanguage.parser,
"python": pythonLanguage.parser,
"html": htmlLanguage.parser,
"sql": StandardSQL.language.parser,
"markdown": markdownLanguage.parser,
"java": javaLanguage.parser,
"lezer": lezerLanguage.parser,
"php": phpLanguage.parser,
}
const languageMapping = Object.fromEntries(LANGUAGES.map(l => [l.token, l.parser]))
export function configureNesting() {
@ -33,7 +23,7 @@ export function configureNesting() {
let noteLang = node.node.parent.firstChild.getChildren(NoteLanguage)[0]
let langName = input.read(noteLang?.from, noteLang?.to)
//console.log("langName:", langName)
if (langName in languageMapping) {
if (langName in languageMapping && languageMapping[langName] !== null) {
//console.log("found parser for language:", langName)
return {
parser:languageMapping[langName],

View File

@ -10,7 +10,7 @@ export const parser = LRParser.deserialize({
maxTerm: 10,
skippedNodes: [0],
repeatNodeCount: 1,
tokenData: "&s~RYYZq}!Ov#[#]!R#^#_!j#`#a$T#a#b$m#d#e%c#g#h&O#h#i&U%&x%&y&b~vOX~~yP#T#U|~!ROU~~!UP#h#i!X~![P#a#b!_~!bP#`#a!e~!jOT~~!mQ#T#U!s#g#h#w~!vP#j#k!y~!|P#T#U#P~#UPT~#g#h#X~#[P#V#W#_~#bP#f#g#e~#hP#]#^#k~#nP#d#e#q~#tP#h#i!e~#zP#c#d#}~$QP#b#c!e~$WP#X#Y$Z~$^P#n#o$a~$dP#X#Y$g~$jP#f#g!e~$pP#T#U$s~$vP#f#g$y~$|P#_#`%P~%SP#W#X%V~%YP#c#d%]~%`P#k#l#}~%fQ#[#]%l#m#n%r~%oP#d#e!e~%uP#h#i%x~%{P#[#]#w~&RP#e#f!_~&XP#X#Y&[~&_P#l#m#q~&eP%&x%&y&h~&kP%&x%&y&n~&sOY~",
tokenData: "'S~RZYZt}!Oy#V#W!U#[#]!g#^#_!y#`#a$d#a#b$|#d#e%r#g#h&_#h#i&e%&x%&y&q~yOX~~|P#T#U!P~!UOU~~!XP#g#h![~!_P#g#h!b~!gOT~~!jP#h#i!m~!pP#a#b!s~!vP#`#a!b~!|Q#T#U#S#g#h$W~#VP#j#k#Y~#]P#T#U#`~#ePT~#g#h#h~#kP#V#W#n~#qP#f#g#t~#wP#]#^#z~#}P#d#e$Q~$TP#h#i!b~$ZP#c#d$^~$aP#b#c!b~$gP#X#Y$j~$mP#n#o$p~$sP#X#Y$v~$yP#f#g!b~%PP#T#U%S~%VP#f#g%Y~%]P#_#`%`~%cP#W#X%f~%iP#c#d%l~%oP#k#l$^~%uQ#[#]%{#m#n&R~&OP#d#e!b~&UP#h#i&X~&[P#[#]$W~&bP#e#f!s~&hP#X#Y&k~&nP#l#m$Q~&tP%&x%&y&w~&zP%&x%&y&}~'SOY~",
tokenizers: [0, noteContent],
topRules: {"Document":[0,2]},
tokenPrec: 0

View File

@ -3,10 +3,12 @@ import { EditorView } from "codemirror";
import { redoDepth } from "@codemirror/commands";
import { getActiveNoteBlock, blockState } from "../block/block";
import { levenshtein_distance } from "./levenshtein";
import { HIGHLIGHTJS_TO_TOKEN } from "../languages";
import { LANGUAGES } from "../languages";
import { changeLanguageTo } from "../block/commands";
import { LANGUAGE_CHANGE } from "../annotation";
const HIGHLIGHTJS_TO_TOKEN = Object.fromEntries(LANGUAGES.map(l => [l.highlightjs,l.token]))
export function languageDetection(getView) {
const previousBlockContent = []

View File

@ -1,17 +1,35 @@
export const LANGUAGE_TOKENS = [
"text",
"javascript",
"json",
"python",
"html",
"sql",
"markdown",
"java",
"lezer",
"php",
import { jsonLanguage } from "@codemirror/lang-json"
import { pythonLanguage } from "@codemirror/lang-python"
import { javascriptLanguage } from "@codemirror/lang-javascript"
import { htmlLanguage } from "@codemirror/lang-html"
import { StandardSQL } from "@codemirror/lang-sql"
import { markdownLanguage } from "@codemirror/lang-markdown"
import { javaLanguage } from "@codemirror/lang-java"
import { lezerLanguage } from "@codemirror/lang-lezer"
import { phpLanguage } from "@codemirror/lang-php"
import { cssLanguage } from "@codemirror/lang-css"
class Language {
constructor(token, name, parser, highlightjs) {
this.token = token
this.name = name
this.parser = parser
this.highlightjs = highlightjs
}
}
export const LANGUAGES = [
new Language("text", "Plain Text", null, "plaintext"),
new Language("javascript", "JavaScript", javascriptLanguage.parser, "javascript"),
new Language("json", "JSON", jsonLanguage.parser, "json"),
new Language("python", "Python", pythonLanguage.parser, "python"),
new Language("html", "HTML", htmlLanguage.parser, "html"),
new Language("sql", "SQL", StandardSQL.language.parser, "sql"),
new Language("markdown", "Markdown", markdownLanguage.parser, "markdown"),
new Language("java", "Java", javaLanguage.parser, "java"),
new Language("lezer", "Lezer", lezerLanguage.parser, "lezer"),
new Language("php", "PHP", phpLanguage.parser, "php"),
new Language("css", "CSS", cssLanguage.parser, "css"),
]
export const HIGHLIGHTJS_TO_TOKEN = Object.fromEntries(LANGUAGE_TOKENS.map(l => [l,l]))
HIGHLIGHTJS_TO_TOKEN["plaintext"] = "text"