Add support for more languages (#69)

* Contain language selection dialog in an element that can be scrolled, and automatically scroll it if needed when navigating the list with arrow keys

* Add support for more languages:

Clojure, Erlang, Golang, Lezer, Ruby, Shell, YAML

* Move prettier auto format settings for languages into Language() class

* Remove invalid import

* Fix bug that could cause auto formatting to fail for the last block.
Add tests for language auto detection and formatting.

* Fix broken tests

* Fix language auto detection on Safari Webkit which was broken

* Remove unnecessary wait time
This commit is contained in:
Jonatan Heyman 2023-12-26 00:27:43 +01:00 committed by GitHub
parent 6eda3efa63
commit bb511b868b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 361 additions and 70 deletions

View File

@ -17,17 +17,24 @@ Available for Mac, Windows, and Linux.
- Syntax highlighting
- C++
- C#
- Clojure
- CSS
- Erlang
- Go
- HTML
- Java
- JavaScript
- JSON
- Lezer
- Markdown
- PHP
- Python
- Ruby
- Rust
- Shell
- SQL
- XML
- YAML
- Language auto-detection
- Auto-formatting
- Math/Calculator mode

10
package-lock.json generated
View File

@ -27,6 +27,7 @@
"@codemirror/lang-sql": "^6.5.4",
"@codemirror/lang-xml": "^6.0.2",
"@codemirror/language": "^6.9.3",
"@codemirror/legacy-modes": "^6.3.3",
"@codemirror/lint": "^6.4.2",
"@codemirror/search": "^6.5.5",
"@codemirror/state": "^6.3.3",
@ -453,6 +454,15 @@
"style-mod": "^4.0.0"
}
},
"node_modules/@codemirror/legacy-modes": {
"version": "6.3.3",
"resolved": "https://registry.npmjs.org/@codemirror/legacy-modes/-/legacy-modes-6.3.3.tgz",
"integrity": "sha512-X0Z48odJ0KIoh/HY8Ltz75/4tDYc9msQf1E/2trlxFaFFhgjpVHjZ/BCXe1Lk7s4Gd67LL/CeEEHNI+xHOiESg==",
"dev": true,
"dependencies": {
"@codemirror/language": "^6.0.0"
}
},
"node_modules/@codemirror/lint": {
"version": "6.4.2",
"resolved": "https://registry.npmjs.org/@codemirror/lint/-/lint-6.4.2.tgz",

View File

@ -46,6 +46,7 @@
"@codemirror/lang-sql": "^6.5.4",
"@codemirror/lang-xml": "^6.0.2",
"@codemirror/language": "^6.9.3",
"@codemirror/legacy-modes": "^6.3.3",
"@codemirror/lint": "^6.4.2",
"@codemirror/search": "^6.5.5",
"@codemirror/state": "^6.3.3",

View File

@ -41,7 +41,7 @@ export default defineConfig({
name: 'firefox',
use: { ...devices['Desktop Firefox'] },
},
{
name: 'webkit',
use: { ...devices['Desktop Safari'] },

View File

@ -14,6 +14,12 @@ GUESSLANG_LANGUAGES = [
"rs",
"md",
"cs",
"rb",
"sh",
"yaml",
"go",
"clj",
"erl",
]
const guessLang = new self.GuessLang()

View File

@ -37,9 +37,20 @@
if (event.key === "ArrowDown") {
this.selected = Math.min(this.selected + 1, this.filteredItems.length - 1)
event.preventDefault()
if (this.selected === this.filteredItems.length - 1) {
this.$refs.container.scrollIntoView({block: "end"})
} else {
this.$refs.item[this.selected].scrollIntoView({block: "nearest"})
}
} else if (event.key === "ArrowUp") {
this.selected = Math.max(this.selected - 1, 0)
event.preventDefault()
if (this.selected === 0) {
this.$refs.container.scrollIntoView({block: "start"})
} else {
this.$refs.item[this.selected].scrollIntoView({block: "nearest"})
}
} else if (event.key === "Enter") {
this.selectItem(this.filteredItems[this.selected].token)
event.preventDefault()
@ -69,28 +80,38 @@
</script>
<template>
<form class="language-selector" tabindex="-1" @focusout="onFocusOut" ref="container">
<input
type="text"
ref="input"
@keydown="onKeydown"
@input="onInput"
v-model="filter"
/>
<ul class="items">
<li
v-for="item, idx in filteredItems"
:key="item.token"
:class="idx === selected ? 'selected' : ''"
@click="selectItem(item.token)"
>
{{ item.name }}
</li>
</ul>
</form>
<div class="scroller">
<form class="language-selector" tabindex="-1" @focusout="onFocusOut" ref="container">
<input
type="text"
ref="input"
@keydown="onKeydown"
@input="onInput"
v-model="filter"
/>
<ul class="items">
<li
v-for="item, idx in filteredItems"
:key="item.token"
:class="idx === selected ? 'selected' : ''"
@click="selectItem(item.token)"
ref="item"
>
{{ item.name }}
</li>
</ul>
</form>
</div>
</template>
<style scoped lang="sass">
.scroller
overflow: auto
position: fixed
top: 0
left: 0
bottom: 0
right: 0
.language-selector
font-size: 13px
padding: 10px

View File

@ -1,22 +1,8 @@
import { EditorSelection } from "@codemirror/state"
import * as prettier from "prettier/standalone"
import babelParser from "prettier/plugins/babel.mjs"
import htmlParser from "prettier/esm/parser-html.mjs"
import cssParser from "prettier/esm/parser-postcss.mjs"
import markdownParser from "prettier/esm/parser-markdown.mjs"
import * as prettierPluginEstree from "prettier/plugins/estree.mjs";
import { getActiveNoteBlock } from "./block.js"
const PARSER_MAP = {
"json": {parser:"json-stringify", plugins: [babelParser, prettierPluginEstree]},
"javascript": {parser:"babel", plugins: [babelParser, prettierPluginEstree]},
"html": {parser:"html", plugins: [htmlParser]},
"css": {parser:"css", plugins: [cssParser]},
"markdown": {parser:"markdown", plugins: [markdownParser]},
}
import { getLanguage } from "../languages.js"
export const formatBlockContent = async ({ state, dispatch }) => {
@ -24,9 +10,10 @@ export const formatBlockContent = async ({ state, dispatch }) => {
return false
const block = getActiveNoteBlock(state)
const langName = block.language.name
if (!(langName in PARSER_MAP))
const language = getLanguage(block.language.name)
if (!language.prettier) {
return false
}
// get current cursor position
const cursorPos = state.selection.asSingle().ranges[0].head
@ -49,17 +36,17 @@ export const formatBlockContent = async ({ state, dispatch }) => {
if (useFormat) {
formattedContent = {
formatted: await prettier.format(content, {
parser: PARSER_MAP[langName].parser,
plugins: PARSER_MAP[langName].plugins,
parser: language.prettier.parser,
plugins: language.prettier.plugins,
tabWidth: state.tabSize,
}),
cursorOffset: cursorPos == block.content.from ? 0 : content.length,
}
formattedContent.cursorOffset = cursorPos == block.content.from ? 0 : formattedContent.formatted.length
} else {
formattedContent = await prettier.formatWithCursor(content, {
cursorOffset: cursorPos - block.content.from,
parser: PARSER_MAP[langName].parser,
plugins: PARSER_MAP[langName].plugins,
parser: language.prettier.parser,
plugins: language.prettier.plugins,
tabWidth: state.tabSize,
})
}
@ -75,7 +62,7 @@ export const formatBlockContent = async ({ state, dispatch }) => {
to: block.content.to,
insert: formattedContent.formatted,
},
selection: EditorSelection.cursor(block.content.from + formattedContent.cursorOffset),
selection: EditorSelection.cursor(block.content.from + Math.min(formattedContent.cursorOffset, formattedContent.formatted.length)),
}, {
userEvent: "input",
scrollIntoView: true,

View File

@ -11,7 +11,7 @@ NoteDelimiter {
@tokens {
noteDelimiterMark { "∞∞∞" }
NoteLanguage { "text" | "math" | "javascript" | "json" | "python" | "html" | "sql" | "markdown" | "java" | "php" | "css" | "xml" | "cpp" | "rust" | "csharp" }
NoteLanguage { "text" | "math" | "javascript" | "json" | "python" | "html" | "sql" | "markdown" | "java" | "php" | "css" | "xml" | "cpp" | "rust" | "csharp" | "ruby" | "shell" | "yaml" | "golang" | "clojure" | "erlang" | "lezer" }
Auto { "-a" }
noteDelimiterEnter { "\n" }
//NoteContent { String }

View File

@ -10,7 +10,7 @@ export const parser = LRParser.deserialize({
maxTerm: 10,
skippedNodes: [0],
repeatNodeCount: 1,
tokenData: "'f~R[YZw}!O|#V#W!X#[#]#S#^#_#f#a#b%P#d#e&O#f#g&e#g#h&q#h#i&w#l#m#Y%&x%&y'T~|OX~~!PP#T#U!S~!XOU~~![Q#d#e!b#g#h!m~!eP#d#e!h~!mOT~~!pQ#[#]!v#g#h!h~!yP#T#U!|~#PP#f#g!b~#VP#h#i#Y~#]P#a#b#`~#cP#`#a!h~#iQ#T#U#o#g#h$s~#rP#j#k#u~#xP#T#U#{~$QPT~#g#h$T~$WP#V#W$Z~$^P#f#g$a~$dP#]#^$g~$jP#d#e$m~$pP#h#i!h~$vP#c#d$y~$|P#b#c!h~%SP#T#U%V~%YQ#f#g%`#h#i%x~%cP#_#`%f~%iP#W#X%l~%oP#c#d%r~%uP#k#l$y~%{P#[#]!h~&RQ#[#]!b#m#n&X~&[P#h#i&_~&bP#[#]$s~&hP#i#j&k~&nP#g#h$m~&tP#e#f#`~&zP#X#Y&}~'QP#l#m$m~'WP%&x%&y'Z~'^P%&x%&y'a~'fOY~",
tokenData: "*c~R`YZ!T}!O!Y#V#W!e#X#Y$R#Z#[$q#[#]$w#^#_%Z#`#a&t#a#b'^#d#e(]#f#g(r#g#h)X#h#i)n#l#m$}#m#n)z%&x%&y*Q~!YOX~~!]P#T#U!`~!eOU~~!hR#`#a!q#d#e#f#g#h#l~!tP#c#d!w~!zP#^#_!}~#QP#i#j#T~#WP#f#g#Z~#^P#X#Y#a~#fOT~~#iP#d#e#a~#oQ#[#]#u#g#h#a~#xP#T#U#{~$OP#f#g#f~$UP#f#g$X~$[P#`#a$_~$bP#T#U$e~$hP#b#c$k~$nP#Z#[#a~$tP#c#d$X~$zP#h#i$}~%QP#a#b%T~%WP#`#a#a~%^Q#T#U%d#g#h&h~%gP#j#k%j~%mP#T#U%p~%uPT~#g#h%x~%{P#V#W&O~&RP#f#g&U~&XP#]#^&[~&_P#d#e&b~&eP#h#i#a~&kP#c#d&n~&qP#b#c#a~&wP#X#Y&z~&}P#n#o'Q~'TP#X#Y'W~'ZP#f#g#a~'aP#T#U'd~'gQ#f#g'm#h#i(V~'pP#_#`'s~'vP#W#X'y~'|P#c#d(P~(SP#k#l&n~(YP#[#]#a~(`Q#[#]#f#m#n(f~(iP#h#i(l~(oP#[#]&h~(uP#i#j(x~({Q#U#V)R#g#h&b~)UP#m#n#a~)[Q#[#])b#e#f%T~)eP#X#Y)h~)kP#`#a%T~)qP#X#Y)t~)wP#l#m&b~)}P#T#U$}~*TP%&x%&y*W~*ZP%&x%&y*^~*cOY~",
tokenizers: [0, noteContent],
topRules: {"Document":[0,2]},
tokenPrec: 0

View File

@ -9,6 +9,21 @@ import { LANGUAGE_CHANGE } from "../annotation";
const GUESSLANG_TO_TOKEN = Object.fromEntries(LANGUAGES.map(l => [l.guesslang,l.token]))
function requestIdleCallbackCompat(cb) {
if (window.requestIdleCallback) {
return window.requestIdleCallback(cb)
} else {
return setTimeout(cb, 0)
}
}
function cancelIdleCallbackCompat(id) {
if (window.cancelIdleCallback) {
window.cancelIdleCallback(id)
} else {
clearTimeout(id)
}
}
export function languageDetection(getView) {
const previousBlockContent = {}
@ -45,11 +60,11 @@ export function languageDetection(getView) {
const plugin = EditorView.updateListener.of(update => {
if (update.docChanged) {
if (idleCallbackId !== null) {
cancelIdleCallback(idleCallbackId)
cancelIdleCallbackCompat(idleCallbackId)
idleCallbackId = null
}
idleCallbackId = requestIdleCallback(() => {
idleCallbackId = requestIdleCallbackCompat(() => {
idleCallbackId = null
const range = update.state.selection.asSingle().ranges[0]

View File

@ -13,31 +13,189 @@ import { xmlLanguage } from "@codemirror/lang-xml"
import { rustLanguage } from "@codemirror/lang-rust"
import { csharpLanguage } from "@replit/codemirror-lang-csharp"
import { StreamLanguage } from "@codemirror/language"
import { ruby } from "@codemirror/legacy-modes/mode/ruby"
import { shell } from "@codemirror/legacy-modes/mode/shell"
import { yaml } from "@codemirror/legacy-modes/mode/yaml"
import { go } from "@codemirror/legacy-modes/mode/go"
import { clojure } from "@codemirror/legacy-modes/mode/clojure"
import { erlang } from "@codemirror/legacy-modes/mode/erlang"
import babelPrettierPlugin from "prettier/plugins/babel.mjs"
import htmlPrettierPlugin from "prettier/esm/parser-html.mjs"
import cssPrettierPlugin from "prettier/esm/parser-postcss.mjs"
import markdownPrettierPlugin from "prettier/esm/parser-markdown.mjs"
import yamlPrettierPlugin from "prettier/plugins/yaml.mjs"
import * as prettierPluginEstree from "prettier/plugins/estree.mjs";
class Language {
constructor(token, name, parser, guesslang, supportsFormat = false) {
/**
* @param token: The token used to identify the language in the buffer content
* @param name: The name of the language
* @param parser: The Lezer parser used to parse the language
* @param guesslang: The name of the language as used by the guesslang library
* @param prettier: The prettier configuration for the language (if any)
*/
constructor({ token, name, parser, guesslang, prettier }) {
this.token = token
this.name = name
this.parser = parser
this.guesslang = guesslang
this.supportsFormat = supportsFormat
this.prettier = prettier
}
get supportsFormat() {
return !!this.prettier
}
}
export const LANGUAGES = [
new Language("text", "Plain Text", null, null),
new Language("math", "Math", null, null),
new Language("javascript", "JavaScript", javascriptLanguage.parser, "js", true),
new Language("json", "JSON", jsonLanguage.parser, "json", true),
new Language("python", "Python", pythonLanguage.parser, "py"),
new Language("html", "HTML", htmlLanguage.parser, "html", true),
new Language("sql", "SQL", StandardSQL.language.parser, "sql"),
new Language("markdown", "Markdown", markdownLanguage.parser, "md", true),
new Language("java", "Java", javaLanguage.parser, "java"),
//new Language("lezer", "Lezer", lezerLanguage.parser, "lezer"),
new Language("php", "PHP", phpLanguage.parser, "php"),
new Language("css", "CSS", cssLanguage.parser, "css", true),
new Language("xml", "XML", xmlLanguage.parser, "xml"),
new Language("cpp", "C++", cppLanguage.parser, "cpp"),
new Language("rust", "Rust", rustLanguage.parser, "rust"),
new Language("csharp", "C#", csharpLanguage.parser, "cs"),
new Language({
token: "text",
name: "Plain Text",
parser: null,
guesslang: null,
}),
new Language({
token: "math",
name: "Math",
parser: null,
guesslang: null,
}),
new Language({
token: "json",
name: "JSON",
parser: jsonLanguage.parser,
guesslang: "json",
prettier: {parser:"json-stringify", plugins: [babelPrettierPlugin, prettierPluginEstree]},
}),
new Language({
token: "python",
name: "Python",
parser: pythonLanguage.parser,
guesslang: "py",
}),
new Language({
token: "html",
name: "HTML",
parser: htmlLanguage.parser,
guesslang: "html",
prettier: {parser:"html", plugins: [htmlPrettierPlugin]},
}),
new Language({
token: "sql",
name: "SQL",
parser: StandardSQL.language.parser,
guesslang: "sql",
}),
new Language({
token: "markdown",
name: "Markdown",
parser: markdownLanguage.parser,
guesslang: "md",
prettier: {parser:"markdown", plugins: [markdownPrettierPlugin]},
}),
new Language({
token: "java",
name: "Java",
parser: javaLanguage.parser,
guesslang: "java",
}),
new Language({
token: "lezer",
name: "Lezer",
parser: lezerLanguage.parser,
guesslang: null,
}),
new Language({
token: "php",
name: "PHP",
parser: phpLanguage.parser,
guesslang: "php",
}),
new Language({
token: "css",
name: "CSS",
parser: cssLanguage.parser,
guesslang: "css",
prettier: {parser:"css", plugins: [cssPrettierPlugin]},
}),
new Language({
token: "xml",
name: "XML",
parser: xmlLanguage.parser,
guesslang: "xml",
}),
new Language({
token: "cpp",
name: "C++",
parser: cppLanguage.parser,
guesslang: "cpp",
}),
new Language({
token: "rust",
name: "Rust",
parser: rustLanguage.parser,
guesslang: "rs",
}),
new Language({
token: "csharp",
name: "C#",
parser: csharpLanguage.parser,
guesslang: "cs",
}),
new Language({
token: "ruby",
name: "Ruby",
parser: StreamLanguage.define(ruby).parser,
guesslang: "rb",
}),
new Language({
token: "shell",
name: "Shell",
parser: StreamLanguage.define(shell).parser,
guesslang: "sh",
}),
new Language({
token: "yaml",
name: "YAML",
parser: StreamLanguage.define(yaml).parser,
guesslang: "yaml",
prettier: {parser:"yaml", plugins: [yamlPrettierPlugin]},
}),
new Language({
token: "golang",
name: "Go",
parser: StreamLanguage.define(go).parser,
guesslang: "go",
}),
new Language({
token: "clojure",
name: "Clojure",
parser: StreamLanguage.define(clojure).parser,
guesslang: "clj",
}),
new Language({
token: "erlang",
name: "Erlang",
parser: StreamLanguage.define(erlang).parser,
guesslang: "erl",
}),
new Language({
token: "javascript",
name: "JavaScript",
parser: javascriptLanguage.parser,
guesslang: "js",
prettier: {parser:"babel", plugins: [babelPrettierPlugin, prettierPluginEstree]},
}),
]
const languageMapping = Object.fromEntries(LANGUAGES.map(l => [l.token, l]))
export function getLanguage(token) {
return languageMapping[token]
}

46
tests/formatting.spec.js Normal file
View File

@ -0,0 +1,46 @@
import { test, expect } from "@playwright/test";
import { HeynotePage } from "./test-utils.js";
let heynotePage
test.beforeEach(async ({ page }) => {
console.log("beforeEach")
heynotePage = new HeynotePage(page)
await heynotePage.goto()
})
test("JSON formatting", async ({ page }) => {
heynotePage.setContent(`
json
{"test": 1, "key2": "hey!"}
`)
expect(await page.locator("css=.status .status-block.lang")).toHaveText("JSON")
await page.locator("css=.status-block.format").click()
await page.waitForTimeout(100)
expect(await heynotePage.getBlockContent(0)).toBe(`{
"test": 1,
"key2": "hey!"
}
`)
})
test("JSON formatting (cursor at start)", async ({ page }) => {
heynotePage.setContent(`
json
{"test": 1, "key2": "hey!"}
`)
expect(await page.locator("css=.status .status-block.lang")).toHaveText("JSON")
for (let i=0; i<5; i++) {
await page.locator("body").press("ArrowUp")
}
await page.locator("css=.status-block.format").click()
await page.waitForTimeout(100)
expect(await heynotePage.getBlockContent(0)).toBe(`{
"test": 1,
"key2": "hey!"
}
`)
const block = (await heynotePage.getBlocks())[0]
expect(await page.evaluate(() => window._heynote_editor.view.state.selection.main.from)).toBe(block.content.from)
})

View File

@ -0,0 +1,39 @@
import { test, expect } from "@playwright/test";
import { HeynotePage } from "./test-utils.js";
let heynotePage
test.beforeEach(async ({ page }) => {
console.log("beforeEach")
heynotePage = new HeynotePage(page)
await heynotePage.goto()
})
test("test valid JSON detection", async ({ page }) => {
await page.locator("body").pressSequentially(`
{"test": 1, "key2": "hey!"}
`)
await expect(page.locator("css=.status .status-block.lang")).toHaveText("JSON (auto)")
const block = (await heynotePage.getBlocks())[0]
expect(block.language.name).toBe("json")
expect(block.language.auto).toBeTruthy()
})
test("python detection", async ({ page }) => {
await page.locator("body").pressSequentially(`
# import complex math module
import cmath
# calculate the discriminant
d = (b**2) - (4*a*c)
# find two solutions
sol1 = (-b-cmath.sqrt(d))/(2*a)
sol2 = (-b+cmath.sqrt(d))/(2*a)
print('The solution are {0} and {1}'.format(sol1,sol2))
`)
await expect(page.locator("css=.status .status-block.lang")).toHaveText("Python (auto)")
})

View File

@ -10,24 +10,25 @@ test.beforeEach(async ({ page }) => {
});
test("test markdown mode", async ({ page }) => {
heynotePage.setContent(`
await heynotePage.setContent(`
markdown
# Markdown!
- [ ] todo
- [x] done
`)
await page.waitForTimeout(200)
//await page.locator("body").pressSequentially("test")
expect(await page.locator("css=.status .status-block.lang")).toHaveText("Markdown")
await expect(page.locator("css=.status .status-block.lang")).toHaveText("Markdown")
})
test("checkbox toggle", async ({ page }) => {
heynotePage.setContent(`
await heynotePage.setContent(`
markdown
- [ ] todo
`)
const checkbox = await page.locator("css=.cm-content input[type=checkbox]")
expect(checkbox).toHaveCount(1)
await expect(checkbox).toHaveCount(1)
await checkbox.click()
expect(await heynotePage.getBlockContent(0)).toBe("- [x] todo\n")
await checkbox.click()

View File

@ -29,7 +29,7 @@ export class HeynotePage {
async getContent() {
return await this.page.evaluate(() => window._heynote_editor.getContent())
}
async setContent(content) {
await this.page.evaluate((content) => window._heynote_editor.setContent(content), content)
}