Build script codegen initial impl

Cargo.lock

@@ -17,6 +17,21 @@ dependencies = [
  "memchr",
 ]
 
+[[package]]
+name = "android-tzdata"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0"
+
+[[package]]
+name = "android_system_properties"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "ansi_colours"
 version = "1.2.2"
@@ -129,6 +144,7 @@ dependencies = [
  "globset",
  "grep-cli",
  "home",
+ "indexmap 2.1.0",
  "nix",
  "nu-ansi-term",
  "once_cell",
@@ -140,12 +156,14 @@ dependencies = [
  "run_script",
  "semver",
  "serde",
+ "serde_with",
  "serde_yaml",
  "serial_test",
  "shell-words",
  "syntect",
  "tempfile",
  "thiserror",
+ "toml",
  "unicode-width",
  "wait-timeout",
  "walkdir",
@@ -210,6 +228,12 @@ dependencies = [
  "sys-info",
 ]
 
+[[package]]
+name = "bumpalo"
+version = "3.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec"
+
 [[package]]
 name = "bytemuck"
 version = "1.12.1"
@@ -224,11 +248,12 @@ checksum = "a3e368af43e418a04d52505cf3dbc23dda4e3407ae2fa99fd0e4f308ce546acc"
 
 [[package]]
 name = "cc"
-version = "1.0.73"
+version = "1.0.83"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11"
+checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0"
 dependencies = [
  "jobserver",
+ "libc",
 ]
 
 [[package]]
@@ -237,6 +262,19 @@ version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
+[[package]]
+name = "chrono"
+version = "0.4.31"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38"
+dependencies = [
+ "android-tzdata",
+ "iana-time-zone",
+ "num-traits",
+ "serde",
+ "windows-targets 0.48.0",
+]
+
 [[package]]
 name = "clap"
 version = "4.4.10"
@@ -305,6 +343,12 @@ dependencies = [
  "memchr",
 ]
 
+[[package]]
+name = "core-foundation-sys"
+version = "0.8.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa"
+
 [[package]]
 name = "crc32fast"
 version = "1.3.2"
@@ -314,6 +358,41 @@ dependencies = [
  "cfg-if",
 ]
 
+[[package]]
+name = "darling"
+version = "0.20.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0209d94da627ab5605dcccf08bb18afa5009cfbef48d8a8b7d7bdbc79be25c5e"
+dependencies = [
+ "darling_core",
+ "darling_macro",
+]
+
+[[package]]
+name = "darling_core"
+version = "0.20.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "177e3443818124b357d8e76f53be906d60937f0d3a90773a664fa63fa253e621"
+dependencies = [
+ "fnv",
+ "ident_case",
+ "proc-macro2",
+ "quote",
+ "strsim",
+ "syn 2.0.32",
+]
+
+[[package]]
+name = "darling_macro"
+version = "0.20.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5"
+dependencies = [
+ "darling_core",
+ "quote",
+ "syn 2.0.32",
+]
+
 [[package]]
 name = "dashmap"
 version = "5.4.0"
@@ -571,6 +650,12 @@ version = "0.14.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7dfda62a12f55daeae5015f81b0baea145391cb4520f86c248fc615d72640d12"
 
+[[package]]
+name = "hex"
+version = "0.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
+
 [[package]]
 name = "home"
 version = "0.5.5"
@@ -580,6 +665,35 @@ dependencies = [
  "windows-sys 0.48.0",
 ]
 
+[[package]]
+name = "iana-time-zone"
+version = "0.1.58"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8326b86b6cff230b97d0d312a6c40a60726df3332e721f72a1b035f451663b20"
+dependencies = [
+ "android_system_properties",
+ "core-foundation-sys",
+ "iana-time-zone-haiku",
+ "js-sys",
+ "wasm-bindgen",
+ "windows-core",
+]
+
+[[package]]
+name = "iana-time-zone-haiku"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f"
+dependencies = [
+ "cc",
+]
+
+[[package]]
+name = "ident_case"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
+
 [[package]]
 name = "idna"
 version = "0.3.0"
@@ -598,16 +712,18 @@ checksum = "10a35a97730320ffe8e2d410b5d3b69279b98d2c14bdb8b70ea89ecf7888d41e"
 dependencies = [
  "autocfg",
  "hashbrown 0.12.3",
+ "serde",
 ]
 
 [[package]]
 name = "indexmap"
-version = "2.0.2"
+version = "2.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8adf3ddd720272c6ea8bf59463c04e0f93d0bbf7c5439b691bca2987e0270897"
+checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f"
 dependencies = [
  "equivalent",
  "hashbrown 0.14.1",
+ "serde",
 ]
 
 [[package]]
@@ -634,6 +750,15 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "js-sys"
+version = "0.3.65"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "54c0c35952f67de54bb584e9fd912b3023117cbafc0a77d8f3dee1fb5f572fe8"
+dependencies = [
+ "wasm-bindgen",
+]
+
 [[package]]
 name = "lazy_static"
 version = "1.4.0"
@@ -1109,13 +1234,51 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "serde_spanned"
+version = "0.6.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "12022b835073e5b11e90a14f86838ceb1c8fb0325b72416845c487ac0fa95e80"
+dependencies = [
+ "serde",
+]
+
+[[package]]
+name = "serde_with"
+version = "3.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "64cd236ccc1b7a29e7e2739f27c0b2dd199804abc4290e32f59f3b68d6405c23"
+dependencies = [
+ "base64",
+ "chrono",
+ "hex",
+ "indexmap 1.9.1",
+ "indexmap 2.1.0",
+ "serde",
+ "serde_json",
+ "serde_with_macros",
+ "time",
+]
+
+[[package]]
+name = "serde_with_macros"
+version = "3.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "93634eb5f75a2323b16de4748022ac4297f9e76b6dced2be287a099f41b5e788"
+dependencies = [
+ "darling",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.32",
+]
+
 [[package]]
 name = "serde_yaml"
 version = "0.9.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1a49e178e4452f45cb61d0cd8cebc1b0fafd3e41929e996cef79aa3aca91f574"
 dependencies = [
- "indexmap 2.0.2",
+ "indexmap 2.1.0",
  "itoa",
  "ryu",
  "serde",
@@ -1296,6 +1459,7 @@ dependencies = [
  "itoa",
  "libc",
  "num_threads",
+ "serde",
 ]
 
 [[package]]
@@ -1313,6 +1477,41 @@ version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c"
 
+[[package]]
+name = "toml"
+version = "0.8.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8ff9e3abce27ee2c9a37f9ad37238c1bdd4e789c84ba37df76aa4d528f5072cc"
+dependencies = [
+ "indexmap 2.1.0",
+ "serde",
+ "serde_spanned",
+ "toml_datetime",
+ "toml_edit",
+]
+
+[[package]]
+name = "toml_datetime"
+version = "0.6.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3550f4e9685620ac18a50ed434eb3aec30db8ba93b0287467bca5826ea25baf1"
+dependencies = [
+ "serde",
+]
+
+[[package]]
+name = "toml_edit"
+version = "0.20.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "70f427fce4d84c72b5b732388bf4a9f4531b53f74e2887e3ecb2481f68f66d81"
+dependencies = [
+ "indexmap 2.1.0",
+ "serde",
+ "serde_spanned",
+ "toml_datetime",
+ "winnow",
+]
+
 [[package]]
 name = "unicode-bidi"
 version = "0.3.8"
@@ -1394,6 +1593,60 @@ version = "0.11.0+wasi-snapshot-preview1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
 
+[[package]]
+name = "wasm-bindgen"
+version = "0.2.88"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7daec296f25a1bae309c0cd5c29c4b260e510e6d813c286b19eaadf409d40fce"
+dependencies = [
+ "cfg-if",
+ "wasm-bindgen-macro",
+]
+
+[[package]]
+name = "wasm-bindgen-backend"
+version = "0.2.88"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e397f4664c0e4e428e8313a469aaa58310d302159845980fd23b0f22a847f217"
+dependencies = [
+ "bumpalo",
+ "log",
+ "once_cell",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.32",
+ "wasm-bindgen-shared",
+]
+
+[[package]]
+name = "wasm-bindgen-macro"
+version = "0.2.88"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5961017b3b08ad5f3fe39f1e79877f8ee7c23c5e5fd5eb80de95abc41f1f16b2"
+dependencies = [
+ "quote",
+ "wasm-bindgen-macro-support",
+]
+
+[[package]]
+name = "wasm-bindgen-macro-support"
+version = "0.2.88"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c5353b8dab669f5e10f5bd76df26a9360c748f054f862ff5f3f8aae0c7fb3907"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.32",
+ "wasm-bindgen-backend",
+ "wasm-bindgen-shared",
+]
+
+[[package]]
+name = "wasm-bindgen-shared"
+version = "0.2.88"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0d046c5d029ba91a1ed14da14dca44b68bf2f124cfbaf741c54151fdb3e0750b"
+
 [[package]]
 name = "wild"
 version = "2.2.0"
@@ -1434,6 +1687,15 @@ version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
 
+[[package]]
+name = "windows-core"
+version = "0.50.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "af6041b3f84485c21b57acdc0fee4f4f0c93f426053dc05fa5d6fc262537bbff"
+dependencies = [
+ "windows-targets 0.48.0",
+]
+
 [[package]]
 name = "windows-sys"
 version = "0.45.0"
@@ -1566,6 +1828,15 @@ version = "0.48.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a"
 
+[[package]]
+name = "winnow"
+version = "0.5.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "176b6138793677221d420fd2f0aeeced263f197688b36484660da767bca2fa32"
+dependencies = [
+ "memchr",
+]
+
 [[package]]
 name = "yaml-rust"
 version = "0.4.5"

Cargo.toml

@@ -100,6 +100,11 @@ nix = { version = "0.26.4", default-features = false, features = ["term"] }
 
 [build-dependencies]
 anyhow = "1.0.75"
+indexmap = { version = "2.1.0", features = ["serde"] }
+serde = { version = "1.0", features = ["derive"] }
+serde_with = "3.4.0"
+toml = { version = "0.8.6", features = ["preserve_order"] }
+walkdir = "2.4"
 
 [build-dependencies.clap]
 version = "4.4.10"

build/main.rs

@@ -1,5 +1,6 @@
 #[cfg(feature = "application")]
 mod application;
+mod syntax_mapping;
 mod util;
 
 fn main() -> anyhow::Result<()> {
@@ -7,6 +8,8 @@ fn main() -> anyhow::Result<()> {
     // see: https://doc.rust-lang.org/cargo/reference/build-scripts.html#rerun-if-changed
     println!("cargo:rerun-if-changed=build/");
 
+    syntax_mapping::build_static_mappings()?;
+
     #[cfg(feature = "application")]
     application::gen_man_and_comp()?;
 

build/syntax_mapping.rs

@@ -0,0 +1,106 @@
+use std::{convert::Infallible, env, fs, path::Path, str::FromStr};
+
+use anyhow::anyhow;
+use indexmap::IndexMap;
+use serde::Deserialize;
+use serde_with::DeserializeFromStr;
+use walkdir::WalkDir;
+
+/// Known mapping targets.
+///
+/// Corresponds to `syntax_mapping::MappingTarget`.
+#[derive(Clone, Debug, Eq, PartialEq, Hash, DeserializeFromStr)]
+pub enum MappingTarget {
+    MapTo(String),
+    MapToUnknown,
+    MapExtensionToUnknown,
+}
+impl FromStr for MappingTarget {
+    type Err = Infallible;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        match s {
+            "MappingTarget::MapToUnknown" => Ok(Self::MapToUnknown),
+            "MappingTarget::MapExtensionToUnknown" => Ok(Self::MapExtensionToUnknown),
+            syntax => Ok(Self::MapTo(syntax.into())),
+        }
+    }
+}
+impl MappingTarget {
+    fn codegen(&self) -> String {
+        match self {
+            Self::MapTo(syntax) => format!(r###"MappingTarget::MapTo(r#"{syntax}"#)"###),
+            Self::MapToUnknown => "MappingTarget::MapToUnknown".into(),
+            Self::MapExtensionToUnknown => "MappingTarget::MapExtensionToUnknown".into(),
+        }
+    }
+}
+
+/// A struct that models a single .toml file in /src/syntax_mapping/builtins/.
+#[derive(Clone, Debug, Deserialize)]
+struct MappingDefModel {
+    mappings: IndexMap<MappingTarget, Vec<String>>,
+}
+impl MappingDefModel {
+    fn into_mapping_list(self) -> Vec<(String, MappingTarget)> {
+        self.mappings
+            .into_iter()
+            .flat_map(|(target, matcher)| {
+                matcher
+                    .into_iter()
+                    .map(|rule| (rule, target.clone()))
+                    .collect::<Vec<_>>()
+            })
+            .collect()
+    }
+}
+
+#[derive(Clone, Debug)]
+struct MappingList(Vec<(String, MappingTarget)>);
+impl MappingList {
+    fn codegen(&self) -> String {
+        let array_items: Vec<_> = self
+            .0
+            .iter()
+            .map(|(matcher, target)| format!(r###"(r#"{matcher}"#, {t})"###, t = target.codegen()))
+            .collect();
+        let len = array_items.len();
+
+        format!(
+            "static STATIC_RULES: [(&'static str, &'static str); {len}] = [\n{items}\n];",
+            items = array_items.join(",\n")
+        )
+    }
+}
+
+fn read_all_mappings() -> anyhow::Result<MappingList> {
+    let mut all_mappings = vec![];
+
+    for entry in WalkDir::new("src/syntax_mapping/builtins")
+        .into_iter()
+        .map(|entry| entry.unwrap_or_else(|err| panic!("failed to visit a file: {err}")))
+        .filter(|entry| {
+            let path = entry.path();
+            path.is_file() && path.extension().map(|ext| ext == "toml").unwrap_or(false)
+        })
+    {
+        let toml_string = fs::read_to_string(entry.path())?;
+        let mappings = toml::from_str::<MappingDefModel>(&toml_string)?.into_mapping_list();
+        all_mappings.extend(mappings);
+    }
+
+    Ok(MappingList(all_mappings))
+}
+
+/// Build the static syntax mappings defined in /src/syntax_mapping/builtins/
+/// into a .rs source file, which is to be inserted with `include!`.
+pub fn build_static_mappings() -> anyhow::Result<()> {
+    let mappings = read_all_mappings()?;
+
+    let codegen_path = Path::new(&env::var_os("OUT_DIR").ok_or(anyhow!("OUT_DIR is unset"))?)
+        .join("codegen_static_syntax_mappings.rs");
+
+    fs::write(codegen_path, mappings.codegen())?;
+
+    Ok(())
+}