Add wasm support (#2199)

* Working towards a PoC for wasm

* Move bson and sqlite to plugins

* proof of concept now working

* tests are green

* Add CI test for --no-default-features

* Fix some tests

* Fix clippy and windows build

* More fixes

* Fix the windows build

* Fix the windows test

crates/nu_plugin_to_sqlite/Cargo.toml

@@ -0,0 +1,27 @@
+[package]
+authors = ["The Nu Project Contributors"]
+description = "A converter plugin to the bson format for Nushell"
+edition = "2018"
+license = "MIT"
+name = "nu_plugin_to_sqlite"
+version = "0.16.1"
+
+[lib]
+doctest = false
+
+[dependencies]
+hex = "0.4.2"
+nu-errors = {path = "../nu-errors", version = "0.16.1"}
+nu-plugin = {path = "../nu-plugin", version = "0.16.1"}
+nu-protocol = {path = "../nu-protocol", version = "0.16.1"}
+nu-source = {path = "../nu-source", version = "0.16.1"}
+nu-value-ext = {path = "../nu-value-ext", version = "0.16.1"}
+num-traits = "0.2.12"
+tempfile = "3.1.0"
+
+[dependencies.rusqlite]
+features = ["bundled", "blob"]
+version = "0.23.1"
+
+[build-dependencies]
+nu-build = {version = "0.16.1", path = "../nu-build"}

crates/nu_plugin_to_sqlite/build.rs

@@ -0,0 +1,3 @@
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    nu_build::build()
+}

crates/nu_plugin_to_sqlite/src/lib.rs

@@ -0,0 +1,4 @@
+mod nu;
+mod to_sqlite;
+
+pub use to_sqlite::ToSqlite;

crates/nu_plugin_to_sqlite/src/main.rs

@@ -0,0 +1,6 @@
+use nu_plugin::serve_plugin;
+use nu_plugin_to_sqlite::ToSqlite;
+
+fn main() {
+    serve_plugin(&mut ToSqlite::new())
+}

crates/nu_plugin_to_sqlite/src/nu/mod.rs

@@ -0,0 +1,25 @@
+#[cfg(test)]
+mod tests;
+
+use crate::ToSqlite;
+use nu_errors::ShellError;
+use nu_plugin::Plugin;
+use nu_protocol::{ReturnValue, Signature, Value};
+use nu_source::Tag;
+
+impl Plugin for ToSqlite {
+    fn config(&mut self) -> Result<Signature, ShellError> {
+        Ok(Signature::build("to sqlite")
+            .desc("Convert table into sqlite binary")
+            .filter())
+    }
+
+    fn filter(&mut self, input: Value) -> Result<Vec<ReturnValue>, ShellError> {
+        self.state.push(input);
+        Ok(vec![])
+    }
+
+    fn end_filter(&mut self) -> Result<Vec<ReturnValue>, ShellError> {
+        crate::to_sqlite::to_sqlite(self.state.clone(), Tag::unknown())
+    }
+}

crates/nu_plugin_to_sqlite/src/nu/tests.rs

@@ -0,0 +1 @@
+mod integration {}

crates/nu_plugin_to_sqlite/src/to_sqlite.rs

@@ -0,0 +1,163 @@
+use hex::encode;
+use nu_errors::ShellError;
+use nu_protocol::{Dictionary, Primitive, ReturnSuccess, ReturnValue, UntaggedValue, Value};
+use nu_source::Tag;
+use rusqlite::{Connection, NO_PARAMS};
+use std::io::Read;
+
+#[derive(Default)]
+pub struct ToSqlite {
+    pub state: Vec<Value>,
+}
+
+impl ToSqlite {
+    pub fn new() -> ToSqlite {
+        ToSqlite { state: vec![] }
+    }
+}
+fn comma_concat(acc: String, current: String) -> String {
+    if acc == "" {
+        current
+    } else {
+        format!("{}, {}", acc, current)
+    }
+}
+
+fn get_columns(rows: &[Value]) -> Result<String, std::io::Error> {
+    match &rows[0].value {
+        UntaggedValue::Row(d) => Ok(d
+            .entries
+            .iter()
+            .map(|(k, _v)| k.clone())
+            .fold("".to_string(), comma_concat)),
+        _ => Err(std::io::Error::new(
+            std::io::ErrorKind::Other,
+            "Could not find table column names",
+        )),
+    }
+}
+
+fn nu_value_to_sqlite_string(v: Value) -> String {
+    match &v.value {
+        UntaggedValue::Primitive(p) => match p {
+            Primitive::Nothing => "NULL".into(),
+            Primitive::Int(i) => format!("{}", i),
+            Primitive::Duration(i) => format!("{}", i),
+            Primitive::Decimal(f) => format!("{}", f),
+            Primitive::Filesize(u) => format!("{}", u),
+            Primitive::Pattern(s) => format!("'{}'", s.replace("'", "''")),
+            Primitive::String(s) => format!("'{}'", s.replace("'", "''")),
+            Primitive::Line(s) => format!("'{}'", s.replace("'", "''")),
+            Primitive::Boolean(true) => "1".into(),
+            Primitive::Boolean(_) => "0".into(),
+            Primitive::Date(d) => format!("'{}'", d),
+            Primitive::Path(p) => format!("'{}'", p.display().to_string().replace("'", "''")),
+            Primitive::Binary(u) => format!("x'{}'", encode(u)),
+            Primitive::BeginningOfStream
+            | Primitive::EndOfStream
+            | Primitive::ColumnPath(_)
+            | Primitive::Range(_) => "NULL".into(),
+        },
+        _ => "NULL".into(),
+    }
+}
+
+fn get_insert_values(rows: Vec<Value>) -> Result<String, std::io::Error> {
+    let values: Result<Vec<_>, _> = rows
+        .into_iter()
+        .map(|value| match value.value {
+            UntaggedValue::Row(d) => Ok(format!(
+                "({})",
+                d.entries
+                    .iter()
+                    .map(|(_k, v)| nu_value_to_sqlite_string(v.clone()))
+                    .fold("".to_string(), comma_concat)
+            )),
+            _ => Err(std::io::Error::new(
+                std::io::ErrorKind::Other,
+                "Could not find table column names",
+            )),
+        })
+        .collect();
+    let values = values?;
+    Ok(values.into_iter().fold("".to_string(), comma_concat))
+}
+
+fn generate_statements(table: Dictionary) -> Result<(String, String), std::io::Error> {
+    let table_name = match table.entries.get("table_name") {
+        Some(Value {
+            value: UntaggedValue::Primitive(Primitive::String(table_name)),
+            ..
+        }) => table_name,
+        _ => {
+            return Err(std::io::Error::new(
+                std::io::ErrorKind::Other,
+                "Could not find table name",
+            ))
+        }
+    };
+    let (columns, insert_values) = match table.entries.get("table_values") {
+        Some(Value {
+            value: UntaggedValue::Table(l),
+            ..
+        }) => (get_columns(l), get_insert_values(l.to_vec())),
+        _ => {
+            return Err(std::io::Error::new(
+                std::io::ErrorKind::Other,
+                "Could not find table values",
+            ))
+        }
+    };
+    let create = format!("create table {}({})", table_name, columns?);
+    let insert = format!("insert into {} values {}", table_name, insert_values?);
+    Ok((create, insert))
+}
+
+fn sqlite_input_stream_to_bytes(values: Vec<Value>) -> Result<Value, std::io::Error> {
+    // FIXME: should probably write a sqlite virtual filesystem
+    // that will allow us to use bytes as a file to avoid this
+    // write out, but this will require C code. Might be
+    // best done as a PR to rusqlite.
+    let mut tempfile = tempfile::NamedTempFile::new()?;
+    let conn = match Connection::open(tempfile.path()) {
+        Ok(conn) => conn,
+        Err(e) => return Err(std::io::Error::new(std::io::ErrorKind::Other, e)),
+    };
+    let tag = values[0].tag.clone();
+    for value in values.into_iter() {
+        match &value.value {
+            UntaggedValue::Row(d) => {
+                let (create, insert) = generate_statements(d.to_owned())?;
+                match conn
+                    .execute(&create, NO_PARAMS)
+                    .and_then(|_| conn.execute(&insert, NO_PARAMS))
+                {
+                    Ok(_) => (),
+                    Err(e) => {
+                        return Err(std::io::Error::new(std::io::ErrorKind::Other, e));
+                    }
+                }
+            }
+            other => {
+                return Err(std::io::Error::new(
+                    std::io::ErrorKind::Other,
+                    format!("Expected row, found {:?}", other),
+                ))
+            }
+        }
+    }
+    let mut out = Vec::new();
+    tempfile.read_to_end(&mut out)?;
+    Ok(UntaggedValue::binary(out).into_value(tag))
+}
+
+pub fn to_sqlite(input: Vec<Value>, name_tag: Tag) -> Result<Vec<ReturnValue>, ShellError> {
+    match sqlite_input_stream_to_bytes(input) {
+        Ok(out) => Ok(vec![ReturnSuccess::value(out)]),
+        _ => Err(ShellError::labeled_error(
+            "Expected a table with SQLite-compatible structure from pipeline",
+            "requires SQLite-compatible input",
+            name_tag,
+        )),
+    }
+}