Merge pull request #545 from pmeredit/topic/sqlite

Add SQLite support

Cargo.lock

@@ -715,6 +715,16 @@ dependencies = [
  "synstructure 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "fallible-iterator"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "fallible-streaming-iterator"
+version = "0.1.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [[package]]
 name = "fixedbitset"
 version = "0.1.9"
@@ -1308,6 +1318,16 @@ dependencies = [
  "libc 0.2.60 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "libsqlite3-sys"
+version = "0.16.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "cc 1.0.38 (registry+https://github.com/rust-lang/crates.io-index)",
+ "pkg-config 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)",
+ "vcpkg 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "libz-sys"
 version = "1.0.25"
@@ -1358,6 +1378,14 @@ dependencies = [
  "cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "lru-cache"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "linked-hash-map 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "lzw"
 version = "0.10.0"
@@ -1576,6 +1604,7 @@ dependencies = [
  "git2 0.9.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "glob 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "heim 0.0.7 (registry+https://github.com/rust-lang/crates.io-index)",
+ "hex 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "image 0.22.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "indexmap 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "itertools 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -1596,6 +1625,7 @@ dependencies = [
  "rawkey 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "regex 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "roxmltree 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rusqlite 0.20.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "rustyline 5.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde 1.0.98 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -2244,6 +2274,20 @@ dependencies = [
  "xmlparser 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "rusqlite"
+version = "0.20.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "bitflags 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "fallible-iterator 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "fallible-streaming-iterator 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libsqlite3-sys 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "lru-cache 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "memchr 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "rust-argon2"
 version = "0.5.0"
@@ -3183,6 +3227,8 @@ dependencies = [
 "checksum env_logger 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "aafcde04e90a5226a6443b7aabdb016ba2f8307c847d524724bd9b346dd1a2d3"
 "checksum failure 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "795bd83d3abeb9220f257e597aa0080a508b27533824adf336529648f6abf7e2"
 "checksum failure_derive 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "ea1063915fd7ef4309e222a5a07cf9c319fb9c7836b1f89b85458672dbb127e1"
+"checksum fallible-iterator 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7"
+"checksum fallible-streaming-iterator 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a"
 "checksum fixedbitset 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "86d4de0081402f5e88cdac65c8dcdcc73118c1a7a465e2a05f0da05843a8ea33"
 "checksum flate2 1.0.9 (registry+https://github.com/rust-lang/crates.io-index)" = "550934ad4808d5d39365e5d61727309bf18b3b02c6c56b729cb92e7dd84bc3d8"
 "checksum fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "2fad85553e09a6f881f739c29f0b00b0f01357c743266d478b68951ce23285f3"
@@ -3242,12 +3288,14 @@ dependencies = [
 "checksum libc 0.2.60 (registry+https://github.com/rust-lang/crates.io-index)" = "d44e80633f007889c7eff624b709ab43c92d708caad982295768a7b13ca3b5eb"
 "checksum libgit2-sys 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)" = "4c179ed6d19cd3a051e68c177fbbc214e79ac4724fac3a850ec9f3d3eb8a5578"
 "checksum libnghttp2-sys 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "02254d44f4435dd79e695f2c2b83cd06a47919adea30216ceaf0c57ca0a72463"
+"checksum libsqlite3-sys 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5e5b95e89c330291768dc840238db7f9e204fd208511ab6319b56193a7f2ae25"
 "checksum libz-sys 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)" = "2eb5e43362e38e2bca2fd5f5134c4d4564a23a5c28e9b95411652021a8675ebe"
 "checksum line-wrap 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f30344350a2a51da54c1d53be93fade8a237e545dbcc4bdbe635413f2117cab9"
 "checksum linked-hash-map 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6d262045c5b87c0861b3f004610afd0e2c851e2908d08b6c870cbb9d5f494ecd"
 "checksum linked-hash-map 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "ae91b68aebc4ddb91978b11a1b02ddd8602a05ec19002801c5666000e05e0f83"
 "checksum lock_api 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "62ebf1391f6acad60e5c8b43706dde4582df75c06698ab44511d15016bc2442c"
 "checksum log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)" = "14b6052be84e6b71ab17edffc2eeabf5c2c3ae1fdb464aae35ac50c67a44e1f7"
+"checksum lru-cache 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "31e24f1ad8321ca0e8a1e0ac13f23cb668e6f5466c2c57319f6a5cf1cc8e3b1c"
 "checksum lzw 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7d947cbb889ed21c2a84be6ffbaebf5b4e0f4340638cba0444907e38b56be084"
 "checksum macaddr 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3ff4752cb15cffb3e68f7dcb22e0818ac871f8c98fb07a634a81f41fb202a09f"
 "checksum mach 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "86dd2487cdfea56def77b88438a2c915fb45113c5319bfe7e14306ca4cd0b0e1"
@@ -3342,6 +3390,7 @@ dependencies = [
 "checksum render-tree 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "68ed587df09cfb7ce1bc6fe8f77e24db219f222c049326ccbfb948ec67e31664"
 "checksum result 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "194d8e591e405d1eecf28819740abed6d719d1a2db87fc0bcdedee9a26d55560"
 "checksum roxmltree 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "153c367ce9fb8ef7afe637ef92bd083ba0f88b03ef3fcf0287d40be05ae0a61c"
+"checksum rusqlite 0.20.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2a194373ef527035645a1bc21b10dc2125f73497e6e155771233eb187aedd051"
 "checksum rust-argon2 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "81ed8d04228b44a740c8d46ff872a28e50fff3d659f307ab4da2cc502e019ff3"
 "checksum rust-ini 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3e52c148ef37f8c375d49d5a73aa70713125b7f19095948a923f80afdeb22ec2"
 "checksum rustc-demangle 0.1.15 (registry+https://github.com/rust-lang/crates.io-index)" = "a7f4dccf6f4891ebcc0c39f9b6eb1a83b9bf5d747cb439ec6fba4f3b977038af"

Cargo.toml

@@ -62,6 +62,7 @@ mime = "0.3.13"
 regex = "1.2.1"
 pretty-hex = "0.1.0"
 neso = { version = "0.5.0", optional = true }
+hex = "0.3.2"
 crossterm = "0.10.2"
 tempfile = "3.1.0"
 image = "0.22.1"
@@ -82,6 +83,10 @@ pin-utils = "0.1.0-alpha.4"
 [features]
 raw-key = ["rawkey", "neso"]
 
+[dependencies.rusqlite]
+version = "0.20.0"
+features = ["bundled", "blob"]
+
 [dev-dependencies]
 pretty_assertions = "0.6.1"
 

src/cli.rs

@@ -179,6 +179,7 @@ pub async fn cli() -> Result<(), Box<dyn Error>> {
             whole_stream_command(ToBSON),
             whole_stream_command(ToCSV),
             whole_stream_command(ToJSON),
+            whole_stream_command(ToSQLite),
             whole_stream_command(ToTOML),
             whole_stream_command(ToTSV),
             whole_stream_command(ToYAML),
@@ -193,6 +194,7 @@ pub async fn cli() -> Result<(), Box<dyn Error>> {
             whole_stream_command(FromINI),
             whole_stream_command(FromBSON),
             whole_stream_command(FromJSON),
+            whole_stream_command(FromSQLite),
             whole_stream_command(FromTOML),
             whole_stream_command(FromXML),
             whole_stream_command(FromYAML),

src/commands.rs

@@ -19,6 +19,7 @@ pub(crate) mod from_bson;
 pub(crate) mod from_csv;
 pub(crate) mod from_ini;
 pub(crate) mod from_json;
+pub(crate) mod from_sqlite;
 pub(crate) mod from_toml;
 pub(crate) mod from_tsv;
 pub(crate) mod from_xml;
@@ -52,6 +53,7 @@ pub(crate) mod to_array;
 pub(crate) mod to_bson;
 pub(crate) mod to_csv;
 pub(crate) mod to_json;
+pub(crate) mod to_sqlite;
 pub(crate) mod to_toml;
 pub(crate) mod to_tsv;
 pub(crate) mod to_yaml;
@@ -67,6 +69,7 @@ pub(crate) use command::{
     per_item_command, whole_stream_command, Command, PerItemCommand, RawCommandArgs,
     UnevaluatedCallInfo, WholeStreamCommand,
 };
+
 pub(crate) use config::Config;
 pub(crate) use cp::Cpy;
 pub(crate) use date::Date;
@@ -79,6 +82,7 @@ pub(crate) use from_bson::FromBSON;
 pub(crate) use from_csv::FromCSV;
 pub(crate) use from_ini::FromINI;
 pub(crate) use from_json::FromJSON;
+pub(crate) use from_sqlite::FromSQLite;
 pub(crate) use from_toml::FromTOML;
 pub(crate) use from_tsv::FromTSV;
 pub(crate) use from_xml::FromXML;
@@ -111,6 +115,7 @@ pub(crate) use to_array::ToArray;
 pub(crate) use to_bson::ToBSON;
 pub(crate) use to_csv::ToCSV;
 pub(crate) use to_json::ToJSON;
+pub(crate) use to_sqlite::ToSQLite;
 pub(crate) use to_toml::ToTOML;
 pub(crate) use to_tsv::ToTSV;
 pub(crate) use to_yaml::ToYAML;

src/commands/from_bson.rs

@@ -45,7 +45,7 @@ fn convert_bson_value_to_nu_value(v: &Bson, tag: impl Into<Tag>) -> Tagged<Value
             collected.into_tagged_value()
         }
         Bson::Boolean(b) => Value::Primitive(Primitive::Boolean(*b)).tagged(tag),
-        Bson::Null => Value::Primitive(Primitive::String(String::from(""))).tagged(tag),
+        Bson::Null => Value::Primitive(Primitive::Nothing).tagged(tag),
         Bson::RegExp(r, opts) => {
             let mut collected = TaggedDictBuilder::new(tag);
             collected.insert_tagged(

src/commands/from_sqlite.rs

@@ -0,0 +1,139 @@
+use crate::commands::WholeStreamCommand;
+use crate::errors::ShellError;
+use crate::object::base::OF64;
+use crate::object::{Primitive, TaggedDictBuilder, Value};
+use crate::prelude::*;
+use rusqlite::{types::ValueRef, Connection, Row, NO_PARAMS};
+use std::io::Write;
+use std::path::Path;
+
+pub struct FromSQLite;
+
+impl WholeStreamCommand for FromSQLite {
+    fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        from_sqlite(args, registry)
+    }
+
+    fn name(&self) -> &str {
+        "from-sqlite"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("from-sqlite")
+    }
+}
+
+pub fn convert_sqlite_file_to_nu_value(
+    path: &Path,
+    tag: impl Into<Tag> + Clone,
+) -> Result<Tagged<Value>, rusqlite::Error> {
+    let conn = Connection::open(path)?;
+
+    let mut meta_out = Vec::new();
+    let mut meta_stmt = conn.prepare("select name from sqlite_master where type='table'")?;
+    let mut meta_rows = meta_stmt.query(NO_PARAMS)?;
+    while let Some(meta_row) = meta_rows.next()? {
+        let table_name: String = meta_row.get(0)?;
+        let mut meta_dict = TaggedDictBuilder::new(tag.clone());
+        let mut out = Vec::new();
+        let mut table_stmt = conn.prepare(&format!("select * from [{}]", table_name))?;
+        let mut table_rows = table_stmt.query(NO_PARAMS)?;
+        while let Some(table_row) = table_rows.next()? {
+            out.push(convert_sqlite_row_to_nu_value(table_row, tag.clone())?)
+        }
+        meta_dict.insert_tagged(
+            "table_name".to_string(),
+            Value::Primitive(Primitive::String(table_name)).tagged(tag.clone()),
+        );
+        meta_dict.insert_tagged("table_values", Value::List(out).tagged(tag.clone()));
+        meta_out.push(meta_dict.into_tagged_value());
+    }
+    let tag = tag.into();
+    Ok(Value::List(meta_out).tagged(tag))
+}
+
+fn convert_sqlite_row_to_nu_value(
+    row: &Row,
+    tag: impl Into<Tag> + Clone,
+) -> Result<Tagged<Value>, rusqlite::Error> {
+    let mut collected = TaggedDictBuilder::new(tag.clone());
+    for (i, c) in row.columns().iter().enumerate() {
+        collected.insert_tagged(
+            c.name().to_string(),
+            convert_sqlite_value_to_nu_value(row.get_raw(i), tag.clone()),
+        );
+    }
+    return Ok(collected.into_tagged_value());
+}
+
+fn convert_sqlite_value_to_nu_value(value: ValueRef, tag: impl Into<Tag> + Clone) -> Tagged<Value> {
+    match value {
+        ValueRef::Null => Value::Primitive(Primitive::String(String::from(""))).tagged(tag),
+        ValueRef::Integer(i) => Value::Primitive(Primitive::Int(i)).tagged(tag),
+        ValueRef::Real(f) => Value::Primitive(Primitive::Float(OF64::from(f))).tagged(tag),
+        t @ ValueRef::Text(_) => {
+            // this unwrap is safe because we know the ValueRef is Text.
+            Value::Primitive(Primitive::String(t.as_str().unwrap().to_string())).tagged(tag)
+        }
+        ValueRef::Blob(u) => Value::Binary(u.to_owned()).tagged(tag),
+    }
+}
+
+pub fn from_sqlite_bytes_to_value(
+    mut bytes: Vec<u8>,
+    tag: impl Into<Tag> + Clone,
+) -> Result<Tagged<Value>, std::io::Error> {
+    // FIXME: should probably write a sqlite virtual filesystem
+    // that will allow us to use bytes as a file to avoid this
+    // write out, but this will require C code. Might be
+    // best done as a PR to rusqlite.
+    let mut tempfile = tempfile::NamedTempFile::new()?;
+    tempfile.write_all(bytes.as_mut_slice())?;
+    match convert_sqlite_file_to_nu_value(tempfile.path(), tag) {
+        Ok(value) => Ok(value),
+        Err(e) => Err(std::io::Error::new(std::io::ErrorKind::Other, e)),
+    }
+}
+
+fn from_sqlite(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
+    let args = args.evaluate_once(registry)?;
+    let span = args.name_span();
+    let input = args.input;
+
+    let stream = async_stream_block! {
+        let values: Vec<Tagged<Value>> = input.values.collect().await;
+
+        for value in values {
+            let value_tag = value.tag();
+            match value.item {
+                Value::Binary(vb) =>
+                    match from_sqlite_bytes_to_value(vb, span) {
+                        Ok(x) => yield ReturnSuccess::value(x),
+                        Err(_) => {
+                            yield Err(ShellError::labeled_error_with_secondary(
+                                "Could not parse as SQLite",
+                                "input cannot be parsed as SQLite",
+                                span,
+                                "value originates from here",
+                                value_tag.span,
+                            ))
+                        }
+                    }
+                _ => yield Err(ShellError::labeled_error_with_secondary(
+                    "Expected a string from pipeline",
+                    "requires string input",
+                    span,
+                    "value originates from here",
+                    value_tag.span,
+                )),
+
+            }
+        }
+    };
+
+    Ok(stream.to_output_stream())
+}

src/commands/open.rs

@@ -527,6 +527,16 @@ pub fn parse_binary_as_value(
                 },
             )
         }
+        Some(ref x) if x == "db" => {
+            crate::commands::from_sqlite::from_sqlite_bytes_to_value(contents, contents_tag)
+                .map_err(move |_| {
+                    ShellError::labeled_error(
+                        "Could not open as SQLite",
+                        "could not open as SQLite",
+                        name_span,
+                    )
+                })
+        }
         _ => Ok(Value::Binary(contents).tagged(contents_tag)),
     }
 }

src/commands/to_sqlite.rs

@@ -0,0 +1,183 @@
+use crate::commands::WholeStreamCommand;
+use crate::object::{Dictionary, Primitive, Value};
+use crate::prelude::*;
+use hex::encode;
+use rusqlite::{Connection, NO_PARAMS};
+use std::io::Read;
+
+pub struct ToSQLite;
+
+impl WholeStreamCommand for ToSQLite {
+    fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        to_sqlite(args, registry)
+    }
+
+    fn name(&self) -> &str {
+        "to-sqlite"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("to-sqlite")
+    }
+}
+
+fn comma_concat(acc: String, current: String) -> String {
+    if acc == "" {
+        current
+    } else {
+        format!("{}, {}", acc, current)
+    }
+}
+
+fn get_columns(rows: &Vec<Tagged<Value>>) -> Result<String, std::io::Error> {
+    match &rows[0].item {
+        Value::Object(d) => Ok(d
+            .entries
+            .iter()
+            .map(|(k, _v)| k.clone())
+            .fold("".to_string(), comma_concat)),
+        _ => Err(std::io::Error::new(
+            std::io::ErrorKind::Other,
+            "Could not find table column names",
+        )),
+    }
+}
+
+fn nu_value_to_sqlite_string(v: Value) -> String {
+    match v {
+        Value::Binary(u) => format!("x'{}'", encode(u)),
+        Value::Primitive(p) => match p {
+            Primitive::Nothing => "NULL".into(),
+            Primitive::Int(i) => format!("{}", i),
+            Primitive::Float(f) => format!("{}", f.into_inner()),
+            Primitive::Bytes(u) => format!("{}", u),
+            Primitive::String(s) => format!("'{}'", s.replace("'", "''")),
+            Primitive::Boolean(true) => "1".into(),
+            Primitive::Boolean(_) => "0".into(),
+            Primitive::Date(d) => format!("'{}'", d),
+            Primitive::Path(p) => format!("'{}'", p.display().to_string().replace("'", "''")),
+            Primitive::BeginningOfStream => "NULL".into(),
+            Primitive::EndOfStream => "NULL".into(),
+        },
+        _ => "NULL".into(),
+    }
+}
+
+fn get_insert_values(rows: Vec<Tagged<Value>>) -> Result<String, std::io::Error> {
+    let values: Result<Vec<_>, _> = rows
+        .into_iter()
+        .map(|value| match value.item {
+            Value::Object(d) => Ok(format!(
+                "({})",
+                d.entries
+                    .iter()
+                    .map(|(_k, v)| nu_value_to_sqlite_string(v.item.clone()))
+                    .fold("".to_string(), comma_concat)
+            )),
+            _ => Err(std::io::Error::new(
+                std::io::ErrorKind::Other,
+                "Could not find table column names",
+            )),
+        })
+        .collect();
+    let values = values?;
+    Ok(values.into_iter().fold("".to_string(), comma_concat))
+}
+
+fn generate_statements(table: Dictionary) -> Result<(String, String), std::io::Error> {
+    let table_name = match table.entries.get("table_name") {
+        Some(Tagged {
+            item: Value::Primitive(Primitive::String(table_name)),
+            ..
+        }) => table_name,
+        _ => {
+            return Err(std::io::Error::new(
+                std::io::ErrorKind::Other,
+                "Could not find table name",
+            ))
+        }
+    };
+    let (columns, insert_values) = match table.entries.get("table_values") {
+        Some(Tagged {
+            item: Value::List(l),
+            ..
+        }) => (get_columns(l), get_insert_values(l.to_vec())),
+        _ => {
+            return Err(std::io::Error::new(
+                std::io::ErrorKind::Other,
+                "Could not find table values",
+            ))
+        }
+    };
+    let create = format!("create table {}({})", table_name, columns?);
+    let insert = format!("insert into {} values {}", table_name, insert_values?);
+    Ok((create, insert))
+}
+
+fn sqlite_input_stream_to_bytes(
+    values: Vec<Tagged<Value>>,
+) -> Result<Tagged<Value>, std::io::Error> {
+    // FIXME: should probably write a sqlite virtual filesystem
+    // that will allow us to use bytes as a file to avoid this
+    // write out, but this will require C code. Might be
+    // best done as a PR to rusqlite.
+    let mut tempfile = tempfile::NamedTempFile::new()?;
+    let conn = match Connection::open(tempfile.path()) {
+        Ok(conn) => conn,
+        Err(e) => return Err(std::io::Error::new(std::io::ErrorKind::Other, e)),
+    };
+    let tag = values[0].tag.clone();
+    for value in values.into_iter() {
+        match value.item() {
+            Value::Object(d) => {
+                let (create, insert) = generate_statements(d.to_owned())?;
+                match conn
+                    .execute(&create, NO_PARAMS)
+                    .and_then(|_| conn.execute(&insert, NO_PARAMS))
+                {
+                    Ok(_) => (),
+                    Err(e) => {
+                        println!("{}", create);
+                        println!("{}", insert);
+                        println!("{:?}", e);
+                        return Err(std::io::Error::new(std::io::ErrorKind::Other, e));
+                    }
+                }
+            }
+            other => {
+                return Err(std::io::Error::new(
+                    std::io::ErrorKind::Other,
+                    format!("Expected object, found {:?}", other),
+                ))
+            }
+        }
+    }
+    let mut out = Vec::new();
+    tempfile.read_to_end(&mut out)?;
+    Ok(Value::Binary(out).tagged(tag))
+}
+
+fn to_sqlite(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
+    let args = args.evaluate_once(registry)?;
+    let name_span = args.name_span();
+    let stream = async_stream_block! {
+        let values: Vec<_> = args.input.into_vec().await;
+        match sqlite_input_stream_to_bytes(values) {
+            Ok(out) => {
+                yield ReturnSuccess::value(out)
+            }
+            Err(_) => {
+                yield Err(ShellError::labeled_error(
+                    "Expected an object with SQLite-compatible structure from pipeline",
+                    "requires SQLite-compatible input",
+                    name_span,
+                    ))
+            }
+        };
+    };
+    Ok(stream.to_output_stream())
+}

src/utils.rs

@@ -266,6 +266,10 @@ mod tests {
                     loc: fixtures().join("sample.bson"),
                     at: 0
                 },
+                Res {
+                    loc: fixtures().join("sample.db"),
+                    at: 0
+                },
                 Res {
                     loc: fixtures().join("sample.ini"),
                     at: 0

tests/command_open_tests.rs

@@ -6,23 +6,22 @@ use helpers::{Playground, Stub::*};
 #[test]
 fn recognizes_csv() {
     Playground::setup("open_test_1", |dirs, sandbox| {
-        sandbox
+        sandbox.with_files(vec![FileWithContentToBeTrimmed(
-            .with_files(vec![FileWithContentToBeTrimmed(
+            "nu.zion.csv",
-                "nu.zion.csv",
+            r#"
-                r#"
                     author,lang,source
                     Jonathan Turner,Rust,New Zealand
                     Andres N. Robalino,Rust,Ecuador
                     Yehuda Katz,Rust,Estados Unidos
-                "#
+                "#,
         )]);
 
         let actual = nu!(
             cwd: dirs.test(), h::pipeline(
             r#"
-                open nu.zion.csv 
+                open nu.zion.csv
-                | where author == "Andres N. Robalino" 
+                | where author == "Andres N. Robalino"
-                | get source 
+                | get source
                 | echo $it
             "#
         ));
@@ -46,11 +45,11 @@ fn open_can_parse_bson_2() {
     let actual = nu!(
         cwd: "tests/fixtures/formats", h::pipeline(
         r#"
-            open sample.bson 
+            open sample.bson
-            | get root 
+            | get root
-            | nth 6 
+            | nth 6
-            | get b 
+            | get b
-            | get '$binary_subtype' 
+            | get '$binary_subtype'
             | echo $it
         "#
     ));
@@ -58,6 +57,21 @@ fn open_can_parse_bson_2() {
     assert_eq!(actual, "function");
 }
 
+#[test]
+fn open_can_parse_sqlite() {
+    let actual = nu!(
+        cwd: "tests/fixtures/formats", h::pipeline(
+        r#"
+            open sample.db
+            | get table_values
+            | nth 2
+            | get x
+            | echo $it"#
+    ));
+
+    assert_eq!(actual, "hello");
+}
+
 #[test]
 fn open_can_parse_toml() {
     let actual = nu!(
@@ -74,8 +88,8 @@ fn open_can_parse_tsv() {
         cwd: "tests/fixtures/formats", h::pipeline(
         r#"
             open caco3_plastics.tsv
-            | first 1 
+            | first 1
-            | get origin 
+            | get origin
             | echo $it
         "#
     ));
@@ -88,8 +102,8 @@ fn open_can_parse_json() {
     let actual = nu!(
         cwd: "tests/fixtures/formats", h::pipeline(
         r#"
-            open sgml_description.json 
+            open sgml_description.json
-            | get glossary.GlossDiv.GlossList.GlossEntry.GlossSee 
+            | get glossary.GlossDiv.GlossList.GlossEntry.GlossSee
             | echo $it
         "#
     ));

tests/filters_test.rs

@@ -16,14 +16,13 @@ fn can_convert_table_to_csv_text_and_from_csv_text_back_into_table() {
 #[test]
 fn converts_structured_table_to_csv_text() {
     Playground::setup("filter_to_csv_test_1", |dirs, sandbox| {
-        sandbox
+        sandbox.with_files(vec![FileWithContentToBeTrimmed(
-            .with_files(vec![FileWithContentToBeTrimmed(
+            "csv_text_sample.txt",
-                "csv_text_sample.txt",
+            r#"
-                r#"
                     importer,shipper,tariff_item,name,origin
                     Plasticos Rival,Reverte,2509000000,Calcium carbonate,Spain
                     Tigre Ecuador,OMYA Andina,3824909999,Calcium carbonate,Colombia
-                "#
+                "#,
         )]);
 
         let actual = nu!(
@@ -47,14 +46,13 @@ fn converts_structured_table_to_csv_text() {
 #[test]
 fn converts_structured_table_to_csv_text_skipping_headers_after_conversion() {
     Playground::setup("filter_to_csv_test_2", |dirs, sandbox| {
-        sandbox
+        sandbox.with_files(vec![FileWithContentToBeTrimmed(
-            .with_files(vec![FileWithContentToBeTrimmed(
+            "csv_text_sample.txt",
-                "csv_text_sample.txt",
+            r#"
-                r#"
                     importer,shipper,tariff_item,name,origin
                     Plasticos Rival,Reverte,2509000000,Calcium carbonate,Spain
                     Tigre Ecuador,OMYA Andina,3824909999,Calcium carbonate,Colombia
-                "#
+                "#,
         )]);
 
         let actual = nu!(
@@ -76,10 +74,9 @@ fn converts_structured_table_to_csv_text_skipping_headers_after_conversion() {
 #[test]
 fn converts_from_csv_text_to_structured_table() {
     Playground::setup("filter_from_csv_test_1", |dirs, sandbox| {
-        sandbox
+        sandbox.with_files(vec![FileWithContentToBeTrimmed(
-            .with_files(vec![FileWithContentToBeTrimmed(
+            "los_tres_amigos.txt",
-                "los_tres_amigos.txt",
+            r#"
-                r#"
                     first_name,last_name,rusty_luck
                     Andrés,Robalino,1
                     Jonathan,Turner,1
@@ -106,10 +103,9 @@ fn converts_from_csv_text_to_structured_table() {
 #[test]
 fn converts_from_csv_text_skipping_headers_to_structured_table() {
     Playground::setup("filter_from_csv_test_2", |dirs, sandbox| {
-        sandbox
+        sandbox.with_files(vec![FileWithContentToBeTrimmed(
-            .with_files(vec![FileWithContentToBeTrimmed(
+            "los_tres_amigos.txt",
-                "los_tres_amigos.txt",
+            r#"
-                r#"
                     first_name,last_name,rusty_luck
                     Andrés,Robalino,1
                     Jonathan,Turner,1
@@ -152,10 +148,9 @@ fn can_convert_table_to_json_text_and_from_json_text_back_into_table() {
 #[test]
 fn converts_from_json_text_to_structured_table() {
     Playground::setup("filter_from_json_test_1", |dirs, sandbox| {
-        sandbox
+        sandbox.with_files(vec![FileWithContentToBeTrimmed(
-            .with_files(vec![FileWithContentToBeTrimmed(
+            "katz.txt",
-                "katz.txt",
+            r#"
-                r#"
                     {
                         "katz": [
                             {"name":   "Yehuda", "rusty_luck": 1}, 
@@ -173,17 +168,15 @@ fn converts_from_json_text_to_structured_table() {
         );
 
         assert_eq!(actual, "4");
-
     })
 }
 
 #[test]
 fn converts_from_json_text_recognizing_objects_independendtly_to_structured_table() {
     Playground::setup("filter_from_json_test_2", |dirs, sandbox| {
-        sandbox
+        sandbox.with_files(vec![FileWithContentToBeTrimmed(
-            .with_files(vec![FileWithContentToBeTrimmed(
+            "katz.txt",
-                "katz.txt",
+            r#"
-                r#"
                     {"name":   "Yehuda", "rusty_luck": 1} 
                     {"name": "Jonathan", "rusty_luck": 1} 
                     {"name":   "Andres", "rusty_luck": 1}
@@ -209,10 +202,9 @@ fn converts_from_json_text_recognizing_objects_independendtly_to_structured_tabl
 #[test]
 fn converts_structured_table_to_json_text() {
     Playground::setup("filter_to_json_test", |dirs, sandbox| {
-        sandbox
+        sandbox.with_files(vec![FileWithContentToBeTrimmed(
-            .with_files(vec![FileWithContentToBeTrimmed(
+            "sample.txt",
-                "sample.txt",
+            r#"
-                r#"
                     JonAndrehudaTZ,3
                     GorbyPuff,100
                 "#,
@@ -250,14 +242,13 @@ fn can_convert_table_to_tsv_text_and_from_tsv_text_back_into_table() {
 #[test]
 fn converts_structured_table_to_tsv_text() {
     Playground::setup("filter_to_tsv_test_1", |dirs, sandbox| {
-        sandbox
+        sandbox.with_files(vec![FileWithContentToBeTrimmed(
-            .with_files(vec![FileWithContentToBeTrimmed(
+            "tsv_text_sample.txt",
-                "tsv_text_sample.txt",
+            r#"
-                r#"
                     importer	shipper	tariff_item	name	origin
                     Plasticos Rival	Reverte	2509000000	Calcium carbonate	Spain
                     Tigre Ecuador	OMYA Andina	3824909999	Calcium carbonate	Colombia
-                "#
+                "#,
         )]);
 
         let actual = nu!(
@@ -281,14 +272,13 @@ fn converts_structured_table_to_tsv_text() {
 #[test]
 fn converts_structured_table_to_tsv_text_skipping_headers_after_conversion() {
     Playground::setup("filter_to_tsv_test_2", |dirs, sandbox| {
-        sandbox
+        sandbox.with_files(vec![FileWithContentToBeTrimmed(
-            .with_files(vec![FileWithContentToBeTrimmed(
+            "tsv_text_sample.txt",
-                "tsv_text_sample.txt",
+            r#"
-                r#"
                     importer    shipper tariff_item name    origin
                     Plasticos Rival Reverte 2509000000  Calcium carbonate   Spain
                     Tigre Ecuador   OMYA Andina 3824909999  Calcium carbonate   Colombia
-                "#
+                "#,
         )]);
 
         let actual = nu!(
@@ -310,10 +300,9 @@ fn converts_structured_table_to_tsv_text_skipping_headers_after_conversion() {
 #[test]
 fn converts_from_tsv_text_to_structured_table() {
     Playground::setup("filter_from_tsv_test_1", |dirs, sandbox| {
-        sandbox
+        sandbox.with_files(vec![FileWithContentToBeTrimmed(
-            .with_files(vec![FileWithContentToBeTrimmed(
+            "los_tres_amigos.txt",
-                "los_tres_amigos.txt",
+            r#"
-                r#"
                     first Name	Last Name	rusty_luck
                     Andrés	Robalino	1
                     Jonathan	Turner	1
@@ -340,10 +329,9 @@ fn converts_from_tsv_text_to_structured_table() {
 #[test]
 fn converts_from_tsv_text_skipping_headers_to_structured_table() {
     Playground::setup("filter_from_tsv_test_2", |dirs, sandbox| {
-        sandbox
+        sandbox.with_files(vec![FileWithContentToBeTrimmed(
-            .with_files(vec![FileWithContentToBeTrimmed(
+            "los_tres_amigos.txt",
-                "los_tres_amigos.txt",
+            r#"
-                r#"
                     first Name	Last Name	rusty_luck
                     Andrés	Robalino	1
                     Jonathan	Turner	1
@@ -354,11 +342,11 @@ fn converts_from_tsv_text_skipping_headers_to_structured_table() {
         let actual = nu!(
             cwd: dirs.test(), h::pipeline(
             r#"
-                open los_tres_amigos.txt 
+                open los_tres_amigos.txt
-                | from-tsv --headerless 
+                | from-tsv --headerless
-                | get Column3 
+                | get Column3
-                | str --to-int 
+                | str --to-int
-                | sum 
+                | sum
                 | echo $it
             "#
         ));
@@ -368,21 +356,50 @@ fn converts_from_tsv_text_skipping_headers_to_structured_table() {
 }
 
 #[test]
-fn can_convert_json_text_to_bson_and_back_into_table() {
+fn can_convert_table_to_bson_and_back_into_table() {
     let actual = nu!(
-        cwd: "tests/fixtures/formats",
+        cwd: "tests/fixtures/formats", h::pipeline(
-        "open sample.bson | to-bson | from-bson | get root | nth 1 | get b | echo $it"
+            r#"
-    );
+               open sample.bson
+               | to-bson
+               | from-bson
+               | get root
+               | nth 1
+               | get b
+               | echo $it"#
+    ));
 
     assert_eq!(actual, "whel");
 }
 
+#[test]
+fn can_convert_table_to_sqlite_and_back_into_table() {
+    let actual = nu!(
+        cwd: "tests/fixtures/formats", h::pipeline(
+             r#"
+                 open sample.db
+                 | to-sqlite
+                 | from-sqlite
+                 | get table_values
+                 | nth 2
+                 | get x
+                 | echo $it"#
+    ));
+
+    assert_eq!(actual, "hello");
+}
+
 #[test]
 fn can_convert_table_to_toml_text_and_from_toml_text_back_into_table() {
     let actual = nu!(
-        cwd: "tests/fixtures/formats",
+        cwd: "tests/fixtures/formats", h::pipeline(
-        "open cargo_sample.toml | to-toml | from-toml | get package.name | echo $it"
+            r#"
-    );
+                open cargo_sample.toml
+                | to-toml
+                | from-toml
+                | get package.name
+                | echo $it"#
+    ));
 
     assert_eq!(actual, "nu");
 }