diff --git a/Cargo.lock b/Cargo.lock index bf713a1326..b91214b5bc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -715,6 +715,16 @@ dependencies = [ "synstructure 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "fallible-iterator" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "fixedbitset" version = "0.1.9" @@ -1308,6 +1318,16 @@ dependencies = [ "libc 0.2.60 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "libsqlite3-sys" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cc 1.0.38 (registry+https://github.com/rust-lang/crates.io-index)", + "pkg-config 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)", + "vcpkg 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "libz-sys" version = "1.0.25" @@ -1358,6 +1378,14 @@ dependencies = [ "cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "lru-cache" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "linked-hash-map 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "lzw" version = "0.10.0" @@ -1576,6 +1604,7 @@ dependencies = [ "git2 0.9.2 (registry+https://github.com/rust-lang/crates.io-index)", "glob 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "heim 0.0.7 (registry+https://github.com/rust-lang/crates.io-index)", + "hex 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "image 0.22.1 (registry+https://github.com/rust-lang/crates.io-index)", "indexmap 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", "itertools 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1596,6 +1625,7 @@ dependencies = [ "rawkey 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "regex 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "roxmltree 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rusqlite 0.20.0 (registry+https://github.com/rust-lang/crates.io-index)", "rustyline 5.0.2 (registry+https://github.com/rust-lang/crates.io-index)", "semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.98 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2244,6 +2274,20 @@ dependencies = [ "xmlparser 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "rusqlite" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bitflags 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "fallible-iterator 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "fallible-streaming-iterator 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", + "libsqlite3-sys 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)", + "lru-cache 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "memchr 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "rust-argon2" version = "0.5.0" @@ -3183,6 +3227,8 @@ dependencies = [ "checksum env_logger 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "aafcde04e90a5226a6443b7aabdb016ba2f8307c847d524724bd9b346dd1a2d3" "checksum failure 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "795bd83d3abeb9220f257e597aa0080a508b27533824adf336529648f6abf7e2" "checksum failure_derive 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "ea1063915fd7ef4309e222a5a07cf9c319fb9c7836b1f89b85458672dbb127e1" +"checksum fallible-iterator 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" +"checksum fallible-streaming-iterator 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" "checksum fixedbitset 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "86d4de0081402f5e88cdac65c8dcdcc73118c1a7a465e2a05f0da05843a8ea33" "checksum flate2 1.0.9 (registry+https://github.com/rust-lang/crates.io-index)" = "550934ad4808d5d39365e5d61727309bf18b3b02c6c56b729cb92e7dd84bc3d8" "checksum fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "2fad85553e09a6f881f739c29f0b00b0f01357c743266d478b68951ce23285f3" @@ -3242,12 +3288,14 @@ dependencies = [ "checksum libc 0.2.60 (registry+https://github.com/rust-lang/crates.io-index)" = "d44e80633f007889c7eff624b709ab43c92d708caad982295768a7b13ca3b5eb" "checksum libgit2-sys 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)" = "4c179ed6d19cd3a051e68c177fbbc214e79ac4724fac3a850ec9f3d3eb8a5578" "checksum libnghttp2-sys 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "02254d44f4435dd79e695f2c2b83cd06a47919adea30216ceaf0c57ca0a72463" +"checksum libsqlite3-sys 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5e5b95e89c330291768dc840238db7f9e204fd208511ab6319b56193a7f2ae25" "checksum libz-sys 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)" = "2eb5e43362e38e2bca2fd5f5134c4d4564a23a5c28e9b95411652021a8675ebe" "checksum line-wrap 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f30344350a2a51da54c1d53be93fade8a237e545dbcc4bdbe635413f2117cab9" "checksum linked-hash-map 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6d262045c5b87c0861b3f004610afd0e2c851e2908d08b6c870cbb9d5f494ecd" "checksum linked-hash-map 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "ae91b68aebc4ddb91978b11a1b02ddd8602a05ec19002801c5666000e05e0f83" "checksum lock_api 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "62ebf1391f6acad60e5c8b43706dde4582df75c06698ab44511d15016bc2442c" "checksum log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)" = "14b6052be84e6b71ab17edffc2eeabf5c2c3ae1fdb464aae35ac50c67a44e1f7" +"checksum lru-cache 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "31e24f1ad8321ca0e8a1e0ac13f23cb668e6f5466c2c57319f6a5cf1cc8e3b1c" "checksum lzw 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7d947cbb889ed21c2a84be6ffbaebf5b4e0f4340638cba0444907e38b56be084" "checksum macaddr 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3ff4752cb15cffb3e68f7dcb22e0818ac871f8c98fb07a634a81f41fb202a09f" "checksum mach 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "86dd2487cdfea56def77b88438a2c915fb45113c5319bfe7e14306ca4cd0b0e1" @@ -3342,6 +3390,7 @@ dependencies = [ "checksum render-tree 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "68ed587df09cfb7ce1bc6fe8f77e24db219f222c049326ccbfb948ec67e31664" "checksum result 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "194d8e591e405d1eecf28819740abed6d719d1a2db87fc0bcdedee9a26d55560" "checksum roxmltree 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "153c367ce9fb8ef7afe637ef92bd083ba0f88b03ef3fcf0287d40be05ae0a61c" +"checksum rusqlite 0.20.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2a194373ef527035645a1bc21b10dc2125f73497e6e155771233eb187aedd051" "checksum rust-argon2 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "81ed8d04228b44a740c8d46ff872a28e50fff3d659f307ab4da2cc502e019ff3" "checksum rust-ini 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3e52c148ef37f8c375d49d5a73aa70713125b7f19095948a923f80afdeb22ec2" "checksum rustc-demangle 0.1.15 (registry+https://github.com/rust-lang/crates.io-index)" = "a7f4dccf6f4891ebcc0c39f9b6eb1a83b9bf5d747cb439ec6fba4f3b977038af" diff --git a/Cargo.toml b/Cargo.toml index ba12a45578..faa162f8cc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -62,6 +62,7 @@ mime = "0.3.13" regex = "1.2.1" pretty-hex = "0.1.0" neso = { version = "0.5.0", optional = true } +hex = "0.3.2" crossterm = "0.10.2" tempfile = "3.1.0" image = "0.22.1" @@ -82,6 +83,10 @@ pin-utils = "0.1.0-alpha.4" [features] raw-key = ["rawkey", "neso"] +[dependencies.rusqlite] +version = "0.20.0" +features = ["bundled", "blob"] + [dev-dependencies] pretty_assertions = "0.6.1" diff --git a/src/cli.rs b/src/cli.rs index e01afe89ff..9ffab09c36 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -179,6 +179,7 @@ pub async fn cli() -> Result<(), Box> { whole_stream_command(ToBSON), whole_stream_command(ToCSV), whole_stream_command(ToJSON), + whole_stream_command(ToSQLite), whole_stream_command(ToTOML), whole_stream_command(ToTSV), whole_stream_command(ToYAML), @@ -193,6 +194,7 @@ pub async fn cli() -> Result<(), Box> { whole_stream_command(FromINI), whole_stream_command(FromBSON), whole_stream_command(FromJSON), + whole_stream_command(FromSQLite), whole_stream_command(FromTOML), whole_stream_command(FromXML), whole_stream_command(FromYAML), diff --git a/src/commands.rs b/src/commands.rs index 12cd6d56ee..2f5d174fdf 100644 --- a/src/commands.rs +++ b/src/commands.rs @@ -19,6 +19,7 @@ pub(crate) mod from_bson; pub(crate) mod from_csv; pub(crate) mod from_ini; pub(crate) mod from_json; +pub(crate) mod from_sqlite; pub(crate) mod from_toml; pub(crate) mod from_tsv; pub(crate) mod from_xml; @@ -52,6 +53,7 @@ pub(crate) mod to_array; pub(crate) mod to_bson; pub(crate) mod to_csv; pub(crate) mod to_json; +pub(crate) mod to_sqlite; pub(crate) mod to_toml; pub(crate) mod to_tsv; pub(crate) mod to_yaml; @@ -67,6 +69,7 @@ pub(crate) use command::{ per_item_command, whole_stream_command, Command, PerItemCommand, RawCommandArgs, UnevaluatedCallInfo, WholeStreamCommand, }; + pub(crate) use config::Config; pub(crate) use cp::Cpy; pub(crate) use date::Date; @@ -79,6 +82,7 @@ pub(crate) use from_bson::FromBSON; pub(crate) use from_csv::FromCSV; pub(crate) use from_ini::FromINI; pub(crate) use from_json::FromJSON; +pub(crate) use from_sqlite::FromSQLite; pub(crate) use from_toml::FromTOML; pub(crate) use from_tsv::FromTSV; pub(crate) use from_xml::FromXML; @@ -111,6 +115,7 @@ pub(crate) use to_array::ToArray; pub(crate) use to_bson::ToBSON; pub(crate) use to_csv::ToCSV; pub(crate) use to_json::ToJSON; +pub(crate) use to_sqlite::ToSQLite; pub(crate) use to_toml::ToTOML; pub(crate) use to_tsv::ToTSV; pub(crate) use to_yaml::ToYAML; diff --git a/src/commands/from_bson.rs b/src/commands/from_bson.rs index e244614ccf..ff1fc267c1 100644 --- a/src/commands/from_bson.rs +++ b/src/commands/from_bson.rs @@ -45,7 +45,7 @@ fn convert_bson_value_to_nu_value(v: &Bson, tag: impl Into) -> Tagged Value::Primitive(Primitive::Boolean(*b)).tagged(tag), - Bson::Null => Value::Primitive(Primitive::String(String::from(""))).tagged(tag), + Bson::Null => Value::Primitive(Primitive::Nothing).tagged(tag), Bson::RegExp(r, opts) => { let mut collected = TaggedDictBuilder::new(tag); collected.insert_tagged( diff --git a/src/commands/from_sqlite.rs b/src/commands/from_sqlite.rs new file mode 100644 index 0000000000..2ebf67690c --- /dev/null +++ b/src/commands/from_sqlite.rs @@ -0,0 +1,139 @@ +use crate::commands::WholeStreamCommand; +use crate::errors::ShellError; +use crate::object::base::OF64; +use crate::object::{Primitive, TaggedDictBuilder, Value}; +use crate::prelude::*; +use rusqlite::{types::ValueRef, Connection, Row, NO_PARAMS}; +use std::io::Write; +use std::path::Path; + +pub struct FromSQLite; + +impl WholeStreamCommand for FromSQLite { + fn run( + &self, + args: CommandArgs, + registry: &CommandRegistry, + ) -> Result { + from_sqlite(args, registry) + } + + fn name(&self) -> &str { + "from-sqlite" + } + + fn signature(&self) -> Signature { + Signature::build("from-sqlite") + } +} + +pub fn convert_sqlite_file_to_nu_value( + path: &Path, + tag: impl Into + Clone, +) -> Result, rusqlite::Error> { + let conn = Connection::open(path)?; + + let mut meta_out = Vec::new(); + let mut meta_stmt = conn.prepare("select name from sqlite_master where type='table'")?; + let mut meta_rows = meta_stmt.query(NO_PARAMS)?; + while let Some(meta_row) = meta_rows.next()? { + let table_name: String = meta_row.get(0)?; + let mut meta_dict = TaggedDictBuilder::new(tag.clone()); + let mut out = Vec::new(); + let mut table_stmt = conn.prepare(&format!("select * from [{}]", table_name))?; + let mut table_rows = table_stmt.query(NO_PARAMS)?; + while let Some(table_row) = table_rows.next()? { + out.push(convert_sqlite_row_to_nu_value(table_row, tag.clone())?) + } + meta_dict.insert_tagged( + "table_name".to_string(), + Value::Primitive(Primitive::String(table_name)).tagged(tag.clone()), + ); + meta_dict.insert_tagged("table_values", Value::List(out).tagged(tag.clone())); + meta_out.push(meta_dict.into_tagged_value()); + } + let tag = tag.into(); + Ok(Value::List(meta_out).tagged(tag)) +} + +fn convert_sqlite_row_to_nu_value( + row: &Row, + tag: impl Into + Clone, +) -> Result, rusqlite::Error> { + let mut collected = TaggedDictBuilder::new(tag.clone()); + for (i, c) in row.columns().iter().enumerate() { + collected.insert_tagged( + c.name().to_string(), + convert_sqlite_value_to_nu_value(row.get_raw(i), tag.clone()), + ); + } + return Ok(collected.into_tagged_value()); +} + +fn convert_sqlite_value_to_nu_value(value: ValueRef, tag: impl Into + Clone) -> Tagged { + match value { + ValueRef::Null => Value::Primitive(Primitive::String(String::from(""))).tagged(tag), + ValueRef::Integer(i) => Value::Primitive(Primitive::Int(i)).tagged(tag), + ValueRef::Real(f) => Value::Primitive(Primitive::Float(OF64::from(f))).tagged(tag), + t @ ValueRef::Text(_) => { + // this unwrap is safe because we know the ValueRef is Text. + Value::Primitive(Primitive::String(t.as_str().unwrap().to_string())).tagged(tag) + } + ValueRef::Blob(u) => Value::Binary(u.to_owned()).tagged(tag), + } +} + +pub fn from_sqlite_bytes_to_value( + mut bytes: Vec, + tag: impl Into + Clone, +) -> Result, std::io::Error> { + // FIXME: should probably write a sqlite virtual filesystem + // that will allow us to use bytes as a file to avoid this + // write out, but this will require C code. Might be + // best done as a PR to rusqlite. + let mut tempfile = tempfile::NamedTempFile::new()?; + tempfile.write_all(bytes.as_mut_slice())?; + match convert_sqlite_file_to_nu_value(tempfile.path(), tag) { + Ok(value) => Ok(value), + Err(e) => Err(std::io::Error::new(std::io::ErrorKind::Other, e)), + } +} + +fn from_sqlite(args: CommandArgs, registry: &CommandRegistry) -> Result { + let args = args.evaluate_once(registry)?; + let span = args.name_span(); + let input = args.input; + + let stream = async_stream_block! { + let values: Vec> = input.values.collect().await; + + for value in values { + let value_tag = value.tag(); + match value.item { + Value::Binary(vb) => + match from_sqlite_bytes_to_value(vb, span) { + Ok(x) => yield ReturnSuccess::value(x), + Err(_) => { + yield Err(ShellError::labeled_error_with_secondary( + "Could not parse as SQLite", + "input cannot be parsed as SQLite", + span, + "value originates from here", + value_tag.span, + )) + } + } + _ => yield Err(ShellError::labeled_error_with_secondary( + "Expected a string from pipeline", + "requires string input", + span, + "value originates from here", + value_tag.span, + )), + + } + } + }; + + Ok(stream.to_output_stream()) +} diff --git a/src/commands/open.rs b/src/commands/open.rs index c4cd057147..af43e698ff 100644 --- a/src/commands/open.rs +++ b/src/commands/open.rs @@ -527,6 +527,16 @@ pub fn parse_binary_as_value( }, ) } + Some(ref x) if x == "db" => { + crate::commands::from_sqlite::from_sqlite_bytes_to_value(contents, contents_tag) + .map_err(move |_| { + ShellError::labeled_error( + "Could not open as SQLite", + "could not open as SQLite", + name_span, + ) + }) + } _ => Ok(Value::Binary(contents).tagged(contents_tag)), } } diff --git a/src/commands/to_sqlite.rs b/src/commands/to_sqlite.rs new file mode 100644 index 0000000000..55e001b0ba --- /dev/null +++ b/src/commands/to_sqlite.rs @@ -0,0 +1,183 @@ +use crate::commands::WholeStreamCommand; +use crate::object::{Dictionary, Primitive, Value}; +use crate::prelude::*; +use hex::encode; +use rusqlite::{Connection, NO_PARAMS}; +use std::io::Read; + +pub struct ToSQLite; + +impl WholeStreamCommand for ToSQLite { + fn run( + &self, + args: CommandArgs, + registry: &CommandRegistry, + ) -> Result { + to_sqlite(args, registry) + } + + fn name(&self) -> &str { + "to-sqlite" + } + + fn signature(&self) -> Signature { + Signature::build("to-sqlite") + } +} + +fn comma_concat(acc: String, current: String) -> String { + if acc == "" { + current + } else { + format!("{}, {}", acc, current) + } +} + +fn get_columns(rows: &Vec>) -> Result { + match &rows[0].item { + Value::Object(d) => Ok(d + .entries + .iter() + .map(|(k, _v)| k.clone()) + .fold("".to_string(), comma_concat)), + _ => Err(std::io::Error::new( + std::io::ErrorKind::Other, + "Could not find table column names", + )), + } +} + +fn nu_value_to_sqlite_string(v: Value) -> String { + match v { + Value::Binary(u) => format!("x'{}'", encode(u)), + Value::Primitive(p) => match p { + Primitive::Nothing => "NULL".into(), + Primitive::Int(i) => format!("{}", i), + Primitive::Float(f) => format!("{}", f.into_inner()), + Primitive::Bytes(u) => format!("{}", u), + Primitive::String(s) => format!("'{}'", s.replace("'", "''")), + Primitive::Boolean(true) => "1".into(), + Primitive::Boolean(_) => "0".into(), + Primitive::Date(d) => format!("'{}'", d), + Primitive::Path(p) => format!("'{}'", p.display().to_string().replace("'", "''")), + Primitive::BeginningOfStream => "NULL".into(), + Primitive::EndOfStream => "NULL".into(), + }, + _ => "NULL".into(), + } +} + +fn get_insert_values(rows: Vec>) -> Result { + let values: Result, _> = rows + .into_iter() + .map(|value| match value.item { + Value::Object(d) => Ok(format!( + "({})", + d.entries + .iter() + .map(|(_k, v)| nu_value_to_sqlite_string(v.item.clone())) + .fold("".to_string(), comma_concat) + )), + _ => Err(std::io::Error::new( + std::io::ErrorKind::Other, + "Could not find table column names", + )), + }) + .collect(); + let values = values?; + Ok(values.into_iter().fold("".to_string(), comma_concat)) +} + +fn generate_statements(table: Dictionary) -> Result<(String, String), std::io::Error> { + let table_name = match table.entries.get("table_name") { + Some(Tagged { + item: Value::Primitive(Primitive::String(table_name)), + .. + }) => table_name, + _ => { + return Err(std::io::Error::new( + std::io::ErrorKind::Other, + "Could not find table name", + )) + } + }; + let (columns, insert_values) = match table.entries.get("table_values") { + Some(Tagged { + item: Value::List(l), + .. + }) => (get_columns(l), get_insert_values(l.to_vec())), + _ => { + return Err(std::io::Error::new( + std::io::ErrorKind::Other, + "Could not find table values", + )) + } + }; + let create = format!("create table {}({})", table_name, columns?); + let insert = format!("insert into {} values {}", table_name, insert_values?); + Ok((create, insert)) +} + +fn sqlite_input_stream_to_bytes( + values: Vec>, +) -> Result, std::io::Error> { + // FIXME: should probably write a sqlite virtual filesystem + // that will allow us to use bytes as a file to avoid this + // write out, but this will require C code. Might be + // best done as a PR to rusqlite. + let mut tempfile = tempfile::NamedTempFile::new()?; + let conn = match Connection::open(tempfile.path()) { + Ok(conn) => conn, + Err(e) => return Err(std::io::Error::new(std::io::ErrorKind::Other, e)), + }; + let tag = values[0].tag.clone(); + for value in values.into_iter() { + match value.item() { + Value::Object(d) => { + let (create, insert) = generate_statements(d.to_owned())?; + match conn + .execute(&create, NO_PARAMS) + .and_then(|_| conn.execute(&insert, NO_PARAMS)) + { + Ok(_) => (), + Err(e) => { + println!("{}", create); + println!("{}", insert); + println!("{:?}", e); + return Err(std::io::Error::new(std::io::ErrorKind::Other, e)); + } + } + } + other => { + return Err(std::io::Error::new( + std::io::ErrorKind::Other, + format!("Expected object, found {:?}", other), + )) + } + } + } + let mut out = Vec::new(); + tempfile.read_to_end(&mut out)?; + Ok(Value::Binary(out).tagged(tag)) +} + +fn to_sqlite(args: CommandArgs, registry: &CommandRegistry) -> Result { + let args = args.evaluate_once(registry)?; + let name_span = args.name_span(); + let stream = async_stream_block! { + let values: Vec<_> = args.input.into_vec().await; + match sqlite_input_stream_to_bytes(values) { + Ok(out) => { + yield ReturnSuccess::value(out) + } + Err(_) => { + yield Err(ShellError::labeled_error( + "Expected an object with SQLite-compatible structure from pipeline", + "requires SQLite-compatible input", + name_span, + )) + } + }; + }; + Ok(stream.to_output_stream()) +} diff --git a/src/utils.rs b/src/utils.rs index 159907ea52..4ed9be2540 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -266,6 +266,10 @@ mod tests { loc: fixtures().join("sample.bson"), at: 0 }, + Res { + loc: fixtures().join("sample.db"), + at: 0 + }, Res { loc: fixtures().join("sample.ini"), at: 0 diff --git a/tests/command_open_tests.rs b/tests/command_open_tests.rs index 0b01cc6b8e..968bc7531b 100644 --- a/tests/command_open_tests.rs +++ b/tests/command_open_tests.rs @@ -6,23 +6,22 @@ use helpers::{Playground, Stub::*}; #[test] fn recognizes_csv() { Playground::setup("open_test_1", |dirs, sandbox| { - sandbox - .with_files(vec![FileWithContentToBeTrimmed( - "nu.zion.csv", - r#" + sandbox.with_files(vec![FileWithContentToBeTrimmed( + "nu.zion.csv", + r#" author,lang,source Jonathan Turner,Rust,New Zealand Andres N. Robalino,Rust,Ecuador Yehuda Katz,Rust,Estados Unidos - "# + "#, )]); let actual = nu!( cwd: dirs.test(), h::pipeline( r#" - open nu.zion.csv - | where author == "Andres N. Robalino" - | get source + open nu.zion.csv + | where author == "Andres N. Robalino" + | get source | echo $it "# )); @@ -46,11 +45,11 @@ fn open_can_parse_bson_2() { let actual = nu!( cwd: "tests/fixtures/formats", h::pipeline( r#" - open sample.bson - | get root - | nth 6 - | get b - | get '$binary_subtype' + open sample.bson + | get root + | nth 6 + | get b + | get '$binary_subtype' | echo $it "# )); @@ -58,6 +57,21 @@ fn open_can_parse_bson_2() { assert_eq!(actual, "function"); } +#[test] +fn open_can_parse_sqlite() { + let actual = nu!( + cwd: "tests/fixtures/formats", h::pipeline( + r#" + open sample.db + | get table_values + | nth 2 + | get x + | echo $it"# + )); + + assert_eq!(actual, "hello"); +} + #[test] fn open_can_parse_toml() { let actual = nu!( @@ -74,8 +88,8 @@ fn open_can_parse_tsv() { cwd: "tests/fixtures/formats", h::pipeline( r#" open caco3_plastics.tsv - | first 1 - | get origin + | first 1 + | get origin | echo $it "# )); @@ -88,8 +102,8 @@ fn open_can_parse_json() { let actual = nu!( cwd: "tests/fixtures/formats", h::pipeline( r#" - open sgml_description.json - | get glossary.GlossDiv.GlossList.GlossEntry.GlossSee + open sgml_description.json + | get glossary.GlossDiv.GlossList.GlossEntry.GlossSee | echo $it "# )); diff --git a/tests/filters_test.rs b/tests/filters_test.rs index deabdaa257..33baf67fe3 100644 --- a/tests/filters_test.rs +++ b/tests/filters_test.rs @@ -16,14 +16,13 @@ fn can_convert_table_to_csv_text_and_from_csv_text_back_into_table() { #[test] fn converts_structured_table_to_csv_text() { Playground::setup("filter_to_csv_test_1", |dirs, sandbox| { - sandbox - .with_files(vec![FileWithContentToBeTrimmed( - "csv_text_sample.txt", - r#" + sandbox.with_files(vec![FileWithContentToBeTrimmed( + "csv_text_sample.txt", + r#" importer,shipper,tariff_item,name,origin Plasticos Rival,Reverte,2509000000,Calcium carbonate,Spain Tigre Ecuador,OMYA Andina,3824909999,Calcium carbonate,Colombia - "# + "#, )]); let actual = nu!( @@ -47,14 +46,13 @@ fn converts_structured_table_to_csv_text() { #[test] fn converts_structured_table_to_csv_text_skipping_headers_after_conversion() { Playground::setup("filter_to_csv_test_2", |dirs, sandbox| { - sandbox - .with_files(vec![FileWithContentToBeTrimmed( - "csv_text_sample.txt", - r#" + sandbox.with_files(vec![FileWithContentToBeTrimmed( + "csv_text_sample.txt", + r#" importer,shipper,tariff_item,name,origin Plasticos Rival,Reverte,2509000000,Calcium carbonate,Spain Tigre Ecuador,OMYA Andina,3824909999,Calcium carbonate,Colombia - "# + "#, )]); let actual = nu!( @@ -76,10 +74,9 @@ fn converts_structured_table_to_csv_text_skipping_headers_after_conversion() { #[test] fn converts_from_csv_text_to_structured_table() { Playground::setup("filter_from_csv_test_1", |dirs, sandbox| { - sandbox - .with_files(vec![FileWithContentToBeTrimmed( - "los_tres_amigos.txt", - r#" + sandbox.with_files(vec![FileWithContentToBeTrimmed( + "los_tres_amigos.txt", + r#" first_name,last_name,rusty_luck Andrés,Robalino,1 Jonathan,Turner,1 @@ -106,10 +103,9 @@ fn converts_from_csv_text_to_structured_table() { #[test] fn converts_from_csv_text_skipping_headers_to_structured_table() { Playground::setup("filter_from_csv_test_2", |dirs, sandbox| { - sandbox - .with_files(vec![FileWithContentToBeTrimmed( - "los_tres_amigos.txt", - r#" + sandbox.with_files(vec![FileWithContentToBeTrimmed( + "los_tres_amigos.txt", + r#" first_name,last_name,rusty_luck Andrés,Robalino,1 Jonathan,Turner,1 @@ -152,10 +148,9 @@ fn can_convert_table_to_json_text_and_from_json_text_back_into_table() { #[test] fn converts_from_json_text_to_structured_table() { Playground::setup("filter_from_json_test_1", |dirs, sandbox| { - sandbox - .with_files(vec![FileWithContentToBeTrimmed( - "katz.txt", - r#" + sandbox.with_files(vec![FileWithContentToBeTrimmed( + "katz.txt", + r#" { "katz": [ {"name": "Yehuda", "rusty_luck": 1}, @@ -173,17 +168,15 @@ fn converts_from_json_text_to_structured_table() { ); assert_eq!(actual, "4"); - }) } #[test] fn converts_from_json_text_recognizing_objects_independendtly_to_structured_table() { Playground::setup("filter_from_json_test_2", |dirs, sandbox| { - sandbox - .with_files(vec![FileWithContentToBeTrimmed( - "katz.txt", - r#" + sandbox.with_files(vec![FileWithContentToBeTrimmed( + "katz.txt", + r#" {"name": "Yehuda", "rusty_luck": 1} {"name": "Jonathan", "rusty_luck": 1} {"name": "Andres", "rusty_luck": 1} @@ -209,10 +202,9 @@ fn converts_from_json_text_recognizing_objects_independendtly_to_structured_tabl #[test] fn converts_structured_table_to_json_text() { Playground::setup("filter_to_json_test", |dirs, sandbox| { - sandbox - .with_files(vec![FileWithContentToBeTrimmed( - "sample.txt", - r#" + sandbox.with_files(vec![FileWithContentToBeTrimmed( + "sample.txt", + r#" JonAndrehudaTZ,3 GorbyPuff,100 "#, @@ -250,14 +242,13 @@ fn can_convert_table_to_tsv_text_and_from_tsv_text_back_into_table() { #[test] fn converts_structured_table_to_tsv_text() { Playground::setup("filter_to_tsv_test_1", |dirs, sandbox| { - sandbox - .with_files(vec![FileWithContentToBeTrimmed( - "tsv_text_sample.txt", - r#" + sandbox.with_files(vec![FileWithContentToBeTrimmed( + "tsv_text_sample.txt", + r#" importer shipper tariff_item name origin Plasticos Rival Reverte 2509000000 Calcium carbonate Spain Tigre Ecuador OMYA Andina 3824909999 Calcium carbonate Colombia - "# + "#, )]); let actual = nu!( @@ -281,14 +272,13 @@ fn converts_structured_table_to_tsv_text() { #[test] fn converts_structured_table_to_tsv_text_skipping_headers_after_conversion() { Playground::setup("filter_to_tsv_test_2", |dirs, sandbox| { - sandbox - .with_files(vec![FileWithContentToBeTrimmed( - "tsv_text_sample.txt", - r#" + sandbox.with_files(vec![FileWithContentToBeTrimmed( + "tsv_text_sample.txt", + r#" importer shipper tariff_item name origin Plasticos Rival Reverte 2509000000 Calcium carbonate Spain Tigre Ecuador OMYA Andina 3824909999 Calcium carbonate Colombia - "# + "#, )]); let actual = nu!( @@ -310,10 +300,9 @@ fn converts_structured_table_to_tsv_text_skipping_headers_after_conversion() { #[test] fn converts_from_tsv_text_to_structured_table() { Playground::setup("filter_from_tsv_test_1", |dirs, sandbox| { - sandbox - .with_files(vec![FileWithContentToBeTrimmed( - "los_tres_amigos.txt", - r#" + sandbox.with_files(vec![FileWithContentToBeTrimmed( + "los_tres_amigos.txt", + r#" first Name Last Name rusty_luck Andrés Robalino 1 Jonathan Turner 1 @@ -340,10 +329,9 @@ fn converts_from_tsv_text_to_structured_table() { #[test] fn converts_from_tsv_text_skipping_headers_to_structured_table() { Playground::setup("filter_from_tsv_test_2", |dirs, sandbox| { - sandbox - .with_files(vec![FileWithContentToBeTrimmed( - "los_tres_amigos.txt", - r#" + sandbox.with_files(vec![FileWithContentToBeTrimmed( + "los_tres_amigos.txt", + r#" first Name Last Name rusty_luck Andrés Robalino 1 Jonathan Turner 1 @@ -354,11 +342,11 @@ fn converts_from_tsv_text_skipping_headers_to_structured_table() { let actual = nu!( cwd: dirs.test(), h::pipeline( r#" - open los_tres_amigos.txt - | from-tsv --headerless - | get Column3 - | str --to-int - | sum + open los_tres_amigos.txt + | from-tsv --headerless + | get Column3 + | str --to-int + | sum | echo $it "# )); @@ -368,21 +356,50 @@ fn converts_from_tsv_text_skipping_headers_to_structured_table() { } #[test] -fn can_convert_json_text_to_bson_and_back_into_table() { +fn can_convert_table_to_bson_and_back_into_table() { let actual = nu!( - cwd: "tests/fixtures/formats", - "open sample.bson | to-bson | from-bson | get root | nth 1 | get b | echo $it" - ); + cwd: "tests/fixtures/formats", h::pipeline( + r#" + open sample.bson + | to-bson + | from-bson + | get root + | nth 1 + | get b + | echo $it"# + )); assert_eq!(actual, "whel"); } +#[test] +fn can_convert_table_to_sqlite_and_back_into_table() { + let actual = nu!( + cwd: "tests/fixtures/formats", h::pipeline( + r#" + open sample.db + | to-sqlite + | from-sqlite + | get table_values + | nth 2 + | get x + | echo $it"# + )); + + assert_eq!(actual, "hello"); +} + #[test] fn can_convert_table_to_toml_text_and_from_toml_text_back_into_table() { let actual = nu!( - cwd: "tests/fixtures/formats", - "open cargo_sample.toml | to-toml | from-toml | get package.name | echo $it" - ); + cwd: "tests/fixtures/formats", h::pipeline( + r#" + open cargo_sample.toml + | to-toml + | from-toml + | get package.name + | echo $it"# + )); assert_eq!(actual, "nu"); } diff --git a/tests/fixtures/formats/sample.db b/tests/fixtures/formats/sample.db new file mode 100644 index 0000000000..df67bfce3b Binary files /dev/null and b/tests/fixtures/formats/sample.db differ