From e94d13da1b210a96d3ef42d7658fdf2c84b68a8c Mon Sep 17 00:00:00 2001 From: Fernando Herrera Date: Sun, 24 Apr 2022 10:29:21 +0100 Subject: [PATCH] Database commands (#5307) * database commands * db commands * filesystem opens sqlite file * clippy error * corrected error in ci file * removes matrix flag from ci * flax matrix for clippy * add conditional compile for tests * add conditional compile for tests * correct order of command * correct error msg * correct typo --- .gitignore | 3 + Cargo.lock | 10 + Cargo.toml | 3 + crates/nu-command/Cargo.toml | 4 +- .../src/database/commands/collect.rs | 49 +++ .../src/database/commands/command.rs | 42 +++ .../src/database/commands/describe.rs | 47 +++ .../nu-command/src/database/commands/from.rs | 130 ++++++++ .../nu-command/src/database/commands/mod.rs | 31 ++ .../nu-command/src/database/commands/open.rs | 52 +++ .../nu-command/src/database/commands/query.rs | 57 ++++ .../src/database/commands/select.rs | 131 ++++++++ .../nu-command/src/database/commands/utils.rs | 15 + crates/nu-command/src/database/mod.rs | 6 +- crates/nu-command/src/database/values/mod.rs | 3 + .../src/database/{ => values}/sqlite.rs | 310 +++++++++++++----- crates/nu-command/src/default_context.rs | 10 +- crates/nu-command/src/filesystem/open.rs | 39 +-- crates/nu-command/src/lib.rs | 10 +- crates/nu-command/src/query/db.rs | 96 ------ crates/nu-command/src/query/mod.rs | 3 - crates/nu-command/tests/commands/mod.rs | 1 + crates/nu-command/tests/commands/open.rs | 3 +- crates/nu-command/tests/commands/query/db.rs | 11 +- crates/nu-command/tests/commands/where_.rs | 2 + 25 files changed, 845 insertions(+), 223 deletions(-) create mode 100644 crates/nu-command/src/database/commands/collect.rs create mode 100644 crates/nu-command/src/database/commands/command.rs create mode 100644 crates/nu-command/src/database/commands/describe.rs create mode 100644 crates/nu-command/src/database/commands/from.rs create mode 100644 crates/nu-command/src/database/commands/mod.rs create mode 100644 crates/nu-command/src/database/commands/open.rs create mode 100644 crates/nu-command/src/database/commands/query.rs create mode 100644 crates/nu-command/src/database/commands/select.rs create mode 100644 crates/nu-command/src/database/commands/utils.rs create mode 100644 crates/nu-command/src/database/values/mod.rs rename crates/nu-command/src/database/{ => values}/sqlite.rs (55%) delete mode 100644 crates/nu-command/src/query/db.rs delete mode 100644 crates/nu-command/src/query/mod.rs diff --git a/.gitignore b/.gitignore index 2a3f38db4f..38a1eb9229 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,6 @@ debian/nu/ # VSCode's IDE items .vscode/* + +# Helix configuration folder +.helix diff --git a/Cargo.lock b/Cargo.lock index 4bbde8fc31..e8098f0183 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2357,6 +2357,7 @@ dependencies = [ "serde_yaml", "sha2 0.10.2", "shadow-rs", + "sqlparser", "strip-ansi-escapes", "sysinfo", "terminal_size", @@ -4051,6 +4052,15 @@ dependencies = [ "lock_api", ] +[[package]] +name = "sqlparser" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e9a527b68048eb95495a1508f6c8395c8defcff5ecdbe8ad4106d08a2ef2a3c" +dependencies = [ + "log", +] + [[package]] name = "stable_deref_trait" version = "1.2.0" diff --git a/Cargo.toml b/Cargo.toml index 126c77d593..ac13681df5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -85,6 +85,9 @@ trash-support = ["nu-command/trash-support"] # Dataframe feature for nushell dataframe = ["nu-command/dataframe"] +# Database commands for nushell +database = ["nu-command/database"] + [profile.release] opt-level = "s" # Optimize for size strip = "debuginfo" diff --git a/crates/nu-command/Cargo.toml b/crates/nu-command/Cargo.toml index 7700799330..4a7d1aa8b9 100644 --- a/crates/nu-command/Cargo.toml +++ b/crates/nu-command/Cargo.toml @@ -24,7 +24,6 @@ nu-term-grid = { path = "../nu-term-grid", version = "0.61.1" } nu-test-support = { path = "../nu-test-support", version = "0.61.1" } nu-utils = { path = "../nu-utils", version = "0.61.1" } nu-ansi-term = "0.45.1" -rusqlite = { version = "0.27.0", features = ["bundled"] } # Potential dependencies for extras base64 = "0.13.0" @@ -81,6 +80,8 @@ uuid = { version = "0.8.2", features = ["v4"] } which = { version = "4.2.2", optional = true } reedline = { git = "https://github.com/nushell/reedline", branch = "main", features = ["bashisms"]} wax = { version = "0.4.0", features = ["diagnostics"] } +rusqlite = { version = "0.27.0", features = ["bundled"], optional = true } +sqlparser = { version = "0.16.0", optional = true } [target.'cfg(unix)'.dependencies] umask = "1.0.0" @@ -105,6 +106,7 @@ trash-support = ["trash"] which-support = ["which"] plugin = ["nu-parser/plugin"] dataframe = ["polars", "num"] +database = ["sqlparser", "rusqlite"] [build-dependencies] shadow-rs = "0.11.0" diff --git a/crates/nu-command/src/database/commands/collect.rs b/crates/nu-command/src/database/commands/collect.rs new file mode 100644 index 0000000000..67749168ef --- /dev/null +++ b/crates/nu-command/src/database/commands/collect.rs @@ -0,0 +1,49 @@ +use nu_protocol::{ + ast::Call, + engine::{Command, EngineState, Stack}, + Category, Example, IntoPipelineData, PipelineData, ShellError, Signature, +}; + +use super::super::SQLiteDatabase; + +#[derive(Clone)] +pub struct CollectDb; + +impl Command for CollectDb { + fn name(&self) -> &str { + "db collect" + } + + fn signature(&self) -> Signature { + Signature::build(self.name()).category(Category::Custom("database".into())) + } + + fn usage(&self) -> &str { + "Query a database using SQL." + } + + fn examples(&self) -> Vec { + vec![Example { + description: "Collect from a select query", + example: "open foo.db | db select a | db from table_1 | db collect", + result: None, + }] + } + + fn search_terms(&self) -> Vec<&str> { + vec!["database", "collect"] + } + + fn run( + &self, + _engine_state: &EngineState, + _stack: &mut Stack, + call: &Call, + input: PipelineData, + ) -> Result { + let db = SQLiteDatabase::try_from_pipeline(input, call.head)?; + + db.collect(call.head) + .map(IntoPipelineData::into_pipeline_data) + } +} diff --git a/crates/nu-command/src/database/commands/command.rs b/crates/nu-command/src/database/commands/command.rs new file mode 100644 index 0000000000..c8c53df99b --- /dev/null +++ b/crates/nu-command/src/database/commands/command.rs @@ -0,0 +1,42 @@ +use nu_engine::get_full_help; +use nu_protocol::{ + ast::Call, + engine::{Command, EngineState, Stack}, + Category, IntoPipelineData, PipelineData, ShellError, Signature, Value, +}; + +#[derive(Clone)] +pub struct Database; + +impl Command for Database { + fn name(&self) -> &str { + "db" + } + + fn usage(&self) -> &str { + "Database commands" + } + + fn signature(&self) -> Signature { + Signature::build(self.name()).category(Category::Custom("database".into())) + } + + fn run( + &self, + engine_state: &EngineState, + stack: &mut Stack, + call: &Call, + _input: PipelineData, + ) -> Result { + Ok(Value::String { + val: get_full_help( + &Database.signature(), + &Database.examples(), + engine_state, + stack, + ), + span: call.head, + } + .into_pipeline_data()) + } +} diff --git a/crates/nu-command/src/database/commands/describe.rs b/crates/nu-command/src/database/commands/describe.rs new file mode 100644 index 0000000000..788d5b5bbb --- /dev/null +++ b/crates/nu-command/src/database/commands/describe.rs @@ -0,0 +1,47 @@ +use nu_protocol::{ + ast::Call, + engine::{Command, EngineState, Stack}, + Category, Example, IntoPipelineData, PipelineData, ShellError, Signature, +}; + +use super::super::SQLiteDatabase; + +#[derive(Clone)] +pub struct DescribeDb; + +impl Command for DescribeDb { + fn name(&self) -> &str { + "db describe" + } + + fn signature(&self) -> Signature { + Signature::build(self.name()).category(Category::Custom("database".into())) + } + + fn usage(&self) -> &str { + "Describes connection and query of the DB object" + } + + fn examples(&self) -> Vec { + vec![Example { + description: "Describe SQLite database constructed query", + example: "db open foo.db | db select table_1 | db describe", + result: None, + }] + } + + fn search_terms(&self) -> Vec<&str> { + vec!["database", "SQLite"] + } + + fn run( + &self, + _engine_state: &EngineState, + _stack: &mut Stack, + call: &Call, + input: PipelineData, + ) -> Result { + let db = SQLiteDatabase::try_from_pipeline(input, call.head)?; + Ok(db.describe(call.head).into_pipeline_data()) + } +} diff --git a/crates/nu-command/src/database/commands/from.rs b/crates/nu-command/src/database/commands/from.rs new file mode 100644 index 0000000000..e5a73f4874 --- /dev/null +++ b/crates/nu-command/src/database/commands/from.rs @@ -0,0 +1,130 @@ +use super::super::SQLiteDatabase; +use nu_engine::CallExt; +use nu_protocol::{ + ast::Call, + engine::{Command, EngineState, Stack}, + Category, Example, IntoPipelineData, PipelineData, ShellError, Signature, SyntaxShape, +}; +use sqlparser::ast::{Ident, ObjectName, Query, Select, SetExpr, TableFactor, TableWithJoins}; + +#[derive(Clone)] +pub struct FromDb; + +impl Command for FromDb { + fn name(&self) -> &str { + "db from" + } + + fn usage(&self) -> &str { + "Select section from query statement for a DB" + } + + fn signature(&self) -> Signature { + Signature::build(self.name()) + .required( + "select", + SyntaxShape::String, + "Name of table to select from", + ) + .category(Category::Custom("database".into())) + } + + fn search_terms(&self) -> Vec<&str> { + vec!["database", "from"] + } + + fn examples(&self) -> Vec { + vec![Example { + description: "Selects table from database", + example: "db open db.mysql | db from table_a", + result: None, + }] + } + + fn run( + &self, + engine_state: &EngineState, + stack: &mut Stack, + call: &Call, + input: PipelineData, + ) -> Result { + let table: String = call.req(engine_state, stack, 0)?; + + let mut db = SQLiteDatabase::try_from_pipeline(input, call.head)?; + db.query = match db.query { + None => Some(create_query(table)), + Some(query) => Some(modify_query(query, table)), + }; + + Ok(db.into_value(call.head).into_pipeline_data()) + } +} + +fn create_query(table: String) -> Query { + Query { + with: None, + body: SetExpr::Select(Box::new(create_select(table))), + order_by: Vec::new(), + limit: None, + offset: None, + fetch: None, + lock: None, + } +} + +fn modify_query(mut query: Query, table: String) -> Query { + query.body = match query.body { + SetExpr::Select(select) => SetExpr::Select(Box::new(modify_select(select, table))), + _ => SetExpr::Select(Box::new(create_select(table))), + }; + + query +} + +fn modify_select(select: Box, expressions: Vec) -> Select { + Select { + projection: create_projection(expressions), + ..select.as_ref().clone() + } +} + +fn create_select(expressions: Vec) -> Select { + Select { + distinct: false, + top: None, + projection: create_projection(expressions), + into: None, + from: Vec::new(), + lateral_views: Vec::new(), + selection: None, + group_by: Vec::new(), + cluster_by: Vec::new(), + distribute_by: Vec::new(), + sort_by: Vec::new(), + having: None, + } +} + +// This function needs more work +// It needs to define alias and functions in the columns +// I assume we will need to define expressions for the columns instead of strings +fn create_projection(expressions: Vec) -> Vec { + expressions + .into_iter() + .map(|expression| { + let expr = Expr::Identifier(Ident { + value: expression, + quote_style: None, + }); + + SelectItem::UnnamedExpr(expr) + }) + .collect() +} diff --git a/crates/nu-command/src/database/commands/utils.rs b/crates/nu-command/src/database/commands/utils.rs new file mode 100644 index 0000000000..58f7589f63 --- /dev/null +++ b/crates/nu-command/src/database/commands/utils.rs @@ -0,0 +1,15 @@ +use nu_protocol::{FromValue, ShellError, Value}; + +pub fn extract_strings(value: Value) -> Result, ShellError> { + match ( + ::from_value(&value), + as FromValue>::from_value(&value), + ) { + (Ok(col), Err(_)) => Ok(vec![col]), + (Err(_), Ok(cols)) => Ok(cols), + _ => Err(ShellError::IncompatibleParametersSingle( + "Expected a string or list of strings".into(), + value.span()?, + )), + } +} diff --git a/crates/nu-command/src/database/mod.rs b/crates/nu-command/src/database/mod.rs index d4f755050d..6b7c7cf12f 100644 --- a/crates/nu-command/src/database/mod.rs +++ b/crates/nu-command/src/database/mod.rs @@ -1,3 +1,5 @@ -mod sqlite; +mod commands; +mod values; -pub use sqlite::SQLiteDatabase; +pub use commands::add_database_decls; +pub(crate) use values::SQLiteDatabase; diff --git a/crates/nu-command/src/database/values/mod.rs b/crates/nu-command/src/database/values/mod.rs new file mode 100644 index 0000000000..b221889db1 --- /dev/null +++ b/crates/nu-command/src/database/values/mod.rs @@ -0,0 +1,3 @@ +mod sqlite; + +pub(crate) use sqlite::SQLiteDatabase; diff --git a/crates/nu-command/src/database/sqlite.rs b/crates/nu-command/src/database/values/sqlite.rs similarity index 55% rename from crates/nu-command/src/database/sqlite.rs rename to crates/nu-command/src/database/values/sqlite.rs index 550661e6be..b4e5aa272f 100644 --- a/crates/nu-command/src/database/sqlite.rs +++ b/crates/nu-command/src/database/values/sqlite.rs @@ -1,35 +1,188 @@ -use std::path::{Path, PathBuf}; +use std::{ + fs::File, + io::Read, + path::{Path, PathBuf}, +}; -use nu_protocol::{CustomValue, ShellError, Span, Spanned, Value}; +use nu_protocol::{CustomValue, PipelineData, ShellError, Span, Spanned, Value}; use rusqlite::{types::ValueRef, Connection, Row}; -use serde::{Deserialize, Serialize}; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; -#[derive(Debug, Serialize, Deserialize)] +use sqlparser::ast::Query; + +const SQLITE_MAGIC_BYTES: &[u8] = "SQLite format 3\0".as_bytes(); + +#[derive(Debug)] pub struct SQLiteDatabase { // I considered storing a SQLite connection here, but decided against it because // 1) YAGNI, 2) it's not obvious how cloning a connection could work, 3) state // management gets tricky quick. Revisit this approach if we find a compelling use case. path: PathBuf, + pub query: Option, +} + +// Mocked serialization of the object +impl Serialize for SQLiteDatabase { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_none() + } +} + +// Mocked deserialization of the object +impl<'de> Deserialize<'de> for SQLiteDatabase { + fn deserialize(_deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let path = std::path::Path::new(""); + Ok(SQLiteDatabase::new(path)) + } } impl SQLiteDatabase { - pub fn new(path: &Path) -> SQLiteDatabase { - SQLiteDatabase { + pub fn new(path: &Path) -> Self { + Self { path: PathBuf::from(path), + query: None, } } + pub fn try_from_path(path: &Path, span: Span) -> Result { + let mut file = File::open(path).map_err(|e| { + ShellError::GenericError( + "Error opening file".into(), + e.to_string(), + Some(span), + None, + Vec::new(), + ) + })?; + + let mut buf: [u8; 16] = [0; 16]; + file.read_exact(&mut buf) + .map_err(|e| { + ShellError::GenericError( + "Error reading file header".into(), + e.to_string(), + Some(span), + None, + Vec::new(), + ) + }) + .and_then(|_| { + if buf == SQLITE_MAGIC_BYTES { + Ok(SQLiteDatabase::new(path)) + } else { + Err(ShellError::GenericError( + "Error reading file".into(), + "Not a SQLite file".into(), + Some(span), + None, + Vec::new(), + )) + } + }) + } + pub fn query(&self, sql: &Spanned, call_span: Span) -> Result { let db = open_sqlite_db(&self.path, call_span)?; - to_shell_error( - run_sql_query(db, sql), - "Failed to query SQLite database", - sql.span, - ) + run_sql_query(db, sql).map_err(|e| { + ShellError::GenericError( + "Failed to query SQLite database".into(), + e.to_string(), + Some(sql.span), + None, + Vec::new(), + ) + }) } - pub fn describe(&self) -> String { - format!("A SQLite database at {:?}", self.path) + pub fn collect(&self, call_span: Span) -> Result { + let sql = match &self.query { + Some(query) => Ok(format!("{}", query)), + None => Err(ShellError::GenericError( + "Error collecting from db".into(), + "No query found in connection".into(), + Some(call_span), + None, + Vec::new(), + )), + }?; + + let sql = Spanned { + item: sql, + span: call_span, + }; + + let db = open_sqlite_db(&self.path, call_span)?; + run_sql_query(db, &sql).map_err(|e| { + ShellError::GenericError( + "Failed to query SQLite database".into(), + e.to_string(), + Some(sql.span), + None, + Vec::new(), + ) + }) + } + + pub fn try_from_value(value: Value) -> Result { + match value { + Value::CustomValue { val, span } => match val.as_any().downcast_ref::() { + Some(db) => Ok(Self { + path: db.path.clone(), + query: db.query.clone(), + }), + None => Err(ShellError::CantConvert( + "database".into(), + "non-database".into(), + span, + None, + )), + }, + x => Err(ShellError::CantConvert( + "database".into(), + x.get_type().to_string(), + x.span()?, + None, + )), + } + } + + pub fn try_from_pipeline(input: PipelineData, span: Span) -> Result { + let value = input.into_value(span); + Self::try_from_value(value) + } + + pub fn into_value(self, span: Span) -> Value { + Value::CustomValue { + val: Box::new(self), + span, + } + } + + pub fn describe(&self, span: Span) -> Value { + let cols = vec!["connection".to_string(), "query".to_string()]; + let connection = Value::String { + val: self.path.to_str().unwrap_or("").to_string(), + span, + }; + + let query = match &self.query { + Some(query) => format!("{query}"), + None => "".into(), + }; + + let query = Value::String { val: query, span }; + + Value::Record { + cols, + vals: vec![connection, query], + span, + } } } @@ -37,6 +190,7 @@ impl CustomValue for SQLiteDatabase { fn clone_value(&self, span: Span) -> Value { let cloned = SQLiteDatabase { path: self.path.clone(), + query: self.query.clone(), }; Value::CustomValue { @@ -51,11 +205,15 @@ impl CustomValue for SQLiteDatabase { fn to_base_value(&self, span: Span) -> Result { let db = open_sqlite_db(&self.path, span)?; - to_shell_error( - read_entire_sqlite_db(db, span), - "Failed to read from SQLite database", - span, - ) + read_entire_sqlite_db(db, span).map_err(|e| { + ShellError::GenericError( + "Failed to read from SQLite database".into(), + e.to_string(), + Some(span), + None, + Vec::new(), + ) + }) } fn as_any(&self) -> &dyn std::any::Any { @@ -70,11 +228,15 @@ impl CustomValue for SQLiteDatabase { fn follow_path_string(&self, _column_name: String, span: Span) -> Result { let db = open_sqlite_db(&self.path, span)?; - to_shell_error( - read_single_table(db, _column_name, span), - "Failed to read from SQLite database", - span, - ) + read_single_table(db, _column_name, span).map_err(|e| { + ShellError::GenericError( + "Failed to read from SQLite database".into(), + e.to_string(), + Some(span), + None, + Vec::new(), + ) + }) } fn typetag_name(&self) -> &'static str { @@ -86,31 +248,52 @@ impl CustomValue for SQLiteDatabase { } } -// TODO: is there a more elegant way to map rusqlite errors to ShellErrors? -fn to_shell_error( - result: Result, - message: &str, - span: Span, -) -> Result { - match result { - Ok(val) => Ok(val), - Err(err) => Err(ShellError::GenericError( - message.to_string(), - err.to_string(), - Some(span), - None, - Vec::new(), - )), - } -} - fn open_sqlite_db(path: &Path, call_span: Span) -> Result { let path = path.to_string_lossy().to_string(); - to_shell_error( - Connection::open(path), - "Failed to open SQLite database", - call_span, - ) + + Connection::open(path).map_err(|e| { + ShellError::GenericError( + "Failed to open SQLite database".into(), + e.to_string(), + Some(call_span), + None, + Vec::new(), + ) + }) +} + +fn run_sql_query(conn: Connection, sql: &Spanned) -> Result { + let mut stmt = conn.prepare(&sql.item)?; + let results = stmt.query([])?; + + let nu_records = results + .mapped(|row| Result::Ok(convert_sqlite_row_to_nu_value(row, sql.span))) + .into_iter() + .collect::, rusqlite::Error>>()?; + + Ok(Value::List { + vals: nu_records, + span: sql.span, + }) +} + +fn read_single_table( + conn: Connection, + table_name: String, + call_span: Span, +) -> Result { + let mut stmt = conn.prepare(&format!("SELECT * FROM {}", table_name))?; + let results = stmt.query([])?; + + let nu_records = results + .mapped(|row| Result::Ok(convert_sqlite_row_to_nu_value(row, call_span))) + .into_iter() + .collect::, rusqlite::Error>>()?; + + Ok(Value::List { + vals: nu_records, + span: call_span, + }) } fn read_entire_sqlite_db(conn: Connection, call_span: Span) -> Result { @@ -146,41 +329,6 @@ fn read_entire_sqlite_db(conn: Connection, call_span: Span) -> Result) -> Result { - let mut stmt = conn.prepare(&sql.item)?; - let mut results = stmt.query([])?; - let mut nu_records = Vec::new(); - - while let Some(table_row) = results.next()? { - nu_records.push(convert_sqlite_row_to_nu_value(table_row, sql.span)) - } - - Ok(Value::List { - vals: nu_records, - span: sql.span, - }) -} - -fn read_single_table( - conn: Connection, - table_name: String, - call_span: Span, -) -> Result { - let mut stmt = conn.prepare(&format!("SELECT * FROM {}", table_name))?; - let mut results = stmt.query([])?; - let mut nu_records = Vec::new(); - - while let Some(table_row) = results.next()? { - nu_records.push(convert_sqlite_row_to_nu_value(table_row, call_span)) - } - - Ok(Value::List { - vals: nu_records, - span: call_span, - }) -} - fn convert_sqlite_row_to_nu_value(row: &Row, span: Span) -> Value { let mut vals = Vec::new(); let colnamestr = row.as_ref().column_names().to_vec(); diff --git a/crates/nu-command/src/default_context.rs b/crates/nu-command/src/default_context.rs index a47236767e..a408587654 100644 --- a/crates/nu-command/src/default_context.rs +++ b/crates/nu-command/src/default_context.rs @@ -23,6 +23,11 @@ pub fn create_default_context(cwd: impl AsRef) -> EngineState { #[cfg(feature = "dataframe")] add_dataframe_decls(&mut working_set); + // Database-related + // Adds all related commands to query databases + #[cfg(feature = "database")] + add_database_decls(&mut working_set); + // Core bind_command! { Alias, @@ -361,11 +366,6 @@ pub fn create_default_context(cwd: impl AsRef) -> EngineState { ViewSource, }; - // Database-related - bind_command! { - QueryDb - }; - // Deprecated bind_command! { PivotDeprecated, diff --git a/crates/nu-command/src/filesystem/open.rs b/crates/nu-command/src/filesystem/open.rs index 57792c0be2..5b9d6cc331 100644 --- a/crates/nu-command/src/filesystem/open.rs +++ b/crates/nu-command/src/filesystem/open.rs @@ -1,5 +1,4 @@ use crate::filesystem::util::BufferedReader; -use crate::SQLiteDatabase; use nu_engine::{eval_block, get_full_help, CallExt}; use nu_protocol::ast::Call; use nu_protocol::engine::{Command, EngineState, Stack}; @@ -7,7 +6,10 @@ use nu_protocol::{ Category, Example, IntoPipelineData, PipelineData, RawStream, ShellError, Signature, Spanned, SyntaxShape, Value, }; -use std::io::{BufReader, Read, Seek}; +use std::io::BufReader; + +#[cfg(feature = "database")] +use crate::database::SQLiteDatabase; #[cfg(unix)] use std::os::unix::fs::PermissionsExt; @@ -44,10 +46,8 @@ impl Command for Open { input: PipelineData, ) -> Result { let raw = call.has_flag("raw"); - let call_span = call.head; let ctrlc = engine_state.ctrlc.clone(); - let path = call.opt::>(engine_state, stack, 0)?; let path = if let Some(path) = path { @@ -105,7 +105,17 @@ impl Command for Open { Vec::new(), )) } else { - let mut file = match std::fs::File::open(path) { + #[cfg(feature = "database")] + if !raw { + let res = SQLiteDatabase::try_from_path(path, arg_span) + .map(|db| db.into_value(call.head).into_pipeline_data()); + + if res.is_ok() { + return res; + } + } + + let file = match std::fs::File::open(path) { Ok(file) => file, Err(err) => { return Err(ShellError::GenericError( @@ -118,25 +128,6 @@ impl Command for Open { } }; - // Peek at the file to see if we can detect a SQLite database - if !raw { - let sqlite_magic_bytes = "SQLite format 3\0".as_bytes(); - let mut buf: [u8; 16] = [0; 16]; - - if file.read_exact(&mut buf).is_ok() && buf == sqlite_magic_bytes { - let custom_val = Value::CustomValue { - val: Box::new(SQLiteDatabase::new(path)), - span: call.head, - }; - - return Ok(custom_val.into_pipeline_data()); - } - - if file.rewind().is_err() { - return Err(ShellError::IOError("Failed to rewind file".into())); - }; - } - let buf_reader = BufReader::new(file); let output = PipelineData::ExternalStream { diff --git a/crates/nu-command/src/lib.rs b/crates/nu-command/src/lib.rs index 2d4f8a6a23..74954c4461 100644 --- a/crates/nu-command/src/lib.rs +++ b/crates/nu-command/src/lib.rs @@ -1,6 +1,5 @@ mod conversions; mod core_commands; -mod database; mod date; mod default_context; mod deprecated; @@ -16,7 +15,6 @@ mod math; mod network; mod path; mod platform; -mod query; mod random; mod shells; mod strings; @@ -25,7 +23,6 @@ mod viewers; pub use conversions::*; pub use core_commands::*; -pub use database::*; pub use date::*; pub use default_context::*; pub use deprecated::*; @@ -42,7 +39,6 @@ pub use math::*; pub use network::*; pub use path::*; pub use platform::*; -pub use query::*; pub use random::*; pub use shells::*; pub use strings::*; @@ -54,3 +50,9 @@ mod dataframe; #[cfg(feature = "dataframe")] pub use dataframe::*; + +#[cfg(feature = "database")] +mod database; + +#[cfg(feature = "database")] +pub use database::*; diff --git a/crates/nu-command/src/query/db.rs b/crates/nu-command/src/query/db.rs deleted file mode 100644 index 4f3e7afc8f..0000000000 --- a/crates/nu-command/src/query/db.rs +++ /dev/null @@ -1,96 +0,0 @@ -use nu_engine::CallExt; -use nu_protocol::{ - ast::Call, - engine::{Command, EngineState, Stack}, - Category, Example, PipelineData, ShellError, Signature, Span, Spanned, SyntaxShape, Value, -}; - -use crate::database::SQLiteDatabase; - -#[derive(Clone)] -pub struct SubCommand; - -impl Command for SubCommand { - fn name(&self) -> &str { - "query db" - } - - fn signature(&self) -> Signature { - Signature::build("query db") - .required( - "query", - SyntaxShape::String, - "SQL to execute against the database", - ) - .category(Category::Date) // TODO: change category - } - - fn usage(&self) -> &str { - "Query a database using SQL." - } - - fn search_terms(&self) -> Vec<&str> { - vec!["database", "SQLite"] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - let sql: Spanned = call.req(engine_state, stack, 0)?; - let call_head = call.head; - - input.map( - move |value| query_input(value, call_head, &sql), - engine_state.ctrlc.clone(), - ) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Get 1 table out of a SQLite database", - example: r#"open foo.db | query db "SELECT * FROM Bar""#, - result: None, - }] - } -} - -fn query_input(input: Value, call_span: Span, sql: &Spanned) -> Value { - // this fn has slightly awkward error handling because it needs to jam errors into Value instead of returning a Result - if let Value::CustomValue { - val, - span: input_span, - } = input - { - let sqlite = val.as_any().downcast_ref::(); - - if let Some(db) = sqlite { - return match db.query(sql, call_span) { - Ok(val) => val, - Err(error) => Value::Error { error }, - }; - } - - return Value::Error { - error: ShellError::PipelineMismatch( - "a SQLite database".to_string(), - call_span, - input_span, - ), - }; - } - - match input.span() { - Ok(input_span) => Value::Error { - error: ShellError::PipelineMismatch( - "a SQLite database".to_string(), - call_span, - input_span, - ), - }, - Err(err) => Value::Error { error: err }, - } -} diff --git a/crates/nu-command/src/query/mod.rs b/crates/nu-command/src/query/mod.rs deleted file mode 100644 index 26a4247dea..0000000000 --- a/crates/nu-command/src/query/mod.rs +++ /dev/null @@ -1,3 +0,0 @@ -mod db; - -pub use db::SubCommand as QueryDb; diff --git a/crates/nu-command/tests/commands/mod.rs b/crates/nu-command/tests/commands/mod.rs index e7ff873a73..9d625154f6 100644 --- a/crates/nu-command/tests/commands/mod.rs +++ b/crates/nu-command/tests/commands/mod.rs @@ -39,6 +39,7 @@ mod open; mod parse; mod path; mod prepend; +#[cfg(feature = "database")] mod query; mod random; mod range; diff --git a/crates/nu-command/tests/commands/open.rs b/crates/nu-command/tests/commands/open.rs index cbb6ebe3ba..f2ef3b593e 100644 --- a/crates/nu-command/tests/commands/open.rs +++ b/crates/nu-command/tests/commands/open.rs @@ -108,8 +108,8 @@ fn parses_more_bson_complexity() { // │ 4 │ │ // ╰───┴──────╯ +#[cfg(feature = "database")] #[test] - fn parses_sqlite() { let actual = nu!( cwd: "tests/fixtures/formats", pipeline( @@ -123,6 +123,7 @@ fn parses_sqlite() { assert_eq!(actual.out, "3"); } +#[cfg(feature = "database")] #[test] fn parses_sqlite_get_column_name() { let actual = nu!( diff --git a/crates/nu-command/tests/commands/query/db.rs b/crates/nu-command/tests/commands/query/db.rs index ad5f8638b1..d9eea3f12e 100644 --- a/crates/nu-command/tests/commands/query/db.rs +++ b/crates/nu-command/tests/commands/query/db.rs @@ -1,13 +1,12 @@ use nu_test_support::{nu, pipeline}; #[test] - fn can_query_single_table() { let actual = nu!( cwd: "tests/fixtures/formats", pipeline( r#" open sample.db - | query db "select * from strings" + | db query "select * from strings" | where x =~ ell | length "# @@ -22,7 +21,7 @@ fn invalid_sql_fails() { cwd: "tests/fixtures/formats", pipeline( r#" open sample.db - | query db "select *asdfasdf" + | db query "select *asdfasdf" "# )); @@ -32,11 +31,11 @@ fn invalid_sql_fails() { #[test] fn invalid_input_fails() { let actual = nu!( - cwd: "tests/fixtures/formats", pipeline( + cwd: "tests/fixtures/formats", pipeline( r#" - "foo" | query db "select * from asdf" + "foo" | db query "select * from asdf" "# )); - assert!(actual.err.contains("pipeline_mismatch")); + assert!(actual.err.contains("can't convert string")); } diff --git a/crates/nu-command/tests/commands/where_.rs b/crates/nu-command/tests/commands/where_.rs index 4575dc7887..0ed3441208 100644 --- a/crates/nu-command/tests/commands/where_.rs +++ b/crates/nu-command/tests/commands/where_.rs @@ -41,6 +41,7 @@ fn where_not_in_table() { assert_eq!(actual.out, "4"); } +#[cfg(feature = "database")] #[test] fn binary_operator_comparisons() { let actual = nu!( @@ -109,6 +110,7 @@ fn binary_operator_comparisons() { assert_eq!(actual.out, "42"); } +#[cfg(feature = "database")] #[test] fn contains_operator() { let actual = nu!(