From 36079f1a3ddfc18a2972495aa2877a795d792050 Mon Sep 17 00:00:00 2001 From: JT <547158+jntrnr@users.noreply.github.com> Date: Tue, 4 Jan 2022 13:01:18 +1100 Subject: [PATCH] Port fetch (with fixes) (#660) * Port fetch to engine-q * Fix check for path as a string * Add a timeout flag and fix some span issues * Add a temporary fetch command that returns byte streams. Got rid of async stuff as we're using the blocking feature of tokio * More tweaks for the bytestream * Rewrite fetch using ByteStreams * buffer read on bytes directly Co-authored-by: Stefan Stanciulescu --- Cargo.lock | 361 ++++++++++++++++++++++- Cargo.toml | 2 +- crates/nu-command/Cargo.toml | 3 + crates/nu-command/src/default_context.rs | 3 + crates/nu-command/src/network/fetch.rs | 338 +++++++++++++++++++++ crates/nu-command/src/network/mod.rs | 4 + crates/nu-protocol/src/shell_error.rs | 4 + 7 files changed, 710 insertions(+), 5 deletions(-) create mode 100644 crates/nu-command/src/network/fetch.rs diff --git a/Cargo.lock b/Cargo.lock index 38dd703f65..a1a8194f10 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -354,6 +354,12 @@ version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" +[[package]] +name = "bytes" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8" + [[package]] name = "bytesize" version = "1.1.0" @@ -508,6 +514,16 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" +[[package]] +name = "core-foundation" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6888e10551bb93e424d8df1d07f1a8b4fceb0001a3a4b048bfc47554946f47b3" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation-sys" version = "0.8.3" @@ -635,7 +651,7 @@ checksum = "22813a6dc45b335f9bade10bf7271dc477e81113e89eb251a0bc2a8a81c536e1" dependencies = [ "bstr", "csv-core", - "itoa", + "itoa 0.4.8", "ryu", "serde", ] @@ -922,6 +938,21 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.0.1" @@ -1109,6 +1140,25 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" +[[package]] +name = "h2" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f072413d126e57991455e0a922b31e4c8ba7c2ffbebf6b78b4f8521397d65cd" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "hash32" version = "0.1.1" @@ -1181,6 +1231,40 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e9025058dae765dee5070ec375f591e2ba14638c63feff74f13805a72e523163" +[[package]] +name = "http" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31f4c6746584866f0feabcc69893c5b51beef3831656a968ed7ae254cdc4fd03" +dependencies = [ + "bytes", + "fnv", + "itoa 1.0.1", +] + +[[package]] +name = "http-body" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ff4f84919677303da5f147645dbea6b1881f368d03ac84e1dc09031ebd7b2c6" +dependencies = [ + "bytes", + "http", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acd94fdbe1d4ff688b67b04eee2e17bd50995534a61539e45adfefb45e5e5503" + +[[package]] +name = "httpdate" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" + [[package]] name = "humantime" version = "1.3.0" @@ -1190,6 +1274,43 @@ dependencies = [ "quick-error", ] +[[package]] +name = "hyper" +version = "0.14.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7ec3e62bdc98a2f0393a5048e4c30ef659440ea6e0e572965103e72bd836f55" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa 0.4.8", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper", + "native-tls", + "tokio", + "tokio-native-tls", +] + [[package]] name = "ical" version = "0.7.0" @@ -1264,6 +1385,12 @@ dependencies = [ "ghost", ] +[[package]] +name = "ipnet" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68f2d64f2edebec4ce84ad108148e67e1064789bee435edc5b60ad398714a3a9" + [[package]] name = "is_ci" version = "1.1.1" @@ -1291,6 +1418,12 @@ version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" +[[package]] +name = "itoa" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aab8fc367588b89dcee83ab0fd66b72b50b72fa1904d7095045ace2b0c81c35" + [[package]] name = "jobserver" version = "0.1.24" @@ -1586,6 +1719,12 @@ dependencies = [ "syn", ] +[[package]] +name = "mime" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" + [[package]] name = "miniz_oxide" version = "0.4.4" @@ -1638,6 +1777,24 @@ dependencies = [ "syn", ] +[[package]] +name = "native-tls" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48ba9f7719b5a0f42f338907614285fb5fd70e53858141f69898a1fb7203b24d" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "nix" version = "0.23.0" @@ -1746,6 +1903,7 @@ dependencies = [ "lscolors", "md-5", "meval", + "mime", "nu-ansi-term", "nu-color-config", "nu-engine", @@ -1762,6 +1920,7 @@ dependencies = [ "rand", "rayon", "regex", + "reqwest", "roxmltree", "rust-embed", "serde", @@ -1992,7 +2151,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bafe4179722c2894288ee77a9f044f02811c86af699344c498b0840c698a2465" dependencies = [ "arrayvec 0.4.12", - "itoa", + "itoa 0.4.8", ] [[package]] @@ -2090,6 +2249,20 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" +[[package]] +name = "openssl" +version = "0.10.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c7ae222234c30df141154f159066c5093ff73b63204dcda7121eb082fc56a95" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-sys", +] + [[package]] name = "openssl-probe" version = "0.1.4" @@ -2647,6 +2820,41 @@ dependencies = [ "winapi", ] +[[package]] +name = "reqwest" +version = "0.11.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c4e0a76dc12a116108933f6301b95e83634e0c47b0afbed6abbaa0601e99258" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "http", + "http-body", + "hyper", + "hyper-tls", + "ipnet", + "js-sys", + "lazy_static", + "log", + "mime", + "native-tls", + "percent-encoding", + "pin-project-lite", + "serde", + "serde_json", + "serde_urlencoded", + "tokio", + "tokio-native-tls", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg", +] + [[package]] name = "result" version = "1.0.0" @@ -2741,12 +2949,45 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "schannel" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f05ba609c234e60bee0d547fe94a4c7e9da733d1c962cf6e59efa4cd9c8bc75" +dependencies = [ + "lazy_static", + "winapi", +] + [[package]] name = "scopeguard" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +[[package]] +name = "security-framework" +version = "2.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "525bc1abfda2e1998d152c45cf13e696f76d0a4972310b22fac1658b05df7c87" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9dd14d83160b528b7bfd66439110573efcfbe281b17fc2ca9f39f550d619c7e" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "semver" version = "0.11.0" @@ -2803,7 +3044,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0ffa0837f2dfa6fb90868c2b5468cad482e175f7dad97e7421951e663f2b527" dependencies = [ "indexmap", - "itoa", + "itoa 0.4.8", "ryu", "serde", ] @@ -2824,7 +3065,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "edfa57a7f8d9c1d260a549e7224100f6c43d43f9103e06dd8b4095a9b2b43ce9" dependencies = [ "form_urlencoded", - "itoa", + "itoa 0.4.8", "ryu", "serde", ] @@ -2952,6 +3193,16 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "45456094d1983e2ee2a18fdfebce3189fa451699d0502cb8e3b49dba5ba41451" +[[package]] +name = "socket2" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dc90fe6c7be1a323296982db1836d1ea9e47b6839496dde9a541bc496df3516" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "spin" version = "0.9.2" @@ -3197,6 +3448,45 @@ dependencies = [ "regex", ] +[[package]] +name = "tokio" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbbf1c778ec206785635ce8ad57fe52b3009ae9e0c9f574a728f3049d3e55838" +dependencies = [ + "bytes", + "libc", + "memchr", + "mio", + "num_cpus", + "pin-project-lite", + "winapi", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e99e1983e5d376cd8eb4b66604d2e99e79f5bd988c3055891dcd8c9e2604cc0" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "log", + "pin-project-lite", + "tokio", +] + [[package]] name = "toml" version = "0.5.8" @@ -3206,6 +3496,32 @@ dependencies = [ "serde", ] +[[package]] +name = "tower-service" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" + +[[package]] +name = "tracing" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "375a639232caf30edfc78e8d89b2d4c375515393e7af7e16f01cd96917fb2105" +dependencies = [ + "cfg-if", + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f4ed65637b8390770814083d20756f87bfa2c21bf2f110babdc5438351746e4" +dependencies = [ + "lazy_static", +] + [[package]] name = "trash" version = "2.0.2" @@ -3221,6 +3537,12 @@ dependencies = [ "windows", ] +[[package]] +name = "try-lock" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" + [[package]] name = "typenum" version = "1.14.0" @@ -3428,6 +3750,16 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "want" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +dependencies = [ + "log", + "try-lock", +] + [[package]] name = "wasi" version = "0.9.0+wasi-snapshot-preview1" @@ -3465,6 +3797,18 @@ dependencies = [ "wasm-bindgen-shared", ] +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e8d7523cb1f2a4c96c1317ca690031b714a51cc14e05f712446691f413f5d39" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "wasm-bindgen-macro" version = "0.2.78" @@ -3565,6 +3909,15 @@ dependencies = [ "windows_gen", ] +[[package]] +name = "winreg" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0120db82e8a1e0b9fb3345a539c478767c0048d842860994d96113d5b667bd69" +dependencies = [ + "winapi", +] + [[package]] name = "xmlparser" version = "0.13.3" diff --git a/Cargo.toml b/Cargo.toml index bf6b508638..211d7bf482 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -59,7 +59,7 @@ pretty_assertions = "1.0.0" [features] plugin = ["nu-plugin", "nu-parser/plugin", "nu-command/plugin", "nu-protocol/plugin", "nu-engine/plugin"] - +fetch-support = ["nu-command/fetch"] default = [ "plugin", "inc", diff --git a/crates/nu-command/Cargo.toml b/crates/nu-command/Cargo.toml index d2603de636..0588982a1e 100644 --- a/crates/nu-command/Cargo.toml +++ b/crates/nu-command/Cargo.toml @@ -68,6 +68,8 @@ sha2 = "0.10.0" base64 = "0.13.0" encoding_rs = "0.8.30" num = { version = "0.4.0", optional = true } +reqwest = {version = "0.11", features = ["blocking"], optional = true } +mime = "0.3.16" [target.'cfg(unix)'.dependencies] umask = "1.0.0" @@ -86,6 +88,7 @@ features = [ trash-support = ["trash"] plugin = ["nu-parser/plugin"] dataframe = ["polars", "num"] +fetch = ["reqwest"] [build-dependencies] shadow-rs = "0.8.1" diff --git a/crates/nu-command/src/default_context.rs b/crates/nu-command/src/default_context.rs index 9992f84210..85197f3a61 100644 --- a/crates/nu-command/src/default_context.rs +++ b/crates/nu-command/src/default_context.rs @@ -297,6 +297,9 @@ pub fn create_default_context() -> EngineState { #[cfg(feature = "plugin")] bind_command!(Register); + #[cfg(feature = "fetch")] + bind_command!(Fetch); + // This is a WIP proof of concept // bind_command!(ListGitBranches, Git, GitCheckout, Source); diff --git a/crates/nu-command/src/network/fetch.rs b/crates/nu-command/src/network/fetch.rs new file mode 100644 index 0000000000..c67b882caa --- /dev/null +++ b/crates/nu-command/src/network/fetch.rs @@ -0,0 +1,338 @@ +use base64::encode; +use nu_engine::CallExt; +use nu_protocol::ast::Call; +use nu_protocol::engine::{Command, EngineState, Stack}; +use nu_protocol::ByteStream; + +use nu_protocol::{ + Category, Example, PipelineData, ShellError, Signature, Span, SyntaxShape, Value, +}; + +use std::io::{BufRead, BufReader, Read}; + +use reqwest::StatusCode; +use std::path::PathBuf; +use std::str::FromStr; +use std::time::Duration; + +#[derive(Clone)] +pub struct SubCommand; + +impl Command for SubCommand { + fn name(&self) -> &str { + "fetch" + } + + fn signature(&self) -> Signature { + Signature::build("fetch") + .desc("Load from a URL into a cell, convert to table if possible (avoid by appending '--raw').") + .required( + "URL", + SyntaxShape::String, + "the URL to fetch the contents from", + ) + .named( + "user", + SyntaxShape::Any, + "the username when authenticating", + Some('u'), + ) + .named( + "password", + SyntaxShape::Any, + "the password when authenticating", + Some('p'), + ) + .named("timeout", SyntaxShape::Int, "timeout period in seconds", Some('t')) + .switch("raw", "fetch contents as text rather than a table", Some('r')) + .filter() + .category(Category::Network) + } + + fn usage(&self) -> &str { + "Fetch the contents from a URL (HTTP GET operation)." + } + + fn run( + &self, + engine_state: &EngineState, + stack: &mut Stack, + call: &Call, + input: PipelineData, + ) -> Result { + run_fetch(engine_state, stack, call, input) + } + + fn examples(&self) -> Vec { + vec![ + Example { + description: "Fetch content from url.com", + example: "fetch url.com", + result: None, + }, + Example { + description: "Fetch content from url.com, with username and password", + example: "fetch -u myuser -p mypass url.com", + result: None, + }, + ] + } +} + +struct Arguments { + url: Option, + raw: bool, + user: Option, + password: Option, + timeout: Option, +} + +fn run_fetch( + engine_state: &EngineState, + stack: &mut Stack, + call: &Call, + _input: PipelineData, +) -> Result { + let args = Arguments { + url: Some(call.req(engine_state, stack, 0)?), + raw: call.has_flag("raw"), + user: call.get_flag(engine_state, stack, "user")?, + password: call.get_flag(engine_state, stack, "password")?, + timeout: call.get_flag(engine_state, stack, "timeout")?, + }; + helper(engine_state, stack, call, args) +} + +// Helper function that actually goes to retrieve the resource from the url given +// The Option return a possible file extension which can be used in AutoConvert commands +fn helper( + engine_state: &EngineState, + stack: &mut Stack, + call: &Call, + args: Arguments, +) -> std::result::Result { + let url_value = if let Some(val) = args.url { + val + } else { + return Err(ShellError::UnsupportedInput( + "Expecting a url as a string but got nothing".to_string(), + call.head, + )); + }; + + let span = url_value.span()?; + let requested_url = url_value.as_string()?; + let url = match url::Url::parse(&requested_url) { + Ok(u) => u, + Err(_e) => { + return Err(ShellError::UnsupportedInput( + "Incomplete or incorrect url. Expected a full url, e.g., https://www.example.com" + .to_string(), + span, + )); + } + }; + let user = args.user.clone(); + let password = args.password; + let timeout = args.timeout; + let raw = args.raw; + let login = match (user, password) { + (Some(user), Some(password)) => Some(encode(&format!("{}:{}", user, password))), + (Some(user), _) => Some(encode(&format!("{}:", user))), + _ => None, + }; + + let client = http_client(); + let mut request = client.get(url); + + if let Some(timeout) = timeout { + let val = timeout.as_i64()?; + if val.is_negative() || val < 1 { + return Err(ShellError::UnsupportedInput( + "Timeout value must be an integer and larger than 0".to_string(), + timeout.span().unwrap_or_else(|_| Span::new(0, 0)), + )); + } + + request = request.timeout(Duration::from_secs(val as u64)); + } + + if let Some(login) = login { + request = request.header("Authorization", format!("Basic {}", login)); + } + + match request.send() { + Ok(resp) => { + // let temp = std::fs::File::create("temp_dwl.txt")?; + // let mut b = BufWriter::new(temp); + // let _bytes = resp.copy_to(&mut b); + // let temp1 = std::fs::File::open("temp_dwl.txt")?; + // let a = BufReader::new(temp1); + + // TODO I guess we should check if all bytes were written/read... + match resp.headers().get("content-type") { + Some(content_type) => { + let content_type = content_type.to_str().map_err(|e| { + ShellError::LabeledError( + e.to_string(), + "MIME type were invalid".to_string(), + ) + })?; + let content_type = mime::Mime::from_str(content_type).map_err(|_| { + ShellError::LabeledError( + format!("MIME type unknown: {}", content_type), + "given unknown MIME type".to_string(), + ) + })?; + let ext = match (content_type.type_(), content_type.subtype()) { + (mime::TEXT, mime::PLAIN) => { + let path_extension = url::Url::parse(&requested_url) + .map_err(|_| { + ShellError::LabeledError( + format!("Cannot parse URL: {}", requested_url), + "cannot parse".to_string(), + ) + })? + .path_segments() + .and_then(|segments| segments.last()) + .and_then(|name| if name.is_empty() { None } else { Some(name) }) + .and_then(|name| { + PathBuf::from(name) + .extension() + .map(|name| name.to_string_lossy().to_string()) + }); + path_extension + } + _ => Some(content_type.subtype().to_string()), + }; + + let buffered_input = BufReader::new(resp); + + let output = PipelineData::ByteStream( + ByteStream { + stream: Box::new(BufferedReader { + input: buffered_input, + }), + ctrlc: engine_state.ctrlc.clone(), + }, + span, + None, + ); + + if raw { + return Ok(output); + } + + if let Some(ext) = ext { + match engine_state.find_decl(format!("from {}", ext).as_bytes()) { + Some(converter_id) => engine_state.get_decl(converter_id).run( + engine_state, + stack, + &Call::new(), + output, + ), + None => Ok(output), + } + } else { + Ok(output) + } + } + None => { + let buffered_input = BufReader::new(resp); + + let output = PipelineData::ByteStream( + ByteStream { + stream: Box::new(BufferedReader { + input: buffered_input, + }), + ctrlc: engine_state.ctrlc.clone(), + }, + span, + None, + ); + Ok(output) + } + } + } + Err(e) if e.is_timeout() => Err(ShellError::NetworkFailure( + format!("Request to {} has timed out", requested_url), + span, + )), + Err(e) if e.is_status() => match e.status() { + Some(err_code) if err_code == StatusCode::NOT_FOUND => Err(ShellError::NetworkFailure( + format!("Requested file not found (404): {:?}", requested_url), + span, + )), + Some(err_code) if err_code == StatusCode::MOVED_PERMANENTLY => { + Err(ShellError::NetworkFailure( + format!("Resource moved permanently (301): {:?}", requested_url), + span, + )) + } + Some(err_code) if err_code == StatusCode::BAD_REQUEST => { + Err(ShellError::NetworkFailure( + format!("Bad request (400) to {:?}", requested_url), + span, + )) + } + Some(err_code) if err_code == StatusCode::FORBIDDEN => Err(ShellError::NetworkFailure( + format!("Access forbidden (403) to {:?}", requested_url), + span, + )), + _ => Err(ShellError::NetworkFailure( + format!( + "Cannot make request to {:?}. Error is {:?}", + requested_url, + e.to_string() + ), + span, + )), + }, + Err(e) => Err(ShellError::NetworkFailure( + format!( + "Cannot make request to {:?}. Error is {:?}", + requested_url, + e.to_string() + ), + span, + )), + } +} + +pub struct BufferedReader { + input: BufReader, +} + +impl Iterator for BufferedReader { + type Item = Result, ShellError>; + + fn next(&mut self) -> Option { + let buffer = self.input.fill_buf(); + match buffer { + Ok(s) => { + let result = s.to_vec(); + + let buffer_len = s.len(); + + if buffer_len == 0 { + None + } else { + self.input.consume(buffer_len); + + Some(Ok(result)) + } + } + Err(e) => Some(Err(ShellError::IOError(e.to_string()))), + } + } +} + +// Only panics if the user agent is invalid but we define it statically so either +// it always or never fails +#[allow(clippy::unwrap_used)] +fn http_client() -> reqwest::blocking::Client { + reqwest::blocking::Client::builder() + .user_agent("nushell") + .build() + .unwrap() +} diff --git a/crates/nu-command/src/network/mod.rs b/crates/nu-command/src/network/mod.rs index 897fc3282d..acf96c78bb 100644 --- a/crates/nu-command/src/network/mod.rs +++ b/crates/nu-command/src/network/mod.rs @@ -1,3 +1,7 @@ +#[cfg(feature = "fetch")] +mod fetch; mod url; pub use self::url::*; +#[cfg(feature = "fetch")] +pub use fetch::SubCommand as Fetch; diff --git a/crates/nu-protocol/src/shell_error.rs b/crates/nu-protocol/src/shell_error.rs index 20ec14b200..d5a78c0eaa 100644 --- a/crates/nu-protocol/src/shell_error.rs +++ b/crates/nu-protocol/src/shell_error.rs @@ -152,6 +152,10 @@ pub enum ShellError { #[diagnostic(code(nu::shell::unsupported_input), url(docsrs))] UnsupportedInput(String, #[label("{0}")] Span), + #[error("Network failure")] + #[diagnostic(code(nu::shell::network_failure), url(docsrs))] + NetworkFailure(String, #[label("{0}")] Span), + #[error("Command not found")] #[diagnostic(code(nu::shell::command_not_found), url(docsrs))] CommandNotFound(#[label("command not found")] Span),