diff --git a/Cargo.lock b/Cargo.lock index df1147928a..c9b3fb4fa5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -483,7 +483,7 @@ dependencies = [ "hex", "hmac", "http 0.2.12", - "http 1.2.0", + "http 1.3.1", "once_cell", "percent-encoding", "sha2", @@ -578,7 +578,7 @@ dependencies = [ "aws-smithy-types", "bytes", "http 0.2.12", - "http 1.2.0", + "http 1.3.1", "pin-project-lite", "tokio", "tracing", @@ -595,7 +595,7 @@ dependencies = [ "bytes", "bytes-utils", "http 0.2.12", - "http 1.2.0", + "http 1.3.1", "http-body 0.4.6", "http-body 1.0.1", "http-body-util", @@ -922,6 +922,12 @@ dependencies = [ "shlex", ] +[[package]] +name = "cesu8" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" + [[package]] name = "cexpr" version = "0.6.0" @@ -1111,6 +1117,16 @@ dependencies = [ "supports-color", ] +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", +] + [[package]] name = "comfy-table" version = "7.1.3" @@ -2279,7 +2295,7 @@ dependencies = [ "fnv", "futures-core", "futures-sink", - "http 1.2.0", + "http 1.3.1", "indexmap", "slab", "tokio", @@ -2421,9 +2437,9 @@ dependencies = [ [[package]] name = "http" -version = "1.2.0" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f16ca2af56261c99fba8bac40a10251ce8188205a4c448fbb745a2e4daa76fea" +checksum = "f4a85d31aea989eead29a3aaf9e1115a180df8282431156e533de47660892565" dependencies = [ "bytes", "fnv", @@ -2448,7 +2464,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", - "http 1.2.0", + "http 1.3.1", ] [[package]] @@ -2459,7 +2475,7 @@ checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" dependencies = [ "bytes", "futures-util", - "http 1.2.0", + "http 1.3.1", "http-body 1.0.1", "pin-project-lite", ] @@ -2529,7 +2545,7 @@ dependencies = [ "futures-channel", "futures-util", "h2 0.4.7", - "http 1.2.0", + "http 1.3.1", "http-body 1.0.1", "httparse", "httpdate", @@ -2563,10 +2579,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08afdbb5c31130e3034af566421053ab03787c640246a446327f550d11bcb333" dependencies = [ "futures-util", - "http 1.2.0", + "http 1.3.1", "hyper 1.5.1", "hyper-util", - "rustls 0.23.20", + "rustls 0.23.28", "rustls-native-certs 0.8.1", "rustls-pki-types", "tokio", @@ -2600,7 +2616,7 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "http 1.2.0", + "http 1.3.1", "http-body 1.0.1", "hyper 1.5.1", "pin-project-lite", @@ -2954,6 +2970,28 @@ version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" +[[package]] +name = "jni" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97" +dependencies = [ + "cesu8", + "cfg-if", + "combine", + "jni-sys", + "log", + "thiserror 1.0.69", + "walkdir", + "windows-sys 0.45.0", +] + +[[package]] +name = "jni-sys" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130" + [[package]] name = "jobserver" version = "0.1.32" @@ -3185,9 +3223,9 @@ checksum = "9374ef4228402d4b7e403e5838cb880d9ee663314b0a900d5a6aabf0c213552e" [[package]] name = "log" -version = "0.4.22" +version = "0.4.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" +checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" [[package]] name = "lru" @@ -3437,7 +3475,7 @@ dependencies = [ "assert-json-diff", "bytes", "futures-util", - "http 1.2.0", + "http 1.3.1", "http-body 1.0.1", "http-body-util", "hyper 1.5.1", @@ -3780,6 +3818,7 @@ dependencies = [ "filetime", "fuzzy-matcher", "getrandom 0.2.15", + "http 1.3.1", "human-date-parser", "indexmap", "indicatif", @@ -3832,7 +3871,7 @@ dependencies = [ "rstest", "rstest_reuse", "rusqlite", - "rustls 0.23.20", + "rustls 0.23.28", "rustls-native-certs 0.8.1", "scopeguard", "serde", @@ -3850,7 +3889,7 @@ dependencies = [ "unicode-segmentation", "unicode-width 0.2.0", "update-informer", - "ureq 2.12.1", + "ureq", "url", "uu_cp", "uu_mkdir", @@ -4591,7 +4630,7 @@ dependencies = [ "chrono", "form_urlencoded", "futures", - "http 1.2.0", + "http 1.3.1", "http-body-util", "humantime", "hyper 1.5.1", @@ -5820,7 +5859,7 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash 2.1.0", - "rustls 0.23.20", + "rustls 0.23.28", "socket2", "thiserror 2.0.12", "tokio", @@ -5838,7 +5877,7 @@ dependencies = [ "rand 0.8.5", "ring", "rustc-hash 2.1.0", - "rustls 0.23.20", + "rustls 0.23.28", "rustls-pki-types", "slab", "thiserror 2.0.12", @@ -6138,7 +6177,7 @@ dependencies = [ "futures-core", "futures-util", "h2 0.4.7", - "http 1.2.0", + "http 1.3.1", "http-body 1.0.1", "http-body-util", "hyper 1.5.1", @@ -6154,7 +6193,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.20", + "rustls 0.23.28", "rustls-native-certs 0.8.1", "rustls-pemfile 2.2.0", "rustls-pki-types", @@ -6413,15 +6452,15 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.20" +version = "0.23.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5065c3f250cbd332cd894be57c40fa52387247659b14a2d6041d121547903b1b" +checksum = "7160e3e10bf4535308537f3c4e1641468cd0e485175d6163087c0393c7d46643" dependencies = [ "log", "once_cell", "ring", "rustls-pki-types", - "rustls-webpki 0.102.8", + "rustls-webpki 0.103.3", "subtle", "zeroize", ] @@ -6470,13 +6509,41 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.10.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16f1201b3c9a7ee8039bcadc17b7e605e2945b27eee7631788c1bd2b0643674b" +checksum = "229a4a4c221013e7e1f1a043678c5cc39fe5171437c88fb47151a21e6f5b5c79" dependencies = [ "web-time", + "zeroize", ] +[[package]] +name = "rustls-platform-verifier" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19787cda76408ec5404443dc8b31795c87cd8fec49762dc75fa727740d34acc1" +dependencies = [ + "core-foundation 0.10.0", + "core-foundation-sys", + "jni", + "log", + "once_cell", + "rustls 0.23.28", + "rustls-native-certs 0.8.1", + "rustls-platform-verifier-android", + "rustls-webpki 0.103.3", + "security-framework 3.0.1", + "security-framework-sys", + "webpki-root-certs 0.26.11", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustls-platform-verifier-android" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" + [[package]] name = "rustls-webpki" version = "0.101.7" @@ -6489,9 +6556,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.102.8" +version = "0.103.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9" +checksum = "e4a72fe2bcf7a6ac6fd7d0b9e5cb68aeb7d4c0a0271730218b3e92d43b4eb435" dependencies = [ "ring", "rustls-pki-types", @@ -7469,7 +7536,7 @@ version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f6d0975eaace0cf0fcadee4e4aaa5da15b5c079146f2cffb67c113be122bf37" dependencies = [ - "rustls 0.23.20", + "rustls 0.23.28", "tokio", ] @@ -7778,49 +7845,30 @@ dependencies = [ "semver", "serde", "serde_json", - "ureq 3.0.3", + "ureq", ] [[package]] name = "ureq" -version = "2.12.1" +version = "3.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02d1a66277ed75f640d608235660df48c8e3c19f3b4edb6a263315626cc3c01d" +checksum = "9f0fde9bc91026e381155f8c67cb354bcd35260b2f4a29bcc84639f762760c39" dependencies = [ "base64 0.22.1", + "cookie_store", + "der", "encoding_rs", "flate2", "log", "native-tls", - "once_cell", - "rustls 0.23.20", + "percent-encoding", + "rustls 0.23.28", + "rustls-pemfile 2.2.0", "rustls-pki-types", + "rustls-platform-verifier", "serde", "serde_json", "socks", - "url", - "webpki-roots 0.26.8", -] - -[[package]] -name = "ureq" -version = "3.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "217751151c53226090391713e533d9a5e904ba2570dabaaace29032687589c3e" -dependencies = [ - "base64 0.22.1", - "cc", - "cookie_store", - "der", - "flate2", - "log", - "native-tls", - "percent-encoding", - "rustls 0.23.20", - "rustls-pemfile 2.2.0", - "rustls-pki-types", - "serde", - "serde_json", "ureq-proto", "utf-8", "webpki-root-certs 0.26.11", @@ -7829,12 +7877,12 @@ dependencies = [ [[package]] name = "ureq-proto" -version = "0.3.5" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae239d0a3341aebc94259414d1dc67cfce87d41cbebc816772c91b77902fafa4" +checksum = "59db78ad1923f2b1be62b6da81fe80b173605ca0d57f85da2e005382adf693f7" dependencies = [ "base64 0.22.1", - "http 1.2.0", + "http 1.3.1", "httparse", "log", ] @@ -8606,6 +8654,15 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets 0.42.2", +] + [[package]] name = "windows-sys" version = "0.48.0" @@ -8633,6 +8690,21 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", +] + [[package]] name = "windows-targets" version = "0.48.5" @@ -8673,6 +8745,12 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + [[package]] name = "windows_aarch64_gnullvm" version = "0.48.5" @@ -8685,6 +8763,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + [[package]] name = "windows_aarch64_msvc" version = "0.48.5" @@ -8697,6 +8781,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +[[package]] +name = "windows_i686_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + [[package]] name = "windows_i686_gnu" version = "0.48.5" @@ -8715,6 +8805,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +[[package]] +name = "windows_i686_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + [[package]] name = "windows_i686_msvc" version = "0.48.5" @@ -8727,6 +8823,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + [[package]] name = "windows_x86_64_gnu" version = "0.48.5" @@ -8739,6 +8841,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" @@ -8751,6 +8859,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" + [[package]] name = "windows_x86_64_msvc" version = "0.48.5" diff --git a/Cargo.toml b/Cargo.toml index dede732a9d..45303e8581 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -93,6 +93,7 @@ fancy-regex = "0.14" filesize = "0.2" filetime = "0.2" heck = "0.5.0" +http = "1.3.1" human-date-parser = "0.3.0" indexmap = "2.10" indicatif = "0.17" @@ -150,7 +151,10 @@ rstest = { version = "0.23", default-features = false } rstest_reuse = "0.7" rusqlite = "0.31" rust-embed = "8.7.0" -rustls = { version = "0.23", default-features = false, features = ["std", "tls12"] } +# We have to fix rustls and ureq versions +# because we use unversioned api to allow users set up their own +# crypto providers (grep for "unversioned") +rustls = { version = "=0.23.28", default-features = false, features = ["std", "tls12"] } rustls-native-certs = "0.8" scopeguard = { version = "1.2.0" } serde = { version = "1.0" } @@ -173,7 +177,7 @@ update-informer = { version = "1.3.0", default-features = false, features = ["gi umask = "2.1" unicode-segmentation = "1.12" unicode-width = "0.2" -ureq = { version = "2.12", default-features = false, features = ["socks-proxy"] } +ureq = { version = "=3.0.12", default-features = false, features = ["socks-proxy"] } url = "2.2" uu_cp = "0.0.30" uu_mkdir = "0.0.30" diff --git a/crates/nu-cli/src/completions/completer.rs b/crates/nu-cli/src/completions/completer.rs index 2589047a16..9de1eb1162 100644 --- a/crates/nu-cli/src/completions/completer.rs +++ b/crates/nu-cli/src/completions/completer.rs @@ -753,7 +753,7 @@ impl NuCompleter { Ok(value) => { log::error!( "External completer returned invalid value of type {}", - value.get_type().to_string() + value.get_type() ); Some(vec![]) } diff --git a/crates/nu-cli/src/completions/custom_completions.rs b/crates/nu-cli/src/completions/custom_completions.rs index 6e6d391c54..1cbd6a1f02 100644 --- a/crates/nu-cli/src/completions/custom_completions.rs +++ b/crates/nu-cli/src/completions/custom_completions.rs @@ -140,7 +140,7 @@ impl Completer for CustomCompletion { _ => { log::error!( "Custom completer returned invalid value of type {}", - value.get_type().to_string() + value.get_type() ); return vec![]; } diff --git a/crates/nu-command/Cargo.toml b/crates/nu-command/Cargo.toml index a0450aff1c..b181053672 100644 --- a/crates/nu-command/Cargo.toml +++ b/crates/nu-command/Cargo.toml @@ -61,6 +61,7 @@ encoding_rs = { workspace = true } fancy-regex = { workspace = true } filesize = { workspace = true } filetime = { workspace = true } +http = {workspace = true} human-date-parser = { workspace = true } indexmap = { workspace = true } indicatif = { workspace = true } @@ -192,6 +193,7 @@ os = [ "uu_uname", "uu_whoami", "which", + "ureq/platform-verifier" ] # The dependencies listed below need 'getrandom'. @@ -220,7 +222,7 @@ rustls-tls = [ "dep:rustls-native-certs", "dep:webpki-roots", "update-informer/rustls-tls", - "ureq/tls", # ureq 3 will has the feature rustls instead + "ureq/rustls", ] plugin = ["nu-parser/plugin", "os"] diff --git a/crates/nu-command/src/network/http/client.rs b/crates/nu-command/src/network/http/client.rs index d3e299b7f4..0d7f73accc 100644 --- a/crates/nu-command/src/network/http/client.rs +++ b/crates/nu-command/src/network/http/client.rs @@ -1,26 +1,35 @@ -use crate::{formats::value_to_json_value, network::tls::tls}; +use crate::{ + formats::value_to_json_value, + network::{http::timeout_extractor_reader::UreqTimeoutExtractorReader, tls::tls}, +}; use base64::{ Engine, alphabet, engine::{GeneralPurpose, general_purpose::PAD}, }; +use http::StatusCode; +use log::error; use multipart_rs::MultipartWriter; use nu_engine::command_prelude::*; use nu_protocol::{ByteStream, LabeledError, Signals, shell_error::io::IoError}; use serde_json::Value as JsonValue; use std::{ collections::HashMap, - error::Error as StdError, io::Cursor, path::PathBuf, str::FromStr, sync::mpsc::{self, RecvTimeoutError}, time::Duration, }; -use ureq::{Error, ErrorKind, Request, Response}; +use ureq::{ + Body, Error, RequestBuilder, ResponseExt, SendBody, + typestate::{WithBody, WithoutBody}, +}; use url::Url; const HTTP_DOCS: &str = "https://www.nushell.sh/cookbook/http.html"; +type Response = http::Response; + type ContentType = String; #[derive(Debug, PartialEq, Eq)] @@ -43,6 +52,20 @@ impl From> for BodyType { } } +trait GetHeader { + fn header(&self, key: &str) -> Option<&str>; +} + +impl GetHeader for Response { + fn header(&self, key: &str) -> Option<&str> { + self.headers().get(key).and_then(|v| { + v.to_str() + .map_err(|e| log::error!("Invalid header {e:?}")) + .ok() + }) + } +} + #[derive(Clone, Copy, PartialEq)] pub enum RedirectMode { Follow, @@ -56,21 +79,24 @@ pub fn http_client( engine_state: &EngineState, stack: &mut Stack, ) -> Result { - let mut agent_builder = ureq::builder() + let mut config_builder = ureq::config::Config::builder() .user_agent("nushell") - .tls_connector(std::sync::Arc::new(tls(allow_insecure)?)); + .save_redirect_history(true) + .http_status_as_error(false) + .max_redirects_will_error(false); if let RedirectMode::Manual | RedirectMode::Error = redirect_mode { - agent_builder = agent_builder.redirects(0); + config_builder = config_builder.max_redirects(0); } if let Some(http_proxy) = retrieve_http_proxy_from_env(engine_state, stack) { - if let Ok(proxy) = ureq::Proxy::new(http_proxy) { - agent_builder = agent_builder.proxy(proxy); + if let Ok(proxy) = ureq::Proxy::new(&http_proxy) { + config_builder = config_builder.proxy(Some(proxy)); } }; - Ok(agent_builder.build()) + config_builder = config_builder.tls_config(tls(allow_insecure)?); + Ok(ureq::Agent::new_with_config(config_builder.build())) } pub fn http_parse_url( @@ -92,7 +118,7 @@ pub fn http_parse_url( msg: "Incomplete or incorrect URL. Expected a full URL, e.g., https://www.example.com".to_string(), input: format!("value: '{requested_url:?}'"), msg_span: call.head, - input_span: span + input_span: span, }); } }; @@ -141,7 +167,9 @@ pub fn response_to_buffer( _ => ByteStreamType::Unknown, }; - let reader = response.into_reader(); + let reader = UreqTimeoutExtractorReader { + r: response.into_body().into_reader(), + }; PipelineData::byte_stream( ByteStream::read(reader, span, engine_state.signals().clone(), response_type) @@ -150,11 +178,11 @@ pub fn response_to_buffer( ) } -pub fn request_add_authorization_header( +pub fn request_add_authorization_header( user: Option, password: Option, - mut request: Request, -) -> Request { + mut request: RequestBuilder, +) -> RequestBuilder { let base64_engine = GeneralPurpose::new(&alphabet::STANDARD, PAD); let login = match (user, password) { @@ -177,7 +205,7 @@ pub fn request_add_authorization_header( }; if let Some(login) = login { - request = request.set("Authorization", &format!("Basic {login}")); + request = request.header("Authorization", &format!("Basic {login}")); } request @@ -200,34 +228,45 @@ impl From for ShellErrorOrRequestError { pub enum HttpBody { Value(Value), ByteStream(ByteStream), - None, +} + +pub fn send_request_no_body( + request: RequestBuilder, + span: Span, + signals: &Signals, +) -> (Result, Headers) { + let headers = extract_request_headers(&request); + let request_url = request.uri_ref().cloned().unwrap_or_default().to_string(); + let result = send_cancellable_request(&request_url, Box::new(|| request.call()), span, signals) + .map_err(|e| request_error_to_shell_error(span, e)); + + (result, headers.unwrap_or_default()) } // remove once all commands have been migrated pub fn send_request( engine_state: &EngineState, - request: Request, - http_body: HttpBody, + request: RequestBuilder, + body: HttpBody, content_type: Option, span: Span, signals: &Signals, -) -> Result { - let request_url = request.url().to_string(); +) -> (Result, Headers) { + let mut request_headers = Headers::new(); + let request_url = request.uri_ref().cloned().unwrap_or_default().to_string(); // hard code serialze_types to false because closures probably shouldn't be // deserialized for send_request but it's required by send_json_request let serialze_types = false; - - match http_body { - HttpBody::None => { - send_cancellable_request(&request_url, Box::new(|| request.call()), span, signals) - } + let response = match body { HttpBody::ByteStream(byte_stream) => { let req = if let Some(content_type) = content_type { - request.set("Content-Type", &content_type) + request.header("Content-Type", &content_type) } else { request }; - + if let Some(h) = extract_request_headers(&req) { + request_headers = h; + } send_cancellable_request_bytes(&request_url, req, byte_stream, span, signals) } HttpBody::Value(body) => { @@ -236,11 +275,15 @@ pub fn send_request( // We should set the content_type if there is one available // when the content type is unknown let req = if let BodyType::Unknown(Some(content_type)) = &body_type { - request.clone().set("Content-Type", content_type) + request.header("Content-Type", content_type) } else { request }; + if let Some(h) = extract_request_headers(&req) { + request_headers = h; + } + match body_type { BodyType::Json => send_json_request( engine_state, @@ -260,14 +303,18 @@ pub fn send_request( } } } - } + }; + + let response = response.map_err(|e| request_error_to_shell_error(span, e)); + + (response, request_headers) } fn send_json_request( engine_state: &EngineState, request_url: &str, body: Value, - req: Request, + req: RequestBuilder, span: Span, signals: &Signals, serialize_types: bool, @@ -311,7 +358,7 @@ fn send_json_request( fn send_form_request( request_url: &str, body: Value, - req: Request, + req: RequestBuilder, span: Span, signals: &Signals, ) -> Result { @@ -321,7 +368,7 @@ fn send_form_request( .iter() .map(|(a, b)| (a.as_str(), b.as_str())) .collect::>(); - req.send_form(&data) + req.send_form(data) }; match body { @@ -364,7 +411,7 @@ fn send_form_request( fn send_multipart_request( request_url: &str, body: Value, - req: Request, + req: RequestBuilder, span: Span, signals: &Signals, ) -> Result { @@ -401,7 +448,7 @@ fn send_multipart_request( let (boundary, data) = (builder.boundary, builder.data); let content_type = format!("multipart/form-data; boundary={boundary}"); - move || req.set("Content-Type", &content_type).send_bytes(&data) + move || req.header("Content-Type", &content_type).send(&data) } _ => { return Err(ShellErrorOrRequestError::ShellError( @@ -418,23 +465,17 @@ fn send_multipart_request( fn send_default_request( request_url: &str, body: Value, - req: Request, + req: RequestBuilder, span: Span, signals: &Signals, ) -> Result { match body { - Value::Binary { val, .. } => send_cancellable_request( - request_url, - Box::new(move || req.send_bytes(&val)), - span, - signals, - ), - Value::String { val, .. } => send_cancellable_request( - request_url, - Box::new(move || req.send_string(&val)), - span, - signals, - ), + Value::Binary { val, .. } => { + send_cancellable_request(request_url, Box::new(move || req.send(&val)), span, signals) + } + Value::String { val, .. } => { + send_cancellable_request(request_url, Box::new(move || req.send(&val)), span, signals) + } _ => Err(ShellErrorOrRequestError::ShellError( ShellError::TypeMismatch { err_message: format!("Accepted types: [binary, string]. Check: {HTTP_DOCS}"), @@ -487,7 +528,7 @@ fn send_cancellable_request( // ureq functions can block for a long time (default 30s?) while attempting to make an HTTP connection fn send_cancellable_request_bytes( request_url: &str, - request: Request, + request: ureq::RequestBuilder, byte_stream: ByteStream, span: Span, signals: &Signals, @@ -511,9 +552,11 @@ fn send_cancellable_request_bytes( }) }) .and_then(|reader| { - request.send(reader).map_err(|e| { - ShellErrorOrRequestError::RequestError(request_url_string, Box::new(e)) - }) + request + .send(SendBody::from_owned_reader(reader)) + .map_err(|e| { + ShellErrorOrRequestError::RequestError(request_url_string, Box::new(e)) + }) }); // may fail if the user has cancelled the operation @@ -537,10 +580,10 @@ fn send_cancellable_request_bytes( } } -pub fn request_set_timeout( +pub fn request_set_timeout( timeout: Option, - mut request: Request, -) -> Result { + mut request: RequestBuilder, +) -> Result, ShellError> { if let Some(timeout) = timeout { let val = timeout.as_duration()?; if val.is_negative() || val < 1 { @@ -550,16 +593,19 @@ pub fn request_set_timeout( }); } - request = request.timeout(Duration::from_nanos(val as u64)); + request = request + .config() + .timeout_global(Some(Duration::from_nanos(val as u64))) + .build() } Ok(request) } -pub fn request_add_custom_headers( +pub fn request_add_custom_headers( headers: Option, - mut request: Request, -) -> Result { + mut request: RequestBuilder, +) -> Result, ShellError> { if let Some(headers) = headers { let mut custom_headers: HashMap = HashMap::new(); @@ -611,7 +657,7 @@ pub fn request_add_custom_headers( for (k, v) in custom_headers { if let Ok(s) = v.coerce_into_string() { - request = request.set(&k, &s); + request = request.header(&k, &s); } } } @@ -619,63 +665,57 @@ pub fn request_add_custom_headers( Ok(request) } -fn handle_response_error(span: Span, requested_url: &str, response_err: Error) -> ShellError { - match response_err { - Error::Status(301, _) => ShellError::NetworkFailure { +fn handle_status_error(span: Span, requested_url: &str, status: StatusCode) -> ShellError { + match status { + StatusCode::MOVED_PERMANENTLY => ShellError::NetworkFailure { msg: format!("Resource moved permanently (301): {requested_url:?}"), span, }, - Error::Status(400, _) => ShellError::NetworkFailure { + StatusCode::BAD_REQUEST => ShellError::NetworkFailure { msg: format!("Bad request (400) to {requested_url:?}"), span, }, - Error::Status(403, _) => ShellError::NetworkFailure { + StatusCode::FORBIDDEN => ShellError::NetworkFailure { msg: format!("Access forbidden (403) to {requested_url:?}"), span, }, - Error::Status(404, _) => ShellError::NetworkFailure { + StatusCode::NOT_FOUND => ShellError::NetworkFailure { msg: format!("Requested file not found (404): {requested_url:?}"), span, }, - Error::Status(408, _) => ShellError::NetworkFailure { + StatusCode::REQUEST_TIMEOUT => ShellError::NetworkFailure { msg: format!("Request timeout (408): {requested_url:?}"), span, }, - Error::Status(_, _) => ShellError::NetworkFailure { + c => ShellError::NetworkFailure { msg: format!( "Cannot make request to {:?}. Error is {:?}", requested_url, - response_err.to_string() + c.to_string() ), span, }, + } +} - Error::Transport(t) => { - let generic_network_failure = || ShellError::NetworkFailure { - msg: t.to_string(), - span, - }; - match t.kind() { - ErrorKind::ConnectionFailed => ShellError::NetworkFailure { - msg: format!( - "Cannot make request to {requested_url}, there was an error establishing a connection.", - ), - span, - }, - ErrorKind::Io => 'io: { - let Some(source) = t.source() else { - break 'io generic_network_failure(); - }; - - let Some(io_error) = source.downcast_ref::() else { - break 'io generic_network_failure(); - }; - - ShellError::Io(IoError::new(io_error, span, None)) - } - _ => generic_network_failure(), - } - } +fn handle_response_error(span: Span, requested_url: &str, response_err: Error) -> ShellError { + match response_err { + Error::ConnectionFailed => ShellError::NetworkFailure { + msg: format!( + "Cannot make request to {requested_url}, there was an error establishing a connection.", + ), + span, + }, + Error::Timeout(..) => ShellError::Io(IoError::new( + ErrorKind::from_std(std::io::ErrorKind::TimedOut), + span, + None, + )), + Error::Io(error) => ShellError::Io(IoError::new(error, span, None)), + e => ShellError::NetworkFailure { + msg: e.to_string(), + span, + }, } } @@ -743,31 +783,60 @@ fn transform_response_using_content_type( pub fn check_response_redirection( redirect_mode: RedirectMode, span: Span, - response: &Result, + resp: &Response, ) -> Result<(), ShellError> { - if let Ok(resp) = response { - if RedirectMode::Error == redirect_mode && (300..400).contains(&resp.status()) { - return Err(ShellError::NetworkFailure { - msg: format!( - "Redirect encountered when redirect handling mode was 'error' ({} {})", - resp.status(), - resp.status_text() - ), - span, - }); - } + if RedirectMode::Error == redirect_mode && (300..400).contains(&resp.status().as_u16()) { + return Err(ShellError::NetworkFailure { + msg: format!( + "Redirect encountered when redirect handling mode was 'error' ({})", + resp.status() + ), + span, + }); } + Ok(()) } -fn request_handle_response_content( +pub(crate) fn handle_response_status( + resp: &Response, + redirect_mode: RedirectMode, + requested_url: &str, + span: Span, + allow_errors: bool, +) -> Result<(), ShellError> { + let manual_redirect = redirect_mode == RedirectMode::Manual; + + let is_success = resp.status().is_success() + || allow_errors + || (resp.status().is_redirection() && manual_redirect); + if is_success { + Ok(()) + } else { + Err(handle_status_error(span, requested_url, resp.status())) + } +} + +pub(crate) struct RequestMetadata<'a> { + pub requested_url: &'a str, + pub span: Span, + pub headers: Headers, + pub redirect_mode: RedirectMode, + pub flags: RequestFlags, +} + +pub(crate) fn request_handle_response( engine_state: &EngineState, stack: &mut Stack, - span: Span, - requested_url: &str, - flags: RequestFlags, + RequestMetadata { + requested_url, + span, + headers, + redirect_mode, + flags, + }: RequestMetadata, + resp: Response, - request: Request, ) -> Result { // #response_to_buffer moves "resp" making it impossible to read headers later. // Wrapping it into a closure to call when needed @@ -787,11 +856,18 @@ fn request_handle_response_content( None => Ok(response_to_buffer(response, engine_state, span)), } }; + handle_response_status( + &resp, + redirect_mode, + requested_url, + span, + flags.allow_errors, + )?; if flags.full { let response_status = resp.status(); - let request_headers_value = headers_to_nu(&extract_request_headers(&request), span) + let request_headers_value = headers_to_nu(&headers, span) .and_then(|data| data.into_value(span)) .unwrap_or(Value::nothing(span)); @@ -803,14 +879,23 @@ fn request_handle_response_content( "request" => request_headers_value, "response" => response_headers_value, }; - + let urls = Value::list( + resp.get_redirect_history() + .into_iter() + .flatten() + .map(|v| Value::string(v.to_string(), span)) + .collect(), + span, + ); let body = consume_response_body(resp)?.into_value(span)?; let full_response = Value::record( record! { + "urls" => urls, "headers" => Value::record(headers, span), "body" => body, - "status" => Value::int(response_status as i64, span), + "status" => Value::int(response_status.as_u16().into(), span), + }, span, ); @@ -821,79 +906,58 @@ fn request_handle_response_content( } } -pub fn request_handle_response( - engine_state: &EngineState, - stack: &mut Stack, - span: Span, - requested_url: &str, - flags: RequestFlags, - response: Result, - request: Request, -) -> Result { - match response { - Ok(resp) => request_handle_response_content( - engine_state, - stack, - span, - requested_url, - flags, - resp, - request, - ), - Err(e) => match e { - ShellErrorOrRequestError::ShellError(e) => Err(e), - ShellErrorOrRequestError::RequestError(_, e) => { - if flags.allow_errors { - if let Error::Status(_, resp) = *e { - Ok(request_handle_response_content( - engine_state, - stack, - span, - requested_url, - flags, - resp, - request, - )?) - } else { - Err(handle_response_error(span, requested_url, *e)) - } - } else { - Err(handle_response_error(span, requested_url, *e)) - } - } - }, - } -} - type Headers = HashMap>; -fn extract_request_headers(request: &Request) -> Headers { - request - .header_names() - .iter() +fn extract_request_headers(request: &RequestBuilder) -> Option { + let headers = request.headers_ref()?; + let headers_str = headers + .keys() .map(|name| { ( - name.clone(), - request.all(name).iter().map(|e| e.to_string()).collect(), + name.to_string().clone(), + headers + .get_all(name) + .iter() + .filter_map(|v| { + v.to_str() + .map_err(|e| { + error!("Invalid header {name:?}: {e:?}"); + }) + .ok() + .map(|s| s.to_string()) + }) + .collect(), + ) + }) + .collect(); + Some(headers_str) +} + +pub(crate) fn extract_response_headers(response: &Response) -> Headers { + let header_map = response.headers(); + header_map + .keys() + .map(|name| { + ( + name.to_string().clone(), + header_map + .get_all(name) + .iter() + .filter_map(|v| { + v.to_str() + .map_err(|e| { + error!("Invalid header {name:?}: {e:?}"); + }) + .ok() + .map(|s| s.to_string()) + }) + .collect(), ) }) .collect() } -fn extract_response_headers(response: &Response) -> Headers { - response - .headers_names() - .iter() - .map(|name| { - ( - name.clone(), - response.all(name).iter().map(|e| e.to_string()).collect(), - ) - }) - .collect() -} - -fn headers_to_nu(headers: &Headers, span: Span) -> Result { +pub(crate) fn headers_to_nu(headers: &Headers, span: Span) -> Result { let mut vals = Vec::with_capacity(headers.len()); for (name, values) in headers { @@ -927,18 +991,12 @@ fn headers_to_nu(headers: &Headers, span: Span) -> Result, -) -> Result { - match response { - Ok(resp) => headers_to_nu(&extract_response_headers(&resp), span), - Err(e) => match e { - ShellErrorOrRequestError::ShellError(e) => Err(e), - ShellErrorOrRequestError::RequestError(requested_url, e) => { - Err(handle_response_error(span, &requested_url, *e)) - } - }, +pub(crate) fn request_error_to_shell_error(span: Span, e: ShellErrorOrRequestError) -> ShellError { + match e { + ShellErrorOrRequestError::ShellError(e) => e, + ShellErrorOrRequestError::RequestError(requested_url, e) => { + handle_response_error(span, &requested_url, *e) + } } } diff --git a/crates/nu-command/src/network/http/delete.rs b/crates/nu-command/src/network/http/delete.rs index f58045df9f..136cc824e0 100644 --- a/crates/nu-command/src/network/http/delete.rs +++ b/crates/nu-command/src/network/http/delete.rs @@ -1,7 +1,8 @@ use crate::network::http::client::{ - HttpBody, RequestFlags, check_response_redirection, http_client, http_parse_redirect_mode, - http_parse_url, request_add_authorization_header, request_add_custom_headers, - request_handle_response, request_set_timeout, send_request, + HttpBody, RequestFlags, RequestMetadata, check_response_redirection, http_client, + http_parse_redirect_mode, http_parse_url, request_add_authorization_header, + request_add_custom_headers, request_handle_response, request_set_timeout, send_request, + send_request_no_body, }; use nu_engine::command_prelude::*; @@ -148,7 +149,7 @@ impl Command for HttpDelete { struct Arguments { url: Value, headers: Option, - data: HttpBody, + data: Option, content_type: Option, raw: bool, insecure: bool, @@ -168,13 +169,13 @@ fn run_delete( ) -> Result { let (data, maybe_metadata) = call .get_flag::(engine_state, stack, "data")? - .map(|v| (HttpBody::Value(v), None)) + .map(|v| (Some(HttpBody::Value(v)), None)) .unwrap_or_else(|| match input { - PipelineData::Value(v, metadata) => (HttpBody::Value(v), metadata), + PipelineData::Value(v, metadata) => (Some(HttpBody::Value(v)), metadata), PipelineData::ByteStream(byte_stream, metadata) => { - (HttpBody::ByteStream(byte_stream), metadata) + (Some(HttpBody::ByteStream(byte_stream)), metadata) } - _ => (HttpBody::None, None), + _ => (None, None), }); let content_type = call .get_flag(engine_state, stack, "content-type")? @@ -216,31 +217,43 @@ fn helper( request = request_set_timeout(args.timeout, request)?; request = request_add_authorization_header(args.user, args.password, request); request = request_add_custom_headers(args.headers, request)?; + let (response, request_headers) = match args.data { + None => send_request_no_body(request, call.head, engine_state.signals()), - let response = send_request( - engine_state, - request.clone(), - args.data, - args.content_type, - call.head, - engine_state.signals(), - ); + Some(body) => send_request( + engine_state, + // Nushell allows sending body via delete method, but not via get. + // We should probably unify the behaviour here. + // + // Sending body with DELETE goes against the spec, but might be useful in some cases, + // see [force_send_body] documentation. + request.force_send_body(), + body, + args.content_type, + span, + engine_state.signals(), + ), + }; let request_flags = RequestFlags { raw: args.raw, full: args.full, allow_errors: args.allow_errors, }; + let response = response?; check_response_redirection(redirect_mode, span, &response)?; request_handle_response( engine_state, stack, - span, - &requested_url, - request_flags, + RequestMetadata { + requested_url: &requested_url, + span, + headers: request_headers, + redirect_mode, + flags: request_flags, + }, response, - request, ) } diff --git a/crates/nu-command/src/network/http/get.rs b/crates/nu-command/src/network/http/get.rs index 39b2d6a7f9..ae9303eb65 100644 --- a/crates/nu-command/src/network/http/get.rs +++ b/crates/nu-command/src/network/http/get.rs @@ -1,12 +1,10 @@ use crate::network::http::client::{ - RequestFlags, check_response_redirection, http_client, http_parse_redirect_mode, - http_parse_url, request_add_authorization_header, request_add_custom_headers, - request_handle_response, request_set_timeout, send_request, + RequestFlags, RequestMetadata, check_response_redirection, http_client, + http_parse_redirect_mode, http_parse_url, request_add_authorization_header, + request_add_custom_headers, request_handle_response, request_set_timeout, send_request_no_body, }; use nu_engine::command_prelude::*; -use super::client::HttpBody; - #[derive(Clone)] pub struct HttpGet; @@ -180,15 +178,8 @@ fn helper( request = request_set_timeout(args.timeout, request)?; request = request_add_authorization_header(args.user, args.password, request); request = request_add_custom_headers(args.headers, request)?; - - let response = send_request( - engine_state, - request.clone(), - HttpBody::None, - None, - call.head, - engine_state.signals(), - ); + let (response, request_headers) = + send_request_no_body(request, call.head, engine_state.signals()); let request_flags = RequestFlags { raw: args.raw, @@ -196,15 +187,20 @@ fn helper( allow_errors: args.allow_errors, }; + let response = response?; + check_response_redirection(redirect_mode, span, &response)?; request_handle_response( engine_state, stack, - span, - &requested_url, - request_flags, + RequestMetadata { + requested_url: &requested_url, + span, + headers: request_headers, + redirect_mode, + flags: request_flags, + }, response, - request, ) } diff --git a/crates/nu-command/src/network/http/head.rs b/crates/nu-command/src/network/http/head.rs index 17993fd3fd..a19add071c 100644 --- a/crates/nu-command/src/network/http/head.rs +++ b/crates/nu-command/src/network/http/head.rs @@ -1,8 +1,7 @@ -use super::client::HttpBody; use crate::network::http::client::{ - check_response_redirection, http_client, http_parse_redirect_mode, http_parse_url, - request_add_authorization_header, request_add_custom_headers, request_handle_response_headers, - request_set_timeout, send_request, + check_response_redirection, extract_response_headers, handle_response_status, headers_to_nu, + http_client, http_parse_redirect_mode, http_parse_url, request_add_authorization_header, + request_add_custom_headers, request_set_timeout, send_request_no_body, }; use nu_engine::command_prelude::*; use nu_protocol::Signals; @@ -140,7 +139,6 @@ fn run_head( } // Helper function that actually goes to retrieve the resource from the url given -// The Option return a possible file extension which can be used in AutoConvert commands fn helper( engine_state: &EngineState, stack: &mut Stack, @@ -159,16 +157,11 @@ fn helper( request = request_add_authorization_header(args.user, args.password, request); request = request_add_custom_headers(args.headers, request)?; - let response = send_request( - engine_state, - request, - HttpBody::None, - None, - call.head, - signals, - ); + let (response, _request_headers) = send_request_no_body(request, call.head, signals); + let response = response?; check_response_redirection(redirect_mode, span, &response)?; - request_handle_response_headers(span, response) + handle_response_status(&response, redirect_mode, &requested_url, span, false)?; + headers_to_nu(&extract_response_headers(&response), span) } #[cfg(test)] diff --git a/crates/nu-command/src/network/http/http_.rs b/crates/nu-command/src/network/http/http_.rs index 9b6eab9e8e..837af3675d 100644 --- a/crates/nu-command/src/network/http/http_.rs +++ b/crates/nu-command/src/network/http/http_.rs @@ -69,7 +69,14 @@ impl Command for Http { ) .switch( "full", - "returns the full response instead of only the body", + r#"Returns the record, containing metainformation about the exchange in addition to the response. + Returning record fields: + - urls: list of url redirects this command had to make to get to the destination + - headers.request: list of headers passed when doing the request + - headers.response: list of received headers + - body: the http body of the response + - status: the http status of the response + "#, Some('f'), ) .switch( diff --git a/crates/nu-command/src/network/http/mod.rs b/crates/nu-command/src/network/http/mod.rs index 7e13c53501..fd5a2eaa74 100644 --- a/crates/nu-command/src/network/http/mod.rs +++ b/crates/nu-command/src/network/http/mod.rs @@ -7,6 +7,7 @@ mod options; mod patch; mod post; mod put; +mod timeout_extractor_reader; pub use delete::HttpDelete; pub use get::HttpGet; diff --git a/crates/nu-command/src/network/http/options.rs b/crates/nu-command/src/network/http/options.rs index ab68e54460..68fa75fd6f 100644 --- a/crates/nu-command/src/network/http/options.rs +++ b/crates/nu-command/src/network/http/options.rs @@ -1,11 +1,10 @@ use crate::network::http::client::{ - RedirectMode, RequestFlags, http_client, http_parse_url, request_add_authorization_header, - request_add_custom_headers, request_handle_response, request_set_timeout, send_request, + RedirectMode, RequestFlags, RequestMetadata, http_client, http_parse_url, + request_add_authorization_header, request_add_custom_headers, request_handle_response, + request_set_timeout, send_request_no_body, }; use nu_engine::command_prelude::*; -use super::client::HttpBody; - #[derive(Clone)] pub struct HttpOptions; @@ -152,22 +151,18 @@ fn helper( ) -> Result { let span = args.url.span(); let (requested_url, _) = http_parse_url(call, span, args.url)?; - - let client = http_client(args.insecure, RedirectMode::Follow, engine_state, stack)?; - let mut request = client.request("OPTIONS", &requested_url); + let redirect_mode = RedirectMode::Follow; + let client = http_client(args.insecure, redirect_mode, engine_state, stack)?; + let mut request = client.options(&requested_url); request = request_set_timeout(args.timeout, request)?; request = request_add_authorization_header(args.user, args.password, request); request = request_add_custom_headers(args.headers, request)?; - let response = send_request( - engine_state, - request.clone(), - HttpBody::None, - None, - call.head, - engine_state.signals(), - ); + let (response, request_headers) = + send_request_no_body(request, call.head, engine_state.signals()); + + let response = response?; // http options' response always showed in header, so we set full to true. // And `raw` is useless too because options method doesn't return body, here we set to true @@ -181,11 +176,14 @@ fn helper( request_handle_response( engine_state, stack, - span, - &requested_url, - request_flags, + RequestMetadata { + requested_url: &requested_url, + span, + headers: request_headers, + redirect_mode, + flags: request_flags, + }, response, - request, ) } diff --git a/crates/nu-command/src/network/http/patch.rs b/crates/nu-command/src/network/http/patch.rs index 2a2c573eb2..3cbc7be6f7 100644 --- a/crates/nu-command/src/network/http/patch.rs +++ b/crates/nu-command/src/network/http/patch.rs @@ -1,7 +1,7 @@ use crate::network::http::client::{ - HttpBody, RequestFlags, check_response_redirection, http_client, http_parse_redirect_mode, - http_parse_url, request_add_authorization_header, request_add_custom_headers, - request_handle_response, request_set_timeout, send_request, + HttpBody, RequestFlags, RequestMetadata, check_response_redirection, http_client, + http_parse_redirect_mode, http_parse_url, request_add_authorization_header, + request_add_custom_headers, request_handle_response, request_set_timeout, send_request, }; use nu_engine::command_prelude::*; @@ -159,19 +159,19 @@ fn run_patch( ) -> Result { let (data, maybe_metadata) = call .opt::(engine_state, stack, 1)? - .map(|v| (HttpBody::Value(v), None)) + .map(|v| (Some(HttpBody::Value(v)), None)) .unwrap_or_else(|| match input { - PipelineData::Value(v, metadata) => (HttpBody::Value(v), metadata), + PipelineData::Value(v, metadata) => (Some(HttpBody::Value(v)), metadata), PipelineData::ByteStream(byte_stream, metadata) => { - (HttpBody::ByteStream(byte_stream), metadata) + (Some(HttpBody::ByteStream(byte_stream)), metadata) } - _ => (HttpBody::None, None), + _ => (None, None), }); let content_type = call .get_flag(engine_state, stack, "content-type")? .or_else(|| maybe_metadata.and_then(|m| m.content_type)); - if let HttpBody::None = data { + let Some(data) = data else { return Err(ShellError::GenericError { error: "Data must be provided either through pipeline or positional argument".into(), msg: "".into(), @@ -179,7 +179,7 @@ fn run_patch( help: None, inner: vec![], }); - } + }; let args = Arguments { url: call.req(engine_state, stack, 0)?, @@ -218,9 +218,9 @@ fn helper( request = request_add_authorization_header(args.user, args.password, request); request = request_add_custom_headers(args.headers, request)?; - let response = send_request( + let (response, request_headers) = send_request( engine_state, - request.clone(), + request, args.data, args.content_type, call.head, @@ -233,15 +233,20 @@ fn helper( allow_errors: args.allow_errors, }; + let response = response?; + check_response_redirection(redirect_mode, span, &response)?; request_handle_response( engine_state, stack, - span, - &requested_url, - request_flags, + RequestMetadata { + requested_url: &requested_url, + span, + headers: request_headers, + redirect_mode, + flags: request_flags, + }, response, - request, ) } diff --git a/crates/nu-command/src/network/http/post.rs b/crates/nu-command/src/network/http/post.rs index 7bc8798753..54f1abd677 100644 --- a/crates/nu-command/src/network/http/post.rs +++ b/crates/nu-command/src/network/http/post.rs @@ -1,7 +1,7 @@ use crate::network::http::client::{ - HttpBody, RequestFlags, check_response_redirection, http_client, http_parse_redirect_mode, - http_parse_url, request_add_authorization_header, request_add_custom_headers, - request_handle_response, request_set_timeout, send_request, + HttpBody, RequestFlags, RequestMetadata, check_response_redirection, http_client, + http_parse_redirect_mode, http_parse_url, request_add_authorization_header, + request_add_custom_headers, request_handle_response, request_set_timeout, send_request, }; use nu_engine::command_prelude::*; @@ -169,19 +169,19 @@ pub fn run_post( ) -> Result { let (data, maybe_metadata) = call .opt::(engine_state, stack, 1)? - .map(|v| (HttpBody::Value(v), None)) + .map(|v| (Some(HttpBody::Value(v)), None)) .unwrap_or_else(|| match input { - PipelineData::Value(v, metadata) => (HttpBody::Value(v), metadata), + PipelineData::Value(v, metadata) => (Some(HttpBody::Value(v)), metadata), PipelineData::ByteStream(byte_stream, metadata) => { - (HttpBody::ByteStream(byte_stream), metadata) + (Some(HttpBody::ByteStream(byte_stream)), metadata) } - _ => (HttpBody::None, None), + _ => (None, None), }); let content_type = call .get_flag(engine_state, stack, "content-type")? .or_else(|| maybe_metadata.and_then(|m| m.content_type)); - if let HttpBody::None = data { + let Some(data) = data else { return Err(ShellError::GenericError { error: "Data must be provided either through pipeline or positional argument".into(), msg: "".into(), @@ -189,7 +189,7 @@ pub fn run_post( help: None, inner: vec![], }); - } + }; let args = Arguments { url: call.req(engine_state, stack, 0)?, @@ -228,9 +228,9 @@ fn helper( request = request_add_authorization_header(args.user, args.password, request); request = request_add_custom_headers(args.headers, request)?; - let response = send_request( + let (response, request_headers) = send_request( engine_state, - request.clone(), + request, args.data, args.content_type, call.head, @@ -243,15 +243,20 @@ fn helper( allow_errors: args.allow_errors, }; + let response = response?; + check_response_redirection(redirect_mode, span, &response)?; request_handle_response( engine_state, stack, - span, - &requested_url, - request_flags, + RequestMetadata { + requested_url: &requested_url, + span, + headers: request_headers, + redirect_mode, + flags: request_flags, + }, response, - request, ) } diff --git a/crates/nu-command/src/network/http/put.rs b/crates/nu-command/src/network/http/put.rs index b651bb6185..84533bbb0a 100644 --- a/crates/nu-command/src/network/http/put.rs +++ b/crates/nu-command/src/network/http/put.rs @@ -1,7 +1,7 @@ use crate::network::http::client::{ - HttpBody, RequestFlags, check_response_redirection, http_client, http_parse_redirect_mode, - http_parse_url, request_add_authorization_header, request_add_custom_headers, - request_handle_response, request_set_timeout, send_request, + HttpBody, RequestFlags, RequestMetadata, check_response_redirection, http_client, + http_parse_redirect_mode, http_parse_url, request_add_authorization_header, + request_add_custom_headers, request_handle_response, request_set_timeout, send_request, }; use nu_engine::command_prelude::*; @@ -159,16 +159,16 @@ fn run_put( ) -> Result { let (data, maybe_metadata) = call .opt::(engine_state, stack, 1)? - .map(|v| (HttpBody::Value(v), None)) + .map(|v| (Some(HttpBody::Value(v)), None)) .unwrap_or_else(|| match input { - PipelineData::Value(v, metadata) => (HttpBody::Value(v), metadata), + PipelineData::Value(v, metadata) => (Some(HttpBody::Value(v)), metadata), PipelineData::ByteStream(byte_stream, metadata) => { - (HttpBody::ByteStream(byte_stream), metadata) + (Some(HttpBody::ByteStream(byte_stream)), metadata) } - _ => (HttpBody::None, None), + _ => (None, None), }); - if let HttpBody::None = data { + let Some(data) = data else { return Err(ShellError::GenericError { error: "Data must be provided either through pipeline or positional argument".into(), msg: "".into(), @@ -176,7 +176,7 @@ fn run_put( help: None, inner: vec![], }); - } + }; let content_type = call .get_flag(engine_state, stack, "content-type")? @@ -219,9 +219,9 @@ fn helper( request = request_add_authorization_header(args.user, args.password, request); request = request_add_custom_headers(args.headers, request)?; - let response = send_request( + let (response, request_headers) = send_request( engine_state, - request.clone(), + request, args.data, args.content_type, call.head, @@ -233,16 +233,20 @@ fn helper( full: args.full, allow_errors: args.allow_errors, }; + let response = response?; check_response_redirection(redirect_mode, span, &response)?; request_handle_response( engine_state, stack, - span, - &requested_url, - request_flags, + RequestMetadata { + requested_url: &requested_url, + span, + headers: request_headers, + redirect_mode, + flags: request_flags, + }, response, - request, ) } diff --git a/crates/nu-command/src/network/http/timeout_extractor_reader.rs b/crates/nu-command/src/network/http/timeout_extractor_reader.rs new file mode 100644 index 0000000000..548e0b00ea --- /dev/null +++ b/crates/nu-command/src/network/http/timeout_extractor_reader.rs @@ -0,0 +1,35 @@ +//! Ureq 3.0.12 converts timeout errors into std::io::ErrorKind::Other: +//! But Nushell infrastructure expects std::io::ErrorKind::Timeout when an operation times out. +//! This is an adapter that converts former into latter. + +use std::io::Read; + +/// Convert errors io errors with [std::io::ErrorKind::Other] and [ureq::Error::Timeout] +/// into io errors with [std::io::ErrorKind::Timeout] +pub struct UreqTimeoutExtractorReader { + pub r: R, +} + +impl Read for UreqTimeoutExtractorReader { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + self.r.read(buf).map_err(|e| { + // TODO: if-let chains when rust 1.88 + + // ureq packages time-outs into "other" + if e.kind() != std::io::ErrorKind::Other { + return e; + } + let ureq_err = match e.downcast::() { + Err(e) => return e, + Ok(e) => e, + }; + match ureq_err { + ureq::Error::Timeout(..) => { + std::io::Error::new(std::io::ErrorKind::TimedOut, ureq_err) + } + // package it back + e => std::io::Error::new(std::io::ErrorKind::Other, e), + } + }) + } +} diff --git a/crates/nu-command/src/network/tls/impl_rustls.rs b/crates/nu-command/src/network/tls/impl_rustls.rs index ac22e8bc19..22b48bf80a 100644 --- a/crates/nu-command/src/network/tls/impl_rustls.rs +++ b/crates/nu-command/src/network/tls/impl_rustls.rs @@ -1,17 +1,8 @@ -use std::{ - ops::Deref, - sync::{Arc, LazyLock, OnceLock}, -}; +use std::sync::{Arc, OnceLock}; -use nu_engine::command_prelude::IoError; use nu_protocol::ShellError; -use rustls::{ - DigitallySignedStruct, RootCertStore, SignatureScheme, - client::danger::{HandshakeSignatureValid, ServerCertVerified, ServerCertVerifier}, - crypto::CryptoProvider, - pki_types::{CertificateDer, ServerName, UnixTime}, -}; -use ureq::TlsConnector; +use rustls::crypto::CryptoProvider; +use ureq::tls::{RootCerts, TlsConfig}; // TODO: replace all these generic errors with proper errors @@ -103,149 +94,24 @@ impl NuCryptoProvider { } } -#[cfg(feature = "os")] -static ROOT_CERT_STORE: LazyLock, ShellError>> = LazyLock::new(|| { - let mut roots = RootCertStore::empty(); - - let native_certs = rustls_native_certs::load_native_certs(); - - let errors: Vec<_> = native_certs - .errors - .into_iter() - .map(|err| match err.kind { - rustls_native_certs::ErrorKind::Io { inner, path } => ShellError::Io( - IoError::new_internal_with_path(inner, err.context, nu_protocol::location!(), path), - ), - rustls_native_certs::ErrorKind::Os(error) => ShellError::GenericError { - error: error.to_string(), - msg: err.context.to_string(), - span: None, - help: None, - inner: vec![], - }, - rustls_native_certs::ErrorKind::Pem(error) => ShellError::GenericError { - error: error.to_string(), - msg: err.context.to_string(), - span: None, - help: None, - inner: vec![], - }, - _ => ShellError::GenericError { - error: String::from("unknown error loading native certs"), - msg: err.context.to_string(), - span: None, - help: None, - inner: vec![], - }, - }) - .collect(); - if !errors.is_empty() { - return Err(ShellError::GenericError { - error: String::from("error loading native certs"), - msg: String::from("could not load native certs"), - span: None, - help: None, - inner: errors, - }); - } - - for cert in native_certs.certs { - roots.add(cert).map_err(|err| ShellError::GenericError { - error: err.to_string(), - msg: String::from("could not add root cert"), - span: None, - help: None, - inner: vec![], - })?; - } - - Ok(Arc::new(roots)) -}); - -#[cfg(not(feature = "os"))] -static ROOT_CERT_STORE: LazyLock, ShellError>> = LazyLock::new(|| { - Ok(Arc::new(rustls::RootCertStore { - roots: webpki_roots::TLS_SERVER_ROOTS.to_vec(), - })) -}); - #[doc = include_str!("./tls.rustdoc.md")] -pub fn tls(allow_insecure: bool) -> Result { +pub fn tls(allow_insecure: bool) -> Result { let crypto_provider = CRYPTO_PROVIDER.get()?; + let config = match allow_insecure { + false => { + #[cfg(feature = "os")] + let certs = RootCerts::PlatformVerifier; - let make_protocol_versions_error = |err: rustls::Error| ShellError::GenericError { - error: err.to_string(), - msg: "crypto provider is incompatible with protocol versions".to_string(), - span: None, - help: None, - inner: vec![], + #[cfg(not(feature = "os"))] + let certs = RootCerts::WebPki; + + TlsConfig::builder() + .unversioned_rustls_crypto_provider(crypto_provider) + .root_certs(certs) + .build() + } + true => TlsConfig::builder().disable_verification(true).build(), }; - let client_config = match allow_insecure { - false => rustls::ClientConfig::builder_with_provider(crypto_provider) - .with_safe_default_protocol_versions() - .map_err(make_protocol_versions_error)? - .with_root_certificates(ROOT_CERT_STORE.deref().clone()?) - .with_no_client_auth(), - true => rustls::ClientConfig::builder_with_provider(crypto_provider) - .with_safe_default_protocol_versions() - .map_err(make_protocol_versions_error)? - .dangerous() - .with_custom_certificate_verifier(Arc::new(UnsecureServerCertVerifier)) - .with_no_client_auth(), - }; - - Ok(Arc::new(client_config)) -} - -#[derive(Debug)] -struct UnsecureServerCertVerifier; - -impl ServerCertVerifier for UnsecureServerCertVerifier { - fn verify_server_cert( - &self, - _end_entity: &CertificateDer<'_>, - _intermediates: &[CertificateDer<'_>], - _server_name: &ServerName<'_>, - _ocsp_response: &[u8], - _now: UnixTime, - ) -> Result { - Ok(ServerCertVerified::assertion()) - } - - fn verify_tls12_signature( - &self, - _message: &[u8], - _cert: &CertificateDer<'_>, - _dss: &DigitallySignedStruct, - ) -> Result { - Ok(HandshakeSignatureValid::assertion()) - } - - fn verify_tls13_signature( - &self, - _message: &[u8], - _cert: &CertificateDer<'_>, - _dss: &DigitallySignedStruct, - ) -> Result { - Ok(HandshakeSignatureValid::assertion()) - } - - fn supported_verify_schemes(&self) -> Vec { - vec![ - SignatureScheme::RSA_PKCS1_SHA1, - SignatureScheme::ECDSA_SHA1_Legacy, - SignatureScheme::RSA_PKCS1_SHA256, - SignatureScheme::ECDSA_NISTP256_SHA256, - SignatureScheme::RSA_PKCS1_SHA384, - SignatureScheme::ECDSA_NISTP384_SHA384, - SignatureScheme::RSA_PKCS1_SHA512, - SignatureScheme::ECDSA_NISTP521_SHA512, - SignatureScheme::RSA_PSS_SHA256, - SignatureScheme::RSA_PSS_SHA384, - SignatureScheme::RSA_PSS_SHA512, - SignatureScheme::ED25519, - SignatureScheme::ED448, - ] - } + Ok(config) } diff --git a/crates/nu-command/tests/commands/network/http/delete.rs b/crates/nu-command/tests/commands/network/http/delete.rs index c503745346..efcdfe7b5f 100644 --- a/crates/nu-command/tests/commands/network/http/delete.rs +++ b/crates/nu-command/tests/commands/network/http/delete.rs @@ -57,7 +57,11 @@ fn http_delete_failed_due_to_server_error() { .as_str() )); - assert!(actual.err.contains("Bad request (400)")) + assert!( + actual.err.contains("Bad request (400)"), + "unexpected error: {:?}", + actual.err + ) } #[test] @@ -140,6 +144,14 @@ fn http_delete_timeout() { format!("http delete --max-time 100ms {url}", url = server.url()).as_str() )); - assert!(&actual.err.contains("nu::shell::io::timed_out")); - assert!(&actual.err.contains("Timed out")); + assert!( + &actual.err.contains("nu::shell::io::timed_out"), + "unexpected error : {:?}", + actual.err + ); + assert!( + &actual.err.contains("Timed out"), + "unexpected error : {:?}", + actual.err + ); } diff --git a/crates/nu-command/tests/commands/network/http/get.rs b/crates/nu-command/tests/commands/network/http/get.rs index a789f1e93a..9be6d18df3 100644 --- a/crates/nu-command/tests/commands/network/http/get.rs +++ b/crates/nu-command/tests/commands/network/http/get.rs @@ -334,6 +334,14 @@ fn http_get_timeout() { format!("http get --max-time 100ms {url}", url = server.url()).as_str() )); - assert!(&actual.err.contains("nu::shell::io::timed_out")); - assert!(&actual.err.contains("Timed out")); + assert!( + &actual.err.contains("nu::shell::io::timed_out"), + "unexpected error: {:?}", + actual.err + ); + assert!( + &actual.err.contains("Timed out"), + "unexpected error: {:?}", + actual.err + ); } diff --git a/crates/nu-command/tests/commands/network/http/head.rs b/crates/nu-command/tests/commands/network/http/head.rs index f9af678fc5..4fe3fb7f3d 100644 --- a/crates/nu-command/tests/commands/network/http/head.rs +++ b/crates/nu-command/tests/commands/network/http/head.rs @@ -36,8 +36,11 @@ fn http_head_failed_due_to_server_error() { ) .as_str() )); - - assert!(actual.err.contains("Bad request (400)")) + assert!( + actual.err.contains("Bad request (400)"), + "Unexpected error: {:?}", + actual.err + ) } #[test] diff --git a/crates/nu-command/tests/commands/network/http/post.rs b/crates/nu-command/tests/commands/network/http/post.rs index 29a82e1a3b..8f98f59d99 100644 --- a/crates/nu-command/tests/commands/network/http/post.rs +++ b/crates/nu-command/tests/commands/network/http/post.rs @@ -101,14 +101,15 @@ fn http_post_failed_due_to_unexpected_body() { assert!(actual.err.contains("Cannot make request")) } +const JSON: &str = r#"{ + "foo": "bar" +}"#; + #[test] fn http_post_json_is_success() { let mut server = Server::new(); - let mock = server - .mock("POST", "/") - .match_body(r#"{"foo":"bar"}"#) - .create(); + let mock = server.mock("POST", "/").match_body(JSON).create(); let actual = nu!(format!( r#"http post -t 'application/json' {url} {{foo: 'bar'}}"#, @@ -116,17 +117,14 @@ fn http_post_json_is_success() { )); mock.assert(); - assert!(actual.out.is_empty()) + assert!(actual.out.is_empty(), "Unexpected output {:?}", actual.out) } #[test] fn http_post_json_string_is_success() { let mut server = Server::new(); - let mock = server - .mock("POST", "/") - .match_body(r#"{"foo":"bar"}"#) - .create(); + let mock = server.mock("POST", "/").match_body(JSON).create(); let actual = nu!(format!( r#"http post -t 'application/json' {url} '{{"foo":"bar"}}'"#, @@ -137,14 +135,17 @@ fn http_post_json_string_is_success() { assert!(actual.out.is_empty()) } +const JSON_LIST: &str = r#"[ + { + "foo": "bar" + } +]"#; + #[test] fn http_post_json_list_is_success() { let mut server = Server::new(); - let mock = server - .mock("POST", "/") - .match_body(r#"[{"foo":"bar"}]"#) - .create(); + let mock = server.mock("POST", "/").match_body(JSON_LIST).create(); let actual = nu!(format!( r#"http post -t 'application/json' {url} [{{foo: "bar"}}]"#,