extern/nushell
1
0
Fork 1
mirror of https://github.com/nushell/nushell.git synced 2025-08-10 04:28:06 +02:00
Code Issues Packages Projects Releases Wiki Activity

Bump ureq, get redirect history. (#16078)

Browse Source
This commit is contained in:
Filipp Samoilov
2025-08-02 14:55:37 +03:00
committed by GitHub
parent 1274d1f7e3
commit 3e37922537
21 changed files with 670 additions and 545 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
crates/nu-command/Cargo.toml
View File

@ -61,6 +61,7 @@ encoding_rs = { workspace = true }
fancy-regex = { workspace = true }
filesize = { workspace = true }
filetime = { workspace = true }
http = {workspace = true}
human-date-parser = { workspace = true }
indexmap = { workspace = true }
indicatif = { workspace = true }
@ -192,6 +193,7 @@ os = [
"uu_uname",
"uu_whoami",
"which",
"ureq/platform-verifier"
]
# The dependencies listed below need 'getrandom'.
@ -220,7 +222,7 @@ rustls-tls = [
"dep:rustls-native-certs",
"dep:webpki-roots",
"update-informer/rustls-tls",
"ureq/tls", # ureq 3 will has the feature rustls instead
"ureq/rustls",
]
plugin = ["nu-parser/plugin", "os"]

438
crates/nu-command/src/network/http/client.rs
View File

@ -1,26 +1,35 @@
use crate::{formats::value_to_json_value, network::tls::tls};
use crate::{
formats::value_to_json_value,
network::{http::timeout_extractor_reader::UreqTimeoutExtractorReader, tls::tls},
};
use base64::{
Engine, alphabet,
engine::{GeneralPurpose, general_purpose::PAD},
};
use http::StatusCode;
use log::error;
use multipart_rs::MultipartWriter;
use nu_engine::command_prelude::*;
use nu_protocol::{ByteStream, LabeledError, Signals, shell_error::io::IoError};
use serde_json::Value as JsonValue;
use std::{
collections::HashMap,
error::Error as StdError,
io::Cursor,
path::PathBuf,
str::FromStr,
sync::mpsc::{self, RecvTimeoutError},
time::Duration,
};
use ureq::{Error, ErrorKind, Request, Response};
use ureq::{
Body, Error, RequestBuilder, ResponseExt, SendBody,
typestate::{WithBody, WithoutBody},
};
use url::Url;
const HTTP_DOCS: &str = "https://www.nushell.sh/cookbook/http.html";
type Response = http::Response<Body>;
type ContentType = String;
#[derive(Debug, PartialEq, Eq)]
@ -43,6 +52,20 @@ impl From<Option<ContentType>> for BodyType {
}
}
trait GetHeader {
fn header(&self, key: &str) -> Option<&str>;
}
impl GetHeader for Response {
fn header(&self, key: &str) -> Option<&str> {
self.headers().get(key).and_then(|v| {
v.to_str()
.map_err(|e| log::error!("Invalid header {e:?}"))
.ok()
})
}
}
#[derive(Clone, Copy, PartialEq)]
pub enum RedirectMode {
Follow,
@ -56,21 +79,24 @@ pub fn http_client(
engine_state: &EngineState,
stack: &mut Stack,
) -> Result<ureq::Agent, ShellError> {
let mut agent_builder = ureq::builder()
let mut config_builder = ureq::config::Config::builder()
.user_agent("nushell")
.tls_connector(std::sync::Arc::new(tls(allow_insecure)?));
.save_redirect_history(true)
.http_status_as_error(false)
.max_redirects_will_error(false);
if let RedirectMode::Manual | RedirectMode::Error = redirect_mode {
agent_builder = agent_builder.redirects(0);
config_builder = config_builder.max_redirects(0);
}
if let Some(http_proxy) = retrieve_http_proxy_from_env(engine_state, stack) {
if let Ok(proxy) = ureq::Proxy::new(http_proxy) {
agent_builder = agent_builder.proxy(proxy);
if let Ok(proxy) = ureq::Proxy::new(&http_proxy) {
config_builder = config_builder.proxy(Some(proxy));
}
};
Ok(agent_builder.build())
config_builder = config_builder.tls_config(tls(allow_insecure)?);
Ok(ureq::Agent::new_with_config(config_builder.build()))
}
pub fn http_parse_url(
@ -92,7 +118,7 @@ pub fn http_parse_url(
msg: "Incomplete or incorrect URL. Expected a full URL, e.g., https://www.example.com".to_string(),
input: format!("value: '{requested_url:?}'"),
msg_span: call.head,
input_span: span
input_span: span,
});
}
};
@ -141,7 +167,9 @@ pub fn response_to_buffer(
_ => ByteStreamType::Unknown,
};
let reader = response.into_reader();
let reader = UreqTimeoutExtractorReader {
r: response.into_body().into_reader(),
};
PipelineData::byte_stream(
ByteStream::read(reader, span, engine_state.signals().clone(), response_type)
@ -150,11 +178,11 @@ pub fn response_to_buffer(
)
}
pub fn request_add_authorization_header(
pub fn request_add_authorization_header<B>(
user: Option<String>,
password: Option<String>,
mut request: Request,
) -> Request {
mut request: RequestBuilder<B>,
) -> RequestBuilder<B> {
let base64_engine = GeneralPurpose::new(&alphabet::STANDARD, PAD);
let login = match (user, password) {
@ -177,7 +205,7 @@ pub fn request_add_authorization_header(
};
if let Some(login) = login {
request = request.set("Authorization", &format!("Basic {login}"));
request = request.header("Authorization", &format!("Basic {login}"));
}
request
@ -200,34 +228,45 @@ impl From<ShellError> for ShellErrorOrRequestError {
pub enum HttpBody {
Value(Value),
ByteStream(ByteStream),
None,
}
pub fn send_request_no_body(
request: RequestBuilder<WithoutBody>,
span: Span,
signals: &Signals,
) -> (Result<Response, ShellError>, Headers) {
let headers = extract_request_headers(&request);
let request_url = request.uri_ref().cloned().unwrap_or_default().to_string();
let result = send_cancellable_request(&request_url, Box::new(|| request.call()), span, signals)
.map_err(|e| request_error_to_shell_error(span, e));
(result, headers.unwrap_or_default())
}
// remove once all commands have been migrated
pub fn send_request(
engine_state: &EngineState,
request: Request,
http_body: HttpBody,
request: RequestBuilder<WithBody>,
body: HttpBody,
content_type: Option<String>,
span: Span,
signals: &Signals,
) -> Result<Response, ShellErrorOrRequestError> {
let request_url = request.url().to_string();
) -> (Result<Response, ShellError>, Headers) {
let mut request_headers = Headers::new();
let request_url = request.uri_ref().cloned().unwrap_or_default().to_string();
// hard code serialze_types to false because closures probably shouldn't be
// deserialized for send_request but it's required by send_json_request
let serialze_types = false;
match http_body {
HttpBody::None => {
send_cancellable_request(&request_url, Box::new(|| request.call()), span, signals)
}
let response = match body {
HttpBody::ByteStream(byte_stream) => {
let req = if let Some(content_type) = content_type {
request.set("Content-Type", &content_type)
request.header("Content-Type", &content_type)
} else {
request
};
if let Some(h) = extract_request_headers(&req) {
request_headers = h;
}
send_cancellable_request_bytes(&request_url, req, byte_stream, span, signals)
}
HttpBody::Value(body) => {
@ -236,11 +275,15 @@ pub fn send_request(
// We should set the content_type if there is one available
// when the content type is unknown
let req = if let BodyType::Unknown(Some(content_type)) = &body_type {
request.clone().set("Content-Type", content_type)
request.header("Content-Type", content_type)
} else {
request
};
if let Some(h) = extract_request_headers(&req) {
request_headers = h;
}
match body_type {
BodyType::Json => send_json_request(
engine_state,
@ -260,14 +303,18 @@ pub fn send_request(
}
}
}
}
};
let response = response.map_err(|e| request_error_to_shell_error(span, e));
(response, request_headers)
}
fn send_json_request(
engine_state: &EngineState,
request_url: &str,
body: Value,
req: Request,
req: RequestBuilder<WithBody>,
span: Span,
signals: &Signals,
serialize_types: bool,
@ -311,7 +358,7 @@ fn send_json_request(
fn send_form_request(
request_url: &str,
body: Value,
req: Request,
req: RequestBuilder<WithBody>,
span: Span,
signals: &Signals,
) -> Result<Response, ShellErrorOrRequestError> {
@ -321,7 +368,7 @@ fn send_form_request(
.iter()
.map(|(a, b)| (a.as_str(), b.as_str()))
.collect::<Vec<(&str, &str)>>();
req.send_form(&data)
req.send_form(data)
};
match body {
@ -364,7 +411,7 @@ fn send_form_request(
fn send_multipart_request(
request_url: &str,
body: Value,
req: Request,
req: RequestBuilder<WithBody>,
span: Span,
signals: &Signals,
) -> Result<Response, ShellErrorOrRequestError> {
@ -401,7 +448,7 @@ fn send_multipart_request(
let (boundary, data) = (builder.boundary, builder.data);
let content_type = format!("multipart/form-data; boundary={boundary}");
move || req.set("Content-Type", &content_type).send_bytes(&data)
move || req.header("Content-Type", &content_type).send(&data)
}
_ => {
return Err(ShellErrorOrRequestError::ShellError(
@ -418,23 +465,17 @@ fn send_multipart_request(
fn send_default_request(
request_url: &str,
body: Value,
req: Request,
req: RequestBuilder<WithBody>,
span: Span,
signals: &Signals,
) -> Result<Response, ShellErrorOrRequestError> {
match body {
Value::Binary { val, .. } => send_cancellable_request(
request_url,
Box::new(move || req.send_bytes(&val)),
span,
signals,
),
Value::String { val, .. } => send_cancellable_request(
request_url,
Box::new(move || req.send_string(&val)),
span,
signals,
),
Value::Binary { val, .. } => {
send_cancellable_request(request_url, Box::new(move || req.send(&val)), span, signals)
}
Value::String { val, .. } => {
send_cancellable_request(request_url, Box::new(move || req.send(&val)), span, signals)
}
_ => Err(ShellErrorOrRequestError::ShellError(
ShellError::TypeMismatch {
err_message: format!("Accepted types: [binary, string]. Check: {HTTP_DOCS}"),
@ -487,7 +528,7 @@ fn send_cancellable_request(
// ureq functions can block for a long time (default 30s?) while attempting to make an HTTP connection
fn send_cancellable_request_bytes(
request_url: &str,
request: Request,
request: ureq::RequestBuilder<WithBody>,
byte_stream: ByteStream,
span: Span,
signals: &Signals,
@ -511,9 +552,11 @@ fn send_cancellable_request_bytes(
})
})
.and_then(|reader| {
request.send(reader).map_err(|e| {
ShellErrorOrRequestError::RequestError(request_url_string, Box::new(e))
})
request
.send(SendBody::from_owned_reader(reader))
.map_err(|e| {
ShellErrorOrRequestError::RequestError(request_url_string, Box::new(e))
})
});
// may fail if the user has cancelled the operation
@ -537,10 +580,10 @@ fn send_cancellable_request_bytes(
}
}
pub fn request_set_timeout(
pub fn request_set_timeout<B>(
timeout: Option<Value>,
mut request: Request,
) -> Result<Request, ShellError> {
mut request: RequestBuilder<B>,
) -> Result<RequestBuilder<B>, ShellError> {
if let Some(timeout) = timeout {
let val = timeout.as_duration()?;
if val.is_negative() || val < 1 {
@ -550,16 +593,19 @@ pub fn request_set_timeout(
});
}
request = request.timeout(Duration::from_nanos(val as u64));
request = request
.config()
.timeout_global(Some(Duration::from_nanos(val as u64)))
.build()
}
Ok(request)
}
pub fn request_add_custom_headers(
pub fn request_add_custom_headers<B>(
headers: Option<Value>,
mut request: Request,
) -> Result<Request, ShellError> {
mut request: RequestBuilder<B>,
) -> Result<RequestBuilder<B>, ShellError> {
if let Some(headers) = headers {
let mut custom_headers: HashMap<String, Value> = HashMap::new();
@ -611,7 +657,7 @@ pub fn request_add_custom_headers(
for (k, v) in custom_headers {
if let Ok(s) = v.coerce_into_string() {
request = request.set(&k, &s);
request = request.header(&k, &s);
}
}
}
@ -619,63 +665,57 @@ pub fn request_add_custom_headers(
Ok(request)
}
fn handle_response_error(span: Span, requested_url: &str, response_err: Error) -> ShellError {
match response_err {
Error::Status(301, _) => ShellError::NetworkFailure {
fn handle_status_error(span: Span, requested_url: &str, status: StatusCode) -> ShellError {
match status {
StatusCode::MOVED_PERMANENTLY => ShellError::NetworkFailure {
msg: format!("Resource moved permanently (301): {requested_url:?}"),
span,
},
Error::Status(400, _) => ShellError::NetworkFailure {
StatusCode::BAD_REQUEST => ShellError::NetworkFailure {
msg: format!("Bad request (400) to {requested_url:?}"),
span,
},
Error::Status(403, _) => ShellError::NetworkFailure {
StatusCode::FORBIDDEN => ShellError::NetworkFailure {
msg: format!("Access forbidden (403) to {requested_url:?}"),
span,
},
Error::Status(404, _) => ShellError::NetworkFailure {
StatusCode::NOT_FOUND => ShellError::NetworkFailure {
msg: format!("Requested file not found (404): {requested_url:?}"),
span,
},
Error::Status(408, _) => ShellError::NetworkFailure {
StatusCode::REQUEST_TIMEOUT => ShellError::NetworkFailure {
msg: format!("Request timeout (408): {requested_url:?}"),
span,
},
Error::Status(_, _) => ShellError::NetworkFailure {
c => ShellError::NetworkFailure {
msg: format!(
"Cannot make request to {:?}. Error is {:?}",
requested_url,
response_err.to_string()
c.to_string()
),
span,
},
}
}
Error::Transport(t) => {
let generic_network_failure = || ShellError::NetworkFailure {
msg: t.to_string(),
span,
};
match t.kind() {
ErrorKind::ConnectionFailed => ShellError::NetworkFailure {
msg: format!(
"Cannot make request to {requested_url}, there was an error establishing a connection.",
),
span,
},
ErrorKind::Io => 'io: {
let Some(source) = t.source() else {
break 'io generic_network_failure();
};
let Some(io_error) = source.downcast_ref::<std::io::Error>() else {
break 'io generic_network_failure();
};
ShellError::Io(IoError::new(io_error, span, None))
}
_ => generic_network_failure(),
}
}
fn handle_response_error(span: Span, requested_url: &str, response_err: Error) -> ShellError {
match response_err {
Error::ConnectionFailed => ShellError::NetworkFailure {
msg: format!(
"Cannot make request to {requested_url}, there was an error establishing a connection.",
),
span,
},
Error::Timeout(..) => ShellError::Io(IoError::new(
ErrorKind::from_std(std::io::ErrorKind::TimedOut),
span,
None,
)),
Error::Io(error) => ShellError::Io(IoError::new(error, span, None)),
e => ShellError::NetworkFailure {
msg: e.to_string(),
span,
},
}
}
@ -743,31 +783,60 @@ fn transform_response_using_content_type(
pub fn check_response_redirection(
redirect_mode: RedirectMode,
span: Span,
response: &Result<Response, ShellErrorOrRequestError>,
resp: &Response,
) -> Result<(), ShellError> {
if let Ok(resp) = response {
if RedirectMode::Error == redirect_mode && (300..400).contains(&resp.status()) {
return Err(ShellError::NetworkFailure {
msg: format!(
"Redirect encountered when redirect handling mode was 'error' ({} {})",
resp.status(),
resp.status_text()
),
span,
});
}
if RedirectMode::Error == redirect_mode && (300..400).contains(&resp.status().as_u16()) {
return Err(ShellError::NetworkFailure {
msg: format!(
"Redirect encountered when redirect handling mode was 'error' ({})",
resp.status()
),
span,
});
}
Ok(())
}
fn request_handle_response_content(
pub(crate) fn handle_response_status(
resp: &Response,
redirect_mode: RedirectMode,
requested_url: &str,
span: Span,
allow_errors: bool,
) -> Result<(), ShellError> {
let manual_redirect = redirect_mode == RedirectMode::Manual;
let is_success = resp.status().is_success()
|| allow_errors
|| (resp.status().is_redirection() && manual_redirect);
if is_success {
Ok(())
} else {
Err(handle_status_error(span, requested_url, resp.status()))
}
}
pub(crate) struct RequestMetadata<'a> {
pub requested_url: &'a str,
pub span: Span,
pub headers: Headers,
pub redirect_mode: RedirectMode,
pub flags: RequestFlags,
}
pub(crate) fn request_handle_response(
engine_state: &EngineState,
stack: &mut Stack,
span: Span,
requested_url: &str,
flags: RequestFlags,
RequestMetadata {
requested_url,
span,
headers,
redirect_mode,
flags,
}: RequestMetadata,
resp: Response,
request: Request,
) -> Result<PipelineData, ShellError> {
// #response_to_buffer moves "resp" making it impossible to read headers later.
// Wrapping it into a closure to call when needed
@ -787,11 +856,18 @@ fn request_handle_response_content(
None => Ok(response_to_buffer(response, engine_state, span)),
}
};
handle_response_status(
&resp,
redirect_mode,
requested_url,
span,
flags.allow_errors,
)?;
if flags.full {
let response_status = resp.status();
let request_headers_value = headers_to_nu(&extract_request_headers(&request), span)
let request_headers_value = headers_to_nu(&headers, span)
.and_then(|data| data.into_value(span))
.unwrap_or(Value::nothing(span));
@ -803,14 +879,23 @@ fn request_handle_response_content(
"request" => request_headers_value,
"response" => response_headers_value,
};
let urls = Value::list(
resp.get_redirect_history()
.into_iter()
.flatten()
.map(|v| Value::string(v.to_string(), span))
.collect(),
span,
);
let body = consume_response_body(resp)?.into_value(span)?;
let full_response = Value::record(
record! {
"urls" => urls,
"headers" => Value::record(headers, span),
"body" => body,
"status" => Value::int(response_status as i64, span),
"status" => Value::int(response_status.as_u16().into(), span),
},
span,
);
@ -821,79 +906,58 @@ fn request_handle_response_content(
}
}
pub fn request_handle_response(
engine_state: &EngineState,
stack: &mut Stack,
span: Span,
requested_url: &str,
flags: RequestFlags,
response: Result<Response, ShellErrorOrRequestError>,
request: Request,
) -> Result<PipelineData, ShellError> {
match response {
Ok(resp) => request_handle_response_content(
engine_state,
stack,
span,
requested_url,
flags,
resp,
request,
),
Err(e) => match e {
ShellErrorOrRequestError::ShellError(e) => Err(e),
ShellErrorOrRequestError::RequestError(_, e) => {
if flags.allow_errors {
if let Error::Status(_, resp) = *e {
Ok(request_handle_response_content(
engine_state,
stack,
span,
requested_url,
flags,
resp,
request,
)?)
} else {
Err(handle_response_error(span, requested_url, *e))
}
} else {
Err(handle_response_error(span, requested_url, *e))
}
}
},
}
}
type Headers = HashMap<String, Vec<String>>;
fn extract_request_headers(request: &Request) -> Headers {
request
.header_names()
.iter()
fn extract_request_headers<B>(request: &RequestBuilder<B>) -> Option<Headers> {
let headers = request.headers_ref()?;
let headers_str = headers
.keys()
.map(|name| {
(
name.clone(),
request.all(name).iter().map(|e| e.to_string()).collect(),
name.to_string().clone(),
headers
.get_all(name)
.iter()
.filter_map(|v| {
v.to_str()
.map_err(|e| {
error!("Invalid header {name:?}: {e:?}");
})
.ok()
.map(|s| s.to_string())
})
.collect(),
)
})
.collect();
Some(headers_str)
}
pub(crate) fn extract_response_headers(response: &Response) -> Headers {
let header_map = response.headers();
header_map
.keys()
.map(|name| {
(
name.to_string().clone(),
header_map
.get_all(name)
.iter()
.filter_map(|v| {
v.to_str()
.map_err(|e| {
error!("Invalid header {name:?}: {e:?}");
})
.ok()
.map(|s| s.to_string())
})
.collect(),
)
})
.collect()
}
fn extract_response_headers(response: &Response) -> Headers {
response
.headers_names()
.iter()
.map(|name| {
(
name.clone(),
response.all(name).iter().map(|e| e.to_string()).collect(),
)
})
.collect()
}
fn headers_to_nu(headers: &Headers, span: Span) -> Result<PipelineData, ShellError> {
pub(crate) fn headers_to_nu(headers: &Headers, span: Span) -> Result<PipelineData, ShellError> {
let mut vals = Vec::with_capacity(headers.len());
for (name, values) in headers {
@ -927,18 +991,12 @@ fn headers_to_nu(headers: &Headers, span: Span) -> Result<PipelineData, ShellErr
Ok(Value::list(vals, span).into_pipeline_data())
}
pub fn request_handle_response_headers(
span: Span,
response: Result<Response, ShellErrorOrRequestError>,
) -> Result<PipelineData, ShellError> {
match response {
Ok(resp) => headers_to_nu(&extract_response_headers(&resp), span),
Err(e) => match e {
ShellErrorOrRequestError::ShellError(e) => Err(e),
ShellErrorOrRequestError::RequestError(requested_url, e) => {
Err(handle_response_error(span, &requested_url, *e))
}
},
pub(crate) fn request_error_to_shell_error(span: Span, e: ShellErrorOrRequestError) -> ShellError {
match e {
ShellErrorOrRequestError::ShellError(e) => e,
ShellErrorOrRequestError::RequestError(requested_url, e) => {
handle_response_error(span, &requested_url, *e)
}
}
}

53
crates/nu-command/src/network/http/delete.rs
View File

@ -1,7 +1,8 @@
use crate::network::http::client::{
HttpBody, RequestFlags, check_response_redirection, http_client, http_parse_redirect_mode,
http_parse_url, request_add_authorization_header, request_add_custom_headers,
request_handle_response, request_set_timeout, send_request,
HttpBody, RequestFlags, RequestMetadata, check_response_redirection, http_client,
http_parse_redirect_mode, http_parse_url, request_add_authorization_header,
request_add_custom_headers, request_handle_response, request_set_timeout, send_request,
send_request_no_body,
};
use nu_engine::command_prelude::*;
@ -148,7 +149,7 @@ impl Command for HttpDelete {
struct Arguments {
url: Value,
headers: Option<Value>,
data: HttpBody,
data: Option<HttpBody>,
content_type: Option<String>,
raw: bool,
insecure: bool,
@ -168,13 +169,13 @@ fn run_delete(
) -> Result<PipelineData, ShellError> {
let (data, maybe_metadata) = call
.get_flag::<Value>(engine_state, stack, "data")?
.map(|v| (HttpBody::Value(v), None))
.map(|v| (Some(HttpBody::Value(v)), None))
.unwrap_or_else(|| match input {
PipelineData::Value(v, metadata) => (HttpBody::Value(v), metadata),
PipelineData::Value(v, metadata) => (Some(HttpBody::Value(v)), metadata),
PipelineData::ByteStream(byte_stream, metadata) => {
(HttpBody::ByteStream(byte_stream), metadata)
(Some(HttpBody::ByteStream(byte_stream)), metadata)
}
_ => (HttpBody::None, None),
_ => (None, None),
});
let content_type = call
.get_flag(engine_state, stack, "content-type")?
@ -216,31 +217,43 @@ fn helper(
request = request_set_timeout(args.timeout, request)?;
request = request_add_authorization_header(args.user, args.password, request);
request = request_add_custom_headers(args.headers, request)?;
let (response, request_headers) = match args.data {
None => send_request_no_body(request, call.head, engine_state.signals()),
let response = send_request(
engine_state,
request.clone(),
args.data,
args.content_type,
call.head,
engine_state.signals(),
);
Some(body) => send_request(
engine_state,
// Nushell allows sending body via delete method, but not via get.
// We should probably unify the behaviour here.
//
// Sending body with DELETE goes against the spec, but might be useful in some cases,
// see [force_send_body] documentation.
request.force_send_body(),
body,
args.content_type,
span,
engine_state.signals(),
),
};
let request_flags = RequestFlags {
raw: args.raw,
full: args.full,
allow_errors: args.allow_errors,
};
let response = response?;
check_response_redirection(redirect_mode, span, &response)?;
request_handle_response(
engine_state,
stack,
span,
&requested_url,
request_flags,
RequestMetadata {
requested_url: &requested_url,
span,
headers: request_headers,
redirect_mode,
flags: request_flags,
},
response,
request,
)
}

32
crates/nu-command/src/network/http/get.rs
View File

@ -1,12 +1,10 @@
use crate::network::http::client::{
RequestFlags, check_response_redirection, http_client, http_parse_redirect_mode,
http_parse_url, request_add_authorization_header, request_add_custom_headers,
request_handle_response, request_set_timeout, send_request,
RequestFlags, RequestMetadata, check_response_redirection, http_client,
http_parse_redirect_mode, http_parse_url, request_add_authorization_header,
request_add_custom_headers, request_handle_response, request_set_timeout, send_request_no_body,
};
use nu_engine::command_prelude::*;
use super::client::HttpBody;
#[derive(Clone)]
pub struct HttpGet;
@ -180,15 +178,8 @@ fn helper(
request = request_set_timeout(args.timeout, request)?;
request = request_add_authorization_header(args.user, args.password, request);
request = request_add_custom_headers(args.headers, request)?;
let response = send_request(
engine_state,
request.clone(),
HttpBody::None,
None,
call.head,
engine_state.signals(),
);
let (response, request_headers) =
send_request_no_body(request, call.head, engine_state.signals());
let request_flags = RequestFlags {
raw: args.raw,
@ -196,15 +187,20 @@ fn helper(
allow_errors: args.allow_errors,
};
let response = response?;
check_response_redirection(redirect_mode, span, &response)?;
request_handle_response(
engine_state,
stack,
span,
&requested_url,
request_flags,
RequestMetadata {
requested_url: &requested_url,
span,
headers: request_headers,
redirect_mode,
flags: request_flags,
},
response,
request,
)
}

21
crates/nu-command/src/network/http/head.rs
View File

@ -1,8 +1,7 @@
use super::client::HttpBody;
use crate::network::http::client::{
check_response_redirection, http_client, http_parse_redirect_mode, http_parse_url,
request_add_authorization_header, request_add_custom_headers, request_handle_response_headers,
request_set_timeout, send_request,
check_response_redirection, extract_response_headers, handle_response_status, headers_to_nu,
http_client, http_parse_redirect_mode, http_parse_url, request_add_authorization_header,
request_add_custom_headers, request_set_timeout, send_request_no_body,
};
use nu_engine::command_prelude::*;
use nu_protocol::Signals;
@ -140,7 +139,6 @@ fn run_head(
}
// Helper function that actually goes to retrieve the resource from the url given
// The Option<String> return a possible file extension which can be used in AutoConvert commands
fn helper(
engine_state: &EngineState,
stack: &mut Stack,
@ -159,16 +157,11 @@ fn helper(
request = request_add_authorization_header(args.user, args.password, request);
request = request_add_custom_headers(args.headers, request)?;
let response = send_request(
engine_state,
request,
HttpBody::None,
None,
call.head,
signals,
);
let (response, _request_headers) = send_request_no_body(request, call.head, signals);
let response = response?;
check_response_redirection(redirect_mode, span, &response)?;
request_handle_response_headers(span, response)
handle_response_status(&response, redirect_mode, &requested_url, span, false)?;
headers_to_nu(&extract_response_headers(&response), span)
}
#[cfg(test)]

9
crates/nu-command/src/network/http/http_.rs
View File

@ -69,7 +69,14 @@ impl Command for Http {
)
.switch(
"full",
"returns the full response instead of only the body",
r#"Returns the record, containing metainformation about the exchange in addition to the response.
Returning record fields:
- urls: list of url redirects this command had to make to get to the destination
- headers.request: list of headers passed when doing the request
- headers.response: list of received headers
- body: the http body of the response
- status: the http status of the response
"#,
Some('f'),
)
.switch(

1
crates/nu-command/src/network/http/mod.rs
View File

@ -7,6 +7,7 @@ mod options;
mod patch;
mod post;
mod put;
mod timeout_extractor_reader;
pub use delete::HttpDelete;
pub use get::HttpGet;

36
crates/nu-command/src/network/http/options.rs
View File

@ -1,11 +1,10 @@
use crate::network::http::client::{
RedirectMode, RequestFlags, http_client, http_parse_url, request_add_authorization_header,
request_add_custom_headers, request_handle_response, request_set_timeout, send_request,
RedirectMode, RequestFlags, RequestMetadata, http_client, http_parse_url,
request_add_authorization_header, request_add_custom_headers, request_handle_response,
request_set_timeout, send_request_no_body,
};
use nu_engine::command_prelude::*;
use super::client::HttpBody;
#[derive(Clone)]
pub struct HttpOptions;
@ -152,22 +151,18 @@ fn helper(
) -> Result<PipelineData, ShellError> {
let span = args.url.span();
let (requested_url, _) = http_parse_url(call, span, args.url)?;
let client = http_client(args.insecure, RedirectMode::Follow, engine_state, stack)?;
let mut request = client.request("OPTIONS", &requested_url);
let redirect_mode = RedirectMode::Follow;
let client = http_client(args.insecure, redirect_mode, engine_state, stack)?;
let mut request = client.options(&requested_url);
request = request_set_timeout(args.timeout, request)?;
request = request_add_authorization_header(args.user, args.password, request);
request = request_add_custom_headers(args.headers, request)?;
let response = send_request(
engine_state,
request.clone(),
HttpBody::None,
None,
call.head,
engine_state.signals(),
);
let (response, request_headers) =
send_request_no_body(request, call.head, engine_state.signals());
let response = response?;
// http options' response always showed in header, so we set full to true.
// And `raw` is useless too because options method doesn't return body, here we set to true
@ -181,11 +176,14 @@ fn helper(
request_handle_response(
engine_state,
stack,
span,
&requested_url,
request_flags,
RequestMetadata {
requested_url: &requested_url,
span,
headers: request_headers,
redirect_mode,
flags: request_flags,
},
response,
request,
)
}

35
crates/nu-command/src/network/http/patch.rs
View File

@ -1,7 +1,7 @@
use crate::network::http::client::{
HttpBody, RequestFlags, check_response_redirection, http_client, http_parse_redirect_mode,
http_parse_url, request_add_authorization_header, request_add_custom_headers,
request_handle_response, request_set_timeout, send_request,
HttpBody, RequestFlags, RequestMetadata, check_response_redirection, http_client,
http_parse_redirect_mode, http_parse_url, request_add_authorization_header,
request_add_custom_headers, request_handle_response, request_set_timeout, send_request,
};
use nu_engine::command_prelude::*;
@ -159,19 +159,19 @@ fn run_patch(
) -> Result<PipelineData, ShellError> {
let (data, maybe_metadata) = call
.opt::<Value>(engine_state, stack, 1)?
.map(|v| (HttpBody::Value(v), None))
.map(|v| (Some(HttpBody::Value(v)), None))
.unwrap_or_else(|| match input {
PipelineData::Value(v, metadata) => (HttpBody::Value(v), metadata),
PipelineData::Value(v, metadata) => (Some(HttpBody::Value(v)), metadata),
PipelineData::ByteStream(byte_stream, metadata) => {
(HttpBody::ByteStream(byte_stream), metadata)
(Some(HttpBody::ByteStream(byte_stream)), metadata)
}
_ => (HttpBody::None, None),
_ => (None, None),
});
let content_type = call
.get_flag(engine_state, stack, "content-type")?
.or_else(|| maybe_metadata.and_then(|m| m.content_type));
if let HttpBody::None = data {
let Some(data) = data else {
return Err(ShellError::GenericError {
error: "Data must be provided either through pipeline or positional argument".into(),
msg: "".into(),
@ -179,7 +179,7 @@ fn run_patch(
help: None,
inner: vec![],
});
}
};
let args = Arguments {
url: call.req(engine_state, stack, 0)?,
@ -218,9 +218,9 @@ fn helper(
request = request_add_authorization_header(args.user, args.password, request);
request = request_add_custom_headers(args.headers, request)?;
let response = send_request(
let (response, request_headers) = send_request(
engine_state,
request.clone(),
request,
args.data,
args.content_type,
call.head,
@ -233,15 +233,20 @@ fn helper(
allow_errors: args.allow_errors,
};
let response = response?;
check_response_redirection(redirect_mode, span, &response)?;
request_handle_response(
engine_state,
stack,
span,
&requested_url,
request_flags,
RequestMetadata {
requested_url: &requested_url,
span,
headers: request_headers,
redirect_mode,
flags: request_flags,
},
response,
request,
)
}

35
crates/nu-command/src/network/http/post.rs
View File

@ -1,7 +1,7 @@
use crate::network::http::client::{
HttpBody, RequestFlags, check_response_redirection, http_client, http_parse_redirect_mode,
http_parse_url, request_add_authorization_header, request_add_custom_headers,
request_handle_response, request_set_timeout, send_request,
HttpBody, RequestFlags, RequestMetadata, check_response_redirection, http_client,
http_parse_redirect_mode, http_parse_url, request_add_authorization_header,
request_add_custom_headers, request_handle_response, request_set_timeout, send_request,
};
use nu_engine::command_prelude::*;
@ -169,19 +169,19 @@ pub fn run_post(
) -> Result<PipelineData, ShellError> {
let (data, maybe_metadata) = call
.opt::<Value>(engine_state, stack, 1)?
.map(|v| (HttpBody::Value(v), None))
.map(|v| (Some(HttpBody::Value(v)), None))
.unwrap_or_else(|| match input {
PipelineData::Value(v, metadata) => (HttpBody::Value(v), metadata),
PipelineData::Value(v, metadata) => (Some(HttpBody::Value(v)), metadata),
PipelineData::ByteStream(byte_stream, metadata) => {
(HttpBody::ByteStream(byte_stream), metadata)
(Some(HttpBody::ByteStream(byte_stream)), metadata)
}
_ => (HttpBody::None, None),
_ => (None, None),
});
let content_type = call
.get_flag(engine_state, stack, "content-type")?
.or_else(|| maybe_metadata.and_then(|m| m.content_type));
if let HttpBody::None = data {
let Some(data) = data else {
return Err(ShellError::GenericError {
error: "Data must be provided either through pipeline or positional argument".into(),
msg: "".into(),
@ -189,7 +189,7 @@ pub fn run_post(
help: None,
inner: vec![],
});
}
};
let args = Arguments {
url: call.req(engine_state, stack, 0)?,
@ -228,9 +228,9 @@ fn helper(
request = request_add_authorization_header(args.user, args.password, request);
request = request_add_custom_headers(args.headers, request)?;
let response = send_request(
let (response, request_headers) = send_request(
engine_state,
request.clone(),
request,
args.data,
args.content_type,
call.head,
@ -243,15 +243,20 @@ fn helper(
allow_errors: args.allow_errors,
};
let response = response?;
check_response_redirection(redirect_mode, span, &response)?;
request_handle_response(
engine_state,
stack,
span,
&requested_url,
request_flags,
RequestMetadata {
requested_url: &requested_url,
span,
headers: request_headers,
redirect_mode,
flags: request_flags,
},
response,
request,
)
}

34
crates/nu-command/src/network/http/put.rs
View File

@ -1,7 +1,7 @@
use crate::network::http::client::{
HttpBody, RequestFlags, check_response_redirection, http_client, http_parse_redirect_mode,
http_parse_url, request_add_authorization_header, request_add_custom_headers,
request_handle_response, request_set_timeout, send_request,
HttpBody, RequestFlags, RequestMetadata, check_response_redirection, http_client,
http_parse_redirect_mode, http_parse_url, request_add_authorization_header,
request_add_custom_headers, request_handle_response, request_set_timeout, send_request,
};
use nu_engine::command_prelude::*;
@ -159,16 +159,16 @@ fn run_put(
) -> Result<PipelineData, ShellError> {
let (data, maybe_metadata) = call
.opt::<Value>(engine_state, stack, 1)?
.map(|v| (HttpBody::Value(v), None))
.map(|v| (Some(HttpBody::Value(v)), None))
.unwrap_or_else(|| match input {
PipelineData::Value(v, metadata) => (HttpBody::Value(v), metadata),
PipelineData::Value(v, metadata) => (Some(HttpBody::Value(v)), metadata),
PipelineData::ByteStream(byte_stream, metadata) => {
(HttpBody::ByteStream(byte_stream), metadata)
(Some(HttpBody::ByteStream(byte_stream)), metadata)
}
_ => (HttpBody::None, None),
_ => (None, None),
});
if let HttpBody::None = data {
let Some(data) = data else {
return Err(ShellError::GenericError {
error: "Data must be provided either through pipeline or positional argument".into(),
msg: "".into(),
@ -176,7 +176,7 @@ fn run_put(
help: None,
inner: vec![],
});
}
};
let content_type = call
.get_flag(engine_state, stack, "content-type")?
@ -219,9 +219,9 @@ fn helper(
request = request_add_authorization_header(args.user, args.password, request);
request = request_add_custom_headers(args.headers, request)?;
let response = send_request(
let (response, request_headers) = send_request(
engine_state,
request.clone(),
request,
args.data,
args.content_type,
call.head,
@ -233,16 +233,20 @@ fn helper(
full: args.full,
allow_errors: args.allow_errors,
};
let response = response?;
check_response_redirection(redirect_mode, span, &response)?;
request_handle_response(
engine_state,
stack,
span,
&requested_url,
request_flags,
RequestMetadata {
requested_url: &requested_url,
span,
headers: request_headers,
redirect_mode,
flags: request_flags,
},
response,
request,
)
}

35
crates/nu-command/src/network/http/timeout_extractor_reader.rs Normal file
View File

@ -0,0 +1,35 @@
//! Ureq 3.0.12 converts timeout errors into std::io::ErrorKind::Other: <https://github.com/algesten/ureq/blob/3.0.12/src/error.rs#L193>
//! But Nushell infrastructure expects std::io::ErrorKind::Timeout when an operation times out.
//! This is an adapter that converts former into latter.
use std::io::Read;
/// Convert errors io errors with [std::io::ErrorKind::Other] and [ureq::Error::Timeout]
/// into io errors with [std::io::ErrorKind::Timeout]
pub struct UreqTimeoutExtractorReader<R> {
pub r: R,
}
impl<R: Read> Read for UreqTimeoutExtractorReader<R> {
fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
self.r.read(buf).map_err(|e| {
// TODO: if-let chains when rust 1.88
// ureq packages time-outs into "other"
if e.kind() != std::io::ErrorKind::Other {
return e;
}
let ureq_err = match e.downcast::<ureq::Error>() {
Err(e) => return e,
Ok(e) => e,
};
match ureq_err {
ureq::Error::Timeout(..) => {
std::io::Error::new(std::io::ErrorKind::TimedOut, ureq_err)
}
// package it back
e => std::io::Error::new(std::io::ErrorKind::Other, e),
}
})
}
}

170
crates/nu-command/src/network/tls/impl_rustls.rs
View File

@ -1,17 +1,8 @@
use std::{
ops::Deref,
sync::{Arc, LazyLock, OnceLock},
};
use std::sync::{Arc, OnceLock};
use nu_engine::command_prelude::IoError;
use nu_protocol::ShellError;
use rustls::{
DigitallySignedStruct, RootCertStore, SignatureScheme,
client::danger::{HandshakeSignatureValid, ServerCertVerified, ServerCertVerifier},
crypto::CryptoProvider,
pki_types::{CertificateDer, ServerName, UnixTime},
};
use ureq::TlsConnector;
use rustls::crypto::CryptoProvider;
use ureq::tls::{RootCerts, TlsConfig};
// TODO: replace all these generic errors with proper errors
@ -103,149 +94,24 @@ impl NuCryptoProvider {
}
}
#[cfg(feature = "os")]
static ROOT_CERT_STORE: LazyLock<Result<Arc<RootCertStore>, ShellError>> = LazyLock::new(|| {
let mut roots = RootCertStore::empty();
let native_certs = rustls_native_certs::load_native_certs();
let errors: Vec<_> = native_certs
.errors
.into_iter()
.map(|err| match err.kind {
rustls_native_certs::ErrorKind::Io { inner, path } => ShellError::Io(
IoError::new_internal_with_path(inner, err.context, nu_protocol::location!(), path),
),
rustls_native_certs::ErrorKind::Os(error) => ShellError::GenericError {
error: error.to_string(),
msg: err.context.to_string(),
span: None,
help: None,
inner: vec![],
},
rustls_native_certs::ErrorKind::Pem(error) => ShellError::GenericError {
error: error.to_string(),
msg: err.context.to_string(),
span: None,
help: None,
inner: vec![],
},
_ => ShellError::GenericError {
error: String::from("unknown error loading native certs"),
msg: err.context.to_string(),
span: None,
help: None,
inner: vec![],
},
})
.collect();
if !errors.is_empty() {
return Err(ShellError::GenericError {
error: String::from("error loading native certs"),
msg: String::from("could not load native certs"),
span: None,
help: None,
inner: errors,
});
}
for cert in native_certs.certs {
roots.add(cert).map_err(|err| ShellError::GenericError {
error: err.to_string(),
msg: String::from("could not add root cert"),
span: None,
help: None,
inner: vec![],
})?;
}
Ok(Arc::new(roots))
});
#[cfg(not(feature = "os"))]
static ROOT_CERT_STORE: LazyLock<Result<Arc<RootCertStore>, ShellError>> = LazyLock::new(|| {
Ok(Arc::new(rustls::RootCertStore {
roots: webpki_roots::TLS_SERVER_ROOTS.to_vec(),
}))
});
#[doc = include_str!("./tls.rustdoc.md")]
pub fn tls(allow_insecure: bool) -> Result<impl TlsConnector, ShellError> {
pub fn tls(allow_insecure: bool) -> Result<TlsConfig, ShellError> {
let crypto_provider = CRYPTO_PROVIDER.get()?;
let config = match allow_insecure {
false => {
#[cfg(feature = "os")]
let certs = RootCerts::PlatformVerifier;
let make_protocol_versions_error = |err: rustls::Error| ShellError::GenericError {
error: err.to_string(),
msg: "crypto provider is incompatible with protocol versions".to_string(),
span: None,
help: None,
inner: vec![],
#[cfg(not(feature = "os"))]
let certs = RootCerts::WebPki;
TlsConfig::builder()
.unversioned_rustls_crypto_provider(crypto_provider)
.root_certs(certs)
.build()
}
true => TlsConfig::builder().disable_verification(true).build(),
};
let client_config = match allow_insecure {
false => rustls::ClientConfig::builder_with_provider(crypto_provider)
.with_safe_default_protocol_versions()
.map_err(make_protocol_versions_error)?
.with_root_certificates(ROOT_CERT_STORE.deref().clone()?)
.with_no_client_auth(),
true => rustls::ClientConfig::builder_with_provider(crypto_provider)
.with_safe_default_protocol_versions()
.map_err(make_protocol_versions_error)?
.dangerous()
.with_custom_certificate_verifier(Arc::new(UnsecureServerCertVerifier))
.with_no_client_auth(),
};
Ok(Arc::new(client_config))
}
#[derive(Debug)]
struct UnsecureServerCertVerifier;
impl ServerCertVerifier for UnsecureServerCertVerifier {
fn verify_server_cert(
&self,
_end_entity: &CertificateDer<'_>,
_intermediates: &[CertificateDer<'_>],
_server_name: &ServerName<'_>,
_ocsp_response: &[u8],
_now: UnixTime,
) -> Result<ServerCertVerified, rustls::Error> {
Ok(ServerCertVerified::assertion())
}
fn verify_tls12_signature(
&self,
_message: &[u8],
_cert: &CertificateDer<'_>,
_dss: &DigitallySignedStruct,
) -> Result<HandshakeSignatureValid, rustls::Error> {
Ok(HandshakeSignatureValid::assertion())
}
fn verify_tls13_signature(
&self,
_message: &[u8],
_cert: &CertificateDer<'_>,
_dss: &DigitallySignedStruct,
) -> Result<HandshakeSignatureValid, rustls::Error> {
Ok(HandshakeSignatureValid::assertion())
}
fn supported_verify_schemes(&self) -> Vec<SignatureScheme> {
vec![
SignatureScheme::RSA_PKCS1_SHA1,
SignatureScheme::ECDSA_SHA1_Legacy,
SignatureScheme::RSA_PKCS1_SHA256,
SignatureScheme::ECDSA_NISTP256_SHA256,
SignatureScheme::RSA_PKCS1_SHA384,
SignatureScheme::ECDSA_NISTP384_SHA384,
SignatureScheme::RSA_PKCS1_SHA512,
SignatureScheme::ECDSA_NISTP521_SHA512,
SignatureScheme::RSA_PSS_SHA256,
SignatureScheme::RSA_PSS_SHA384,
SignatureScheme::RSA_PSS_SHA512,
SignatureScheme::ED25519,
SignatureScheme::ED448,
]
}
Ok(config)
}

18
crates/nu-command/tests/commands/network/http/delete.rs
View File

@ -57,7 +57,11 @@ fn http_delete_failed_due_to_server_error() {
.as_str()
));
assert!(actual.err.contains("Bad request (400)"))
assert!(
actual.err.contains("Bad request (400)"),
"unexpected error: {:?}",
actual.err
)
}
#[test]
@ -140,6 +144,14 @@ fn http_delete_timeout() {
format!("http delete --max-time 100ms {url}", url = server.url()).as_str()
));
assert!(&actual.err.contains("nu::shell::io::timed_out"));
assert!(&actual.err.contains("Timed out"));
assert!(
&actual.err.contains("nu::shell::io::timed_out"),
"unexpected error : {:?}",
actual.err
);
assert!(
&actual.err.contains("Timed out"),
"unexpected error : {:?}",
actual.err
);
}

12
crates/nu-command/tests/commands/network/http/get.rs
View File

@ -334,6 +334,14 @@ fn http_get_timeout() {
format!("http get --max-time 100ms {url}", url = server.url()).as_str()
));
assert!(&actual.err.contains("nu::shell::io::timed_out"));
assert!(&actual.err.contains("Timed out"));
assert!(
&actual.err.contains("nu::shell::io::timed_out"),
"unexpected error: {:?}",
actual.err
);
assert!(
&actual.err.contains("Timed out"),
"unexpected error: {:?}",
actual.err
);
}

7
crates/nu-command/tests/commands/network/http/head.rs
View File

@ -36,8 +36,11 @@ fn http_head_failed_due_to_server_error() {
)
.as_str()
));
assert!(actual.err.contains("Bad request (400)"))
assert!(
actual.err.contains("Bad request (400)"),
"Unexpected error: {:?}",
actual.err
)
}
#[test]

27
crates/nu-command/tests/commands/network/http/post.rs
View File

@ -101,14 +101,15 @@ fn http_post_failed_due_to_unexpected_body() {
assert!(actual.err.contains("Cannot make request"))
}
const JSON: &str = r#"{
"foo": "bar"
}"#;
#[test]
fn http_post_json_is_success() {
let mut server = Server::new();
let mock = server
.mock("POST", "/")
.match_body(r#"{"foo":"bar"}"#)
.create();
let mock = server.mock("POST", "/").match_body(JSON).create();
let actual = nu!(format!(
r#"http post -t 'application/json' {url} {{foo: 'bar'}}"#,
@ -116,17 +117,14 @@ fn http_post_json_is_success() {
));
mock.assert();
assert!(actual.out.is_empty())
assert!(actual.out.is_empty(), "Unexpected output {:?}", actual.out)
}
#[test]
fn http_post_json_string_is_success() {
let mut server = Server::new();
let mock = server
.mock("POST", "/")
.match_body(r#"{"foo":"bar"}"#)
.create();
let mock = server.mock("POST", "/").match_body(JSON).create();
let actual = nu!(format!(
r#"http post -t 'application/json' {url} '{{"foo":"bar"}}'"#,
@ -137,14 +135,17 @@ fn http_post_json_string_is_success() {
assert!(actual.out.is_empty())
}
const JSON_LIST: &str = r#"[
{
"foo": "bar"
}
]"#;
#[test]
fn http_post_json_list_is_success() {
let mut server = Server::new();
let mock = server
.mock("POST", "/")
.match_body(r#"[{"foo":"bar"}]"#)
.create();
let mock = server.mock("POST", "/").match_body(JSON_LIST).create();
let actual = nu!(format!(
r#"http post -t 'application/json' {url} [{{foo: "bar"}}]"#,