mirror of
https://github.com/nushell/nushell.git
synced 2025-08-09 05:34:58 +02:00
fetch -> http get and post -> http post (#7796)
# Updated description by @rgwood This PR changes `fetch` to `http get` and `post` to `http post`. `fetch` and `post` are now deprecated. [I surveyed people on Discord](https://discord.com/channels/601130461678272522/601130461678272524/1065706282566307910) and users strongly approved of this change. # Original Description This PR is related to #2741 and my first pull request in rust :) Implemented a new http mod to better http support and alias `fetch` and `post` commands to `http get` and `http post` respectively. # User-Facing Changes Users will be able to use HTTP method via http command, for example ``` shell > http get "https://www.example.com" <!doctype html> <html> ... ```
This commit is contained in:
KazukiY
GitHub
422
crates/nu-command/src/network/http/get.rs
Normal file
422
crates/nu-command/src/network/http/get.rs
Normal file
@ -0,0 +1,422 @@
|
||||
use base64::{alphabet, engine::general_purpose::PAD, engine::GeneralPurpose, Engine};
|
||||
use nu_engine::CallExt;
|
||||
use nu_protocol::ast::Call;
|
||||
use nu_protocol::engine::{Command, EngineState, Stack};
|
||||
use nu_protocol::util::BufferedReader;
|
||||
use nu_protocol::RawStream;
|
||||
use nu_protocol::{
|
||||
Category, Example, PipelineData, ShellError, Signature, Span, SyntaxShape, Type, Value,
|
||||
};
|
||||
use reqwest::blocking::Response;
|
||||
use reqwest::StatusCode;
|
||||
use std::collections::HashMap;
|
||||
use std::io::BufReader;
|
||||
use std::path::PathBuf;
|
||||
use std::str::FromStr;
|
||||
use std::time::Duration;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct SubCommand;
|
||||
|
||||
impl Command for SubCommand {
|
||||
fn name(&self) -> &str {
|
||||
"http get"
|
||||
}
|
||||
|
||||
fn signature(&self) -> Signature {
|
||||
Signature::build("http get")
|
||||
.input_output_types(vec![(Type::Nothing, Type::Any)])
|
||||
.required(
|
||||
"URL",
|
||||
SyntaxShape::String,
|
||||
"the URL to fetch the contents from",
|
||||
)
|
||||
.named(
|
||||
"user",
|
||||
SyntaxShape::Any,
|
||||
"the username when authenticating",
|
||||
Some('u'),
|
||||
)
|
||||
.named(
|
||||
"password",
|
||||
SyntaxShape::Any,
|
||||
"the password when authenticating",
|
||||
Some('p'),
|
||||
)
|
||||
.named(
|
||||
"timeout",
|
||||
SyntaxShape::Int,
|
||||
"timeout period in seconds",
|
||||
Some('t'),
|
||||
)
|
||||
.named(
|
||||
"headers",
|
||||
SyntaxShape::Any,
|
||||
"custom headers you want to add ",
|
||||
Some('H'),
|
||||
)
|
||||
.switch(
|
||||
"raw",
|
||||
"fetch contents as text rather than a table",
|
||||
Some('r'),
|
||||
)
|
||||
.filter()
|
||||
.category(Category::Network)
|
||||
}
|
||||
|
||||
fn usage(&self) -> &str {
|
||||
"Fetch the contents from a URL."
|
||||
}
|
||||
|
||||
fn extra_usage(&self) -> &str {
|
||||
"Performs HTTP GET operation."
|
||||
}
|
||||
|
||||
fn search_terms(&self) -> Vec<&str> {
|
||||
vec![
|
||||
"network", "fetch", "pull", "request", "download", "curl", "wget",
|
||||
]
|
||||
}
|
||||
|
||||
fn run(
|
||||
&self,
|
||||
engine_state: &EngineState,
|
||||
stack: &mut Stack,
|
||||
call: &Call,
|
||||
input: PipelineData,
|
||||
) -> Result<nu_protocol::PipelineData, nu_protocol::ShellError> {
|
||||
run_fetch(engine_state, stack, call, input)
|
||||
}
|
||||
|
||||
fn examples(&self) -> Vec<Example> {
|
||||
vec![
|
||||
Example {
|
||||
description: "http get content from example.com",
|
||||
example: "http get https://www.example.com",
|
||||
result: None,
|
||||
},
|
||||
Example {
|
||||
description: "http get content from example.com, with username and password",
|
||||
example: "http get -u myuser -p mypass https://www.example.com",
|
||||
result: None,
|
||||
},
|
||||
Example {
|
||||
description: "http get content from example.com, with custom header",
|
||||
example: "http get -H [my-header-key my-header-value] https://www.example.com",
|
||||
result: None,
|
||||
},
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
struct Arguments {
|
||||
url: Value,
|
||||
raw: bool,
|
||||
user: Option<String>,
|
||||
password: Option<String>,
|
||||
timeout: Option<Value>,
|
||||
headers: Option<Value>,
|
||||
}
|
||||
|
||||
fn run_fetch(
|
||||
engine_state: &EngineState,
|
||||
stack: &mut Stack,
|
||||
call: &Call,
|
||||
_input: PipelineData,
|
||||
) -> Result<nu_protocol::PipelineData, nu_protocol::ShellError> {
|
||||
let args = Arguments {
|
||||
url: call.req(engine_state, stack, 0)?,
|
||||
raw: call.has_flag("raw"),
|
||||
user: call.get_flag(engine_state, stack, "user")?,
|
||||
password: call.get_flag(engine_state, stack, "password")?,
|
||||
timeout: call.get_flag(engine_state, stack, "timeout")?,
|
||||
headers: call.get_flag(engine_state, stack, "headers")?,
|
||||
};
|
||||
helper(engine_state, stack, args)
|
||||
}
|
||||
|
||||
// Helper function that actually goes to retrieve the resource from the url given
|
||||
// The Option<String> return a possible file extension which can be used in AutoConvert commands
|
||||
fn helper(
|
||||
engine_state: &EngineState,
|
||||
stack: &mut Stack,
|
||||
args: Arguments,
|
||||
) -> std::result::Result<PipelineData, ShellError> {
|
||||
// There is no need to error-check this, as the URL is already guaranteed by basic nu command argument type checks.
|
||||
let url_value = args.url;
|
||||
|
||||
let span = url_value.span()?;
|
||||
let requested_url = url_value.as_string()?;
|
||||
let url = match url::Url::parse(&requested_url) {
|
||||
Ok(u) => u,
|
||||
Err(_e) => {
|
||||
return Err(ShellError::TypeMismatch(
|
||||
"Incomplete or incorrect URL. Expected a full URL, e.g., https://www.example.com"
|
||||
.to_string(),
|
||||
span,
|
||||
));
|
||||
}
|
||||
};
|
||||
let user = args.user.clone();
|
||||
let password = args.password;
|
||||
let timeout = args.timeout;
|
||||
let headers = args.headers;
|
||||
let raw = args.raw;
|
||||
let base64_engine = GeneralPurpose::new(&alphabet::STANDARD, PAD);
|
||||
|
||||
let login = match (user, password) {
|
||||
(Some(user), Some(password)) => {
|
||||
let mut enc_str = String::new();
|
||||
base64_engine.encode_string(&format!("{}:{}", user, password), &mut enc_str);
|
||||
Some(enc_str)
|
||||
}
|
||||
(Some(user), _) => {
|
||||
let mut enc_str = String::new();
|
||||
base64_engine.encode_string(&format!("{}:", user), &mut enc_str);
|
||||
Some(enc_str)
|
||||
}
|
||||
(_, Some(password)) => {
|
||||
let mut enc_str = String::new();
|
||||
base64_engine.encode_string(&format!(":{}", password), &mut enc_str);
|
||||
Some(enc_str)
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
|
||||
let client = http_client();
|
||||
let mut request = client.get(url);
|
||||
|
||||
if let Some(timeout) = timeout {
|
||||
let val = timeout.as_i64()?;
|
||||
if val.is_negative() || val < 1 {
|
||||
return Err(ShellError::TypeMismatch(
|
||||
"Timeout value must be an integer and larger than 0".to_string(),
|
||||
// timeout is already guaranteed to not be an error
|
||||
timeout.expect_span(),
|
||||
));
|
||||
}
|
||||
|
||||
request = request.timeout(Duration::from_secs(val as u64));
|
||||
}
|
||||
|
||||
if let Some(login) = login {
|
||||
request = request.header("Authorization", format!("Basic {}", login));
|
||||
}
|
||||
|
||||
if let Some(headers) = headers {
|
||||
let mut custom_headers: HashMap<String, Value> = HashMap::new();
|
||||
|
||||
match &headers {
|
||||
Value::List { vals: table, .. } => {
|
||||
if table.len() == 1 {
|
||||
// single row([key1 key2]; [val1 val2])
|
||||
match &table[0] {
|
||||
Value::Record { cols, vals, .. } => {
|
||||
for (k, v) in cols.iter().zip(vals.iter()) {
|
||||
custom_headers.insert(k.to_string(), v.clone());
|
||||
}
|
||||
}
|
||||
|
||||
x => {
|
||||
return Err(ShellError::CantConvert(
|
||||
"string list or single row".into(),
|
||||
x.get_type().to_string(),
|
||||
headers.span().unwrap_or_else(|_| Span::new(0, 0)),
|
||||
None,
|
||||
));
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// primitive values ([key1 val1 key2 val2])
|
||||
for row in table.chunks(2) {
|
||||
if row.len() == 2 {
|
||||
custom_headers.insert(row[0].as_string()?, row[1].clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
x => {
|
||||
return Err(ShellError::CantConvert(
|
||||
"string list or single row".into(),
|
||||
x.get_type().to_string(),
|
||||
headers.span().unwrap_or_else(|_| Span::new(0, 0)),
|
||||
None,
|
||||
));
|
||||
}
|
||||
};
|
||||
|
||||
for (k, v) in &custom_headers {
|
||||
if let Ok(s) = v.as_string() {
|
||||
request = request.header(k, s);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Explicitly turn 4xx and 5xx statuses into errors.
|
||||
match request.send().and_then(|r| r.error_for_status()) {
|
||||
Ok(resp) => match resp.headers().get("content-type") {
|
||||
Some(content_type) => {
|
||||
let content_type = content_type.to_str().map_err(|e| {
|
||||
ShellError::GenericError(
|
||||
e.to_string(),
|
||||
"".to_string(),
|
||||
None,
|
||||
Some("MIME type were invalid".to_string()),
|
||||
Vec::new(),
|
||||
)
|
||||
})?;
|
||||
let content_type = mime::Mime::from_str(content_type).map_err(|_| {
|
||||
ShellError::GenericError(
|
||||
format!("MIME type unknown: {}", content_type),
|
||||
"".to_string(),
|
||||
None,
|
||||
Some("given unknown MIME type".to_string()),
|
||||
Vec::new(),
|
||||
)
|
||||
})?;
|
||||
let ext = match (content_type.type_(), content_type.subtype()) {
|
||||
(mime::TEXT, mime::PLAIN) => {
|
||||
let path_extension = url::Url::parse(&requested_url)
|
||||
.map_err(|_| {
|
||||
ShellError::GenericError(
|
||||
format!("Cannot parse URL: {}", requested_url),
|
||||
"".to_string(),
|
||||
None,
|
||||
Some("cannot parse".to_string()),
|
||||
Vec::new(),
|
||||
)
|
||||
})?
|
||||
.path_segments()
|
||||
.and_then(|segments| segments.last())
|
||||
.and_then(|name| if name.is_empty() { None } else { Some(name) })
|
||||
.and_then(|name| {
|
||||
PathBuf::from(name)
|
||||
.extension()
|
||||
.map(|name| name.to_string_lossy().to_string())
|
||||
});
|
||||
path_extension
|
||||
}
|
||||
_ => Some(content_type.subtype().to_string()),
|
||||
};
|
||||
|
||||
let output = response_to_buffer(resp, engine_state, span);
|
||||
|
||||
if raw {
|
||||
return Ok(output);
|
||||
}
|
||||
|
||||
if let Some(ext) = ext {
|
||||
match engine_state.find_decl(format!("from {}", ext).as_bytes(), &[]) {
|
||||
Some(converter_id) => engine_state.get_decl(converter_id).run(
|
||||
engine_state,
|
||||
stack,
|
||||
&Call::new(span),
|
||||
output,
|
||||
),
|
||||
None => Ok(output),
|
||||
}
|
||||
} else {
|
||||
Ok(output)
|
||||
}
|
||||
}
|
||||
None => Ok(response_to_buffer(resp, engine_state, span)),
|
||||
},
|
||||
Err(e) if e.is_timeout() => Err(ShellError::NetworkFailure(
|
||||
format!("Request to {} has timed out", requested_url),
|
||||
span,
|
||||
)),
|
||||
Err(e) if e.is_status() => match e.status() {
|
||||
Some(err_code) if err_code == StatusCode::NOT_FOUND => Err(ShellError::NetworkFailure(
|
||||
format!("Requested file not found (404): {:?}", requested_url),
|
||||
span,
|
||||
)),
|
||||
Some(err_code) if err_code == StatusCode::MOVED_PERMANENTLY => {
|
||||
Err(ShellError::NetworkFailure(
|
||||
format!("Resource moved permanently (301): {:?}", requested_url),
|
||||
span,
|
||||
))
|
||||
}
|
||||
Some(err_code) if err_code == StatusCode::BAD_REQUEST => {
|
||||
Err(ShellError::NetworkFailure(
|
||||
format!("Bad request (400) to {:?}", requested_url),
|
||||
span,
|
||||
))
|
||||
}
|
||||
Some(err_code) if err_code == StatusCode::FORBIDDEN => Err(ShellError::NetworkFailure(
|
||||
format!("Access forbidden (403) to {:?}", requested_url),
|
||||
span,
|
||||
)),
|
||||
_ => Err(ShellError::NetworkFailure(
|
||||
format!(
|
||||
"Cannot make request to {:?}. Error is {:?}",
|
||||
requested_url,
|
||||
e.to_string()
|
||||
),
|
||||
span,
|
||||
)),
|
||||
},
|
||||
Err(e) => Err(ShellError::NetworkFailure(
|
||||
format!(
|
||||
"Cannot make request to {:?}. Error is {:?}",
|
||||
requested_url,
|
||||
e.to_string()
|
||||
),
|
||||
span,
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
fn response_to_buffer(
|
||||
response: Response,
|
||||
engine_state: &EngineState,
|
||||
span: Span,
|
||||
) -> nu_protocol::PipelineData {
|
||||
// Try to get the size of the file to be downloaded.
|
||||
// This is helpful to show the progress of the stream.
|
||||
let buffer_size = match &response.headers().get("content-length") {
|
||||
Some(content_length) => {
|
||||
let content_length = &(*content_length).clone(); // binding
|
||||
|
||||
let content_length = content_length
|
||||
.to_str()
|
||||
.unwrap_or("")
|
||||
.parse::<u64>()
|
||||
.unwrap_or(0);
|
||||
|
||||
if content_length == 0 {
|
||||
None
|
||||
} else {
|
||||
Some(content_length)
|
||||
}
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
let buffered_input = BufReader::new(response);
|
||||
|
||||
PipelineData::ExternalStream {
|
||||
stdout: Some(RawStream::new(
|
||||
Box::new(BufferedReader {
|
||||
input: buffered_input,
|
||||
}),
|
||||
engine_state.ctrlc.clone(),
|
||||
span,
|
||||
buffer_size,
|
||||
)),
|
||||
stderr: None,
|
||||
exit_code: None,
|
||||
span,
|
||||
metadata: None,
|
||||
trim_end_newline: false,
|
||||
}
|
||||
}
|
||||
|
||||
// Only panics if the user agent is invalid but we define it statically so either
|
||||
// it always or never fails
|
||||
#[allow(clippy::unwrap_used)]
|
||||
fn http_client() -> reqwest::blocking::Client {
|
||||
reqwest::blocking::Client::builder()
|
||||
.user_agent("nushell")
|
||||
.build()
|
||||
.unwrap()
|
||||
}
|
||||
55
crates/nu-command/src/network/http/http_.rs
Normal file
55
crates/nu-command/src/network/http/http_.rs
Normal file
@ -0,0 +1,55 @@
|
||||
use nu_engine::get_full_help;
|
||||
use nu_protocol::{
|
||||
ast::Call,
|
||||
engine::{Command, EngineState, Stack},
|
||||
Category, IntoPipelineData, PipelineData, Signature, Type, Value,
|
||||
};
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct Http;
|
||||
|
||||
impl Command for Http {
|
||||
fn name(&self) -> &str {
|
||||
"http"
|
||||
}
|
||||
|
||||
fn signature(&self) -> Signature {
|
||||
Signature::build("http")
|
||||
.input_output_types(vec![(Type::Nothing, Type::String)])
|
||||
.category(Category::Network)
|
||||
}
|
||||
|
||||
fn usage(&self) -> &str {
|
||||
"Various commands for working with http methods"
|
||||
}
|
||||
|
||||
fn extra_usage(&self) -> &str {
|
||||
"You must use one of the following subcommands. Using this command as-is will only produce this help message."
|
||||
}
|
||||
|
||||
fn search_terms(&self) -> Vec<&str> {
|
||||
vec![
|
||||
"network", "fetch", "pull", "request", "download", "curl", "wget",
|
||||
]
|
||||
}
|
||||
|
||||
fn run(
|
||||
&self,
|
||||
engine_state: &EngineState,
|
||||
stack: &mut Stack,
|
||||
call: &Call,
|
||||
_input: PipelineData,
|
||||
) -> Result<nu_protocol::PipelineData, nu_protocol::ShellError> {
|
||||
Ok(Value::String {
|
||||
val: get_full_help(
|
||||
&Http.signature(),
|
||||
&Http.examples(),
|
||||
engine_state,
|
||||
stack,
|
||||
self.is_parser_keyword(),
|
||||
),
|
||||
span: call.head,
|
||||
}
|
||||
.into_pipeline_data())
|
||||
}
|
||||
}
|
||||
7
crates/nu-command/src/network/http/mod.rs
Normal file
7
crates/nu-command/src/network/http/mod.rs
Normal file
@ -0,0 +1,7 @@
|
||||
mod get;
|
||||
mod http_;
|
||||
mod post;
|
||||
|
||||
pub use get::SubCommand as HttpGet;
|
||||
pub use http_::Http;
|
||||
pub use post::SubCommand as HttpPost;
|
||||
445
crates/nu-command/src/network/http/post.rs
Normal file
445
crates/nu-command/src/network/http/post.rs
Normal file
@ -0,0 +1,445 @@
|
||||
use crate::formats::value_to_json_value;
|
||||
use base64::{alphabet, engine::general_purpose::PAD, engine::GeneralPurpose, Engine};
|
||||
use nu_engine::CallExt;
|
||||
use nu_protocol::ast::Call;
|
||||
use nu_protocol::engine::{Command, EngineState, Stack};
|
||||
use nu_protocol::util::BufferedReader;
|
||||
use nu_protocol::RawStream;
|
||||
use nu_protocol::{
|
||||
Category, Example, PipelineData, ShellError, Signature, Span, SyntaxShape, Type, Value,
|
||||
};
|
||||
use reqwest::{blocking::Response, StatusCode};
|
||||
use std::collections::HashMap;
|
||||
use std::io::BufReader;
|
||||
use std::path::PathBuf;
|
||||
use std::str::FromStr;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct SubCommand;
|
||||
|
||||
impl Command for SubCommand {
|
||||
fn name(&self) -> &str {
|
||||
"http post"
|
||||
}
|
||||
|
||||
fn signature(&self) -> Signature {
|
||||
Signature::build("http post")
|
||||
.input_output_types(vec![(Type::Nothing, Type::Any)])
|
||||
.required("path", SyntaxShape::String, "the URL to post to")
|
||||
.required("body", SyntaxShape::Any, "the contents of the post body")
|
||||
.named(
|
||||
"user",
|
||||
SyntaxShape::Any,
|
||||
"the username when authenticating",
|
||||
Some('u'),
|
||||
)
|
||||
.named(
|
||||
"password",
|
||||
SyntaxShape::Any,
|
||||
"the password when authenticating",
|
||||
Some('p'),
|
||||
)
|
||||
.named(
|
||||
"content-type",
|
||||
SyntaxShape::Any,
|
||||
"the MIME type of content to post",
|
||||
Some('t'),
|
||||
)
|
||||
.named(
|
||||
"content-length",
|
||||
SyntaxShape::Any,
|
||||
"the length of the content being posted",
|
||||
Some('l'),
|
||||
)
|
||||
.named(
|
||||
"headers",
|
||||
SyntaxShape::Any,
|
||||
"custom headers you want to add ",
|
||||
Some('H'),
|
||||
)
|
||||
.switch(
|
||||
"raw",
|
||||
"return values as a string instead of a table",
|
||||
Some('r'),
|
||||
)
|
||||
.switch(
|
||||
"insecure",
|
||||
"allow insecure server connections when using SSL",
|
||||
Some('k'),
|
||||
)
|
||||
.filter()
|
||||
.category(Category::Network)
|
||||
}
|
||||
|
||||
fn usage(&self) -> &str {
|
||||
"Post a body to a URL."
|
||||
}
|
||||
|
||||
fn extra_usage(&self) -> &str {
|
||||
"Performs HTTP POST operation."
|
||||
}
|
||||
|
||||
fn search_terms(&self) -> Vec<&str> {
|
||||
vec!["network", "send", "push"]
|
||||
}
|
||||
|
||||
fn run(
|
||||
&self,
|
||||
engine_state: &EngineState,
|
||||
stack: &mut Stack,
|
||||
call: &Call,
|
||||
input: PipelineData,
|
||||
) -> Result<nu_protocol::PipelineData, nu_protocol::ShellError> {
|
||||
run_post(engine_state, stack, call, input)
|
||||
}
|
||||
fn examples(&self) -> Vec<Example> {
|
||||
vec![
|
||||
Example {
|
||||
description: "Post content to url.com",
|
||||
example: "http post url.com 'body'",
|
||||
result: None,
|
||||
},
|
||||
Example {
|
||||
description: "Post content to url.com, with username and password",
|
||||
example: "http post -u myuser -p mypass url.com 'body'",
|
||||
result: None,
|
||||
},
|
||||
Example {
|
||||
description: "Post content to url.com, with custom header",
|
||||
example: "http post -H [my-header-key my-header-value] url.com",
|
||||
result: None,
|
||||
},
|
||||
Example {
|
||||
description: "Post content to url.com with a json body",
|
||||
example: "http post -t application/json url.com { field: value }",
|
||||
result: None,
|
||||
},
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
struct Arguments {
|
||||
path: Value,
|
||||
body: Value,
|
||||
headers: Option<Value>,
|
||||
raw: bool,
|
||||
insecure: Option<bool>,
|
||||
user: Option<String>,
|
||||
password: Option<String>,
|
||||
content_type: Option<String>,
|
||||
content_length: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(PartialEq, Eq)]
|
||||
enum BodyType {
|
||||
Json,
|
||||
Form,
|
||||
Unknown,
|
||||
}
|
||||
|
||||
fn run_post(
|
||||
engine_state: &EngineState,
|
||||
stack: &mut Stack,
|
||||
call: &Call,
|
||||
_input: PipelineData,
|
||||
) -> Result<nu_protocol::PipelineData, nu_protocol::ShellError> {
|
||||
let args = Arguments {
|
||||
path: call.req(engine_state, stack, 0)?,
|
||||
body: call.req(engine_state, stack, 1)?,
|
||||
headers: call.get_flag(engine_state, stack, "headers")?,
|
||||
raw: call.has_flag("raw"),
|
||||
user: call.get_flag(engine_state, stack, "user")?,
|
||||
password: call.get_flag(engine_state, stack, "password")?,
|
||||
insecure: call.get_flag(engine_state, stack, "insecure")?,
|
||||
content_type: call.get_flag(engine_state, stack, "content-type")?,
|
||||
content_length: call.get_flag(engine_state, stack, "content-length")?,
|
||||
};
|
||||
helper(engine_state, stack, call, args)
|
||||
}
|
||||
// Helper function that actually goes to retrieve the resource from the url given
|
||||
// The Option<String> return a possible file extension which can be used in AutoConvert commands
|
||||
fn helper(
|
||||
engine_state: &EngineState,
|
||||
stack: &mut Stack,
|
||||
call: &Call,
|
||||
args: Arguments,
|
||||
) -> std::result::Result<PipelineData, ShellError> {
|
||||
let url_value = args.path;
|
||||
let body = args.body;
|
||||
let span = url_value.span()?;
|
||||
let requested_url = url_value.as_string()?;
|
||||
let url = match url::Url::parse(&requested_url) {
|
||||
Ok(u) => u,
|
||||
Err(_e) => {
|
||||
return Err(ShellError::UnsupportedInput(
|
||||
"Incomplete or incorrect URL. Expected a full URL, e.g., https://www.example.com"
|
||||
.to_string(),
|
||||
format!("value: '{:?}'", requested_url),
|
||||
call.head,
|
||||
span,
|
||||
));
|
||||
}
|
||||
};
|
||||
let user = args.user.clone();
|
||||
let password = args.password;
|
||||
let headers = args.headers;
|
||||
let location = url;
|
||||
let raw = args.raw;
|
||||
let base64_engine = GeneralPurpose::new(&alphabet::STANDARD, PAD);
|
||||
|
||||
let login = match (user, password) {
|
||||
(Some(user), Some(password)) => {
|
||||
let mut enc_str = String::new();
|
||||
base64_engine.encode_string(&format!("{}:{}", user, password), &mut enc_str);
|
||||
Some(enc_str)
|
||||
}
|
||||
(Some(user), _) => {
|
||||
let mut enc_str = String::new();
|
||||
base64_engine.encode_string(&format!("{}:", user), &mut enc_str);
|
||||
Some(enc_str)
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
|
||||
let body_type = match &args.content_type {
|
||||
Some(it) if it == "application/json" => BodyType::Json,
|
||||
Some(it) if it == "application/x-www-form-urlencoded" => BodyType::Form,
|
||||
_ => BodyType::Unknown,
|
||||
};
|
||||
|
||||
let mut request = http_client(args.insecure.is_some()).post(location);
|
||||
|
||||
// set the content-type header before using e.g., request.json
|
||||
// because that will avoid duplicating the header value
|
||||
if let Some(val) = args.content_type {
|
||||
request = request.header("Content-Type", val);
|
||||
}
|
||||
|
||||
match body {
|
||||
Value::Binary { val, .. } => {
|
||||
request = request.body(val);
|
||||
}
|
||||
Value::String { val, .. } => {
|
||||
request = request.body(val);
|
||||
}
|
||||
Value::Record { .. } if body_type == BodyType::Json => {
|
||||
let data = value_to_json_value(&body)?;
|
||||
request = request.json(&data);
|
||||
}
|
||||
Value::Record { .. } if body_type == BodyType::Form => {
|
||||
let data = value_to_json_value(&body)?;
|
||||
request = request.form(&data);
|
||||
}
|
||||
Value::List { vals, .. } if body_type == BodyType::Form => {
|
||||
if vals.len() % 2 != 0 {
|
||||
return Err(ShellError::IOError("unsupported body input".into()));
|
||||
}
|
||||
let data = vals
|
||||
.chunks(2)
|
||||
.map(|it| Ok((it[0].as_string()?, it[1].as_string()?)))
|
||||
.collect::<Result<Vec<(String, String)>, ShellError>>()?;
|
||||
request = request.form(&data)
|
||||
}
|
||||
_ => {
|
||||
return Err(ShellError::IOError("unsupported body input".into()));
|
||||
}
|
||||
};
|
||||
|
||||
if let Some(val) = args.content_length {
|
||||
request = request.header("Content-Length", val);
|
||||
}
|
||||
if let Some(login) = login {
|
||||
request = request.header("Authorization", format!("Basic {}", login));
|
||||
}
|
||||
|
||||
if let Some(headers) = headers {
|
||||
let mut custom_headers: HashMap<String, Value> = HashMap::new();
|
||||
|
||||
match &headers {
|
||||
Value::List { vals: table, .. } => {
|
||||
if table.len() == 1 {
|
||||
// single row([key1 key2]; [val1 val2])
|
||||
match &table[0] {
|
||||
Value::Record { cols, vals, .. } => {
|
||||
for (k, v) in cols.iter().zip(vals.iter()) {
|
||||
custom_headers.insert(k.to_string(), v.clone());
|
||||
}
|
||||
}
|
||||
|
||||
x => {
|
||||
return Err(ShellError::CantConvert(
|
||||
"string list or single row".into(),
|
||||
x.get_type().to_string(),
|
||||
headers.span().unwrap_or_else(|_| Span::new(0, 0)),
|
||||
None,
|
||||
));
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// primitive values ([key1 val1 key2 val2])
|
||||
for row in table.chunks(2) {
|
||||
if row.len() == 2 {
|
||||
custom_headers.insert(row[0].as_string()?, row[1].clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
x => {
|
||||
return Err(ShellError::CantConvert(
|
||||
"string list or single row".into(),
|
||||
x.get_type().to_string(),
|
||||
headers.span().unwrap_or_else(|_| Span::new(0, 0)),
|
||||
None,
|
||||
));
|
||||
}
|
||||
};
|
||||
|
||||
for (k, v) in &custom_headers {
|
||||
if let Ok(s) = v.as_string() {
|
||||
request = request.header(k, s);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Explicitly turn 4xx and 5xx statuses into errors.
|
||||
match request.send().and_then(|r| r.error_for_status()) {
|
||||
Ok(resp) => match resp.headers().get("content-type") {
|
||||
Some(content_type) => {
|
||||
let content_type = content_type.to_str().map_err(|e| {
|
||||
ShellError::GenericError(
|
||||
e.to_string(),
|
||||
"".to_string(),
|
||||
None,
|
||||
Some("MIME type were invalid".to_string()),
|
||||
Vec::new(),
|
||||
)
|
||||
})?;
|
||||
let content_type = mime::Mime::from_str(content_type).map_err(|_| {
|
||||
ShellError::GenericError(
|
||||
format!("MIME type unknown: {}", content_type),
|
||||
"".to_string(),
|
||||
None,
|
||||
Some("given unknown MIME type".to_string()),
|
||||
Vec::new(),
|
||||
)
|
||||
})?;
|
||||
let ext = match (content_type.type_(), content_type.subtype()) {
|
||||
(mime::TEXT, mime::PLAIN) => {
|
||||
let path_extension = url::Url::parse(&requested_url)
|
||||
.map_err(|_| {
|
||||
ShellError::GenericError(
|
||||
format!("Cannot parse URL: {}", requested_url),
|
||||
"".to_string(),
|
||||
None,
|
||||
Some("cannot parse".to_string()),
|
||||
Vec::new(),
|
||||
)
|
||||
})?
|
||||
.path_segments()
|
||||
.and_then(|segments| segments.last())
|
||||
.and_then(|name| if name.is_empty() { None } else { Some(name) })
|
||||
.and_then(|name| {
|
||||
PathBuf::from(name)
|
||||
.extension()
|
||||
.map(|name| name.to_string_lossy().to_string())
|
||||
});
|
||||
path_extension
|
||||
}
|
||||
_ => Some(content_type.subtype().to_string()),
|
||||
};
|
||||
let output = response_to_buffer(resp, engine_state, span);
|
||||
|
||||
if raw {
|
||||
return Ok(output);
|
||||
}
|
||||
if let Some(ext) = ext {
|
||||
match engine_state.find_decl(format!("from {}", ext).as_bytes(), &[]) {
|
||||
Some(converter_id) => engine_state.get_decl(converter_id).run(
|
||||
engine_state,
|
||||
stack,
|
||||
&Call::new(span),
|
||||
output,
|
||||
),
|
||||
None => Ok(output),
|
||||
}
|
||||
} else {
|
||||
Ok(output)
|
||||
}
|
||||
}
|
||||
None => Ok(response_to_buffer(resp, engine_state, span)),
|
||||
},
|
||||
Err(e) if e.is_status() => match e.status() {
|
||||
Some(err_code) if err_code == StatusCode::NOT_FOUND => Err(ShellError::NetworkFailure(
|
||||
format!("Requested file not found (404): {:?}", requested_url),
|
||||
span,
|
||||
)),
|
||||
Some(err_code) if err_code == StatusCode::MOVED_PERMANENTLY => {
|
||||
Err(ShellError::NetworkFailure(
|
||||
format!("Resource moved permanently (301): {:?}", requested_url),
|
||||
span,
|
||||
))
|
||||
}
|
||||
Some(err_code) if err_code == StatusCode::BAD_REQUEST => {
|
||||
Err(ShellError::NetworkFailure(
|
||||
format!("Bad request (400) to {:?}", requested_url),
|
||||
span,
|
||||
))
|
||||
}
|
||||
Some(err_code) if err_code == StatusCode::FORBIDDEN => Err(ShellError::NetworkFailure(
|
||||
format!("Access forbidden (403) to {:?}", requested_url),
|
||||
span,
|
||||
)),
|
||||
_ => Err(ShellError::NetworkFailure(
|
||||
format!(
|
||||
"Cannot make request to {:?}. Error is {:?}",
|
||||
requested_url,
|
||||
e.to_string()
|
||||
),
|
||||
span,
|
||||
)),
|
||||
},
|
||||
Err(e) => Err(ShellError::NetworkFailure(
|
||||
format!(
|
||||
"Cannot make request to {:?}. Error is {:?}",
|
||||
requested_url,
|
||||
e.to_string()
|
||||
),
|
||||
span,
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
fn response_to_buffer(
|
||||
response: Response,
|
||||
engine_state: &EngineState,
|
||||
span: Span,
|
||||
) -> nu_protocol::PipelineData {
|
||||
let buffered_input = BufReader::new(response);
|
||||
|
||||
PipelineData::ExternalStream {
|
||||
stdout: Some(RawStream::new(
|
||||
Box::new(BufferedReader {
|
||||
input: buffered_input,
|
||||
}),
|
||||
engine_state.ctrlc.clone(),
|
||||
span,
|
||||
None,
|
||||
)),
|
||||
stderr: None,
|
||||
exit_code: None,
|
||||
span,
|
||||
metadata: None,
|
||||
trim_end_newline: false,
|
||||
}
|
||||
}
|
||||
// Only panics if the user agent is invalid but we define it statically so either
|
||||
// it always or never fails
|
||||
#[allow(clippy::unwrap_used)]
|
||||
fn http_client(allow_insecure: bool) -> reqwest::blocking::Client {
|
||||
reqwest::blocking::Client::builder()
|
||||
.user_agent("nushell")
|
||||
.danger_accept_invalid_certs(allow_insecure)
|
||||
.build()
|
||||
.expect("Failed to build reqwest client")
|
||||
}
|
||||
Reference in New Issue
Block a user