holm/nushell
1
1
Fork 0
forked from extern/nushell
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Move fetch to extra and clean up some code (#664)

Browse Source 
* Move fetch to extra

* Move byte stream code to a function instead of copying it twice

* Fix formatting issues

* Make fetch a default command

* Fix formatting
This commit is contained in:
Ștefan 2022-01-06 00:06:16 +01:00 committed by GitHub
parent d0c280f6cc
commit 47544ad219
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 70 additions and 93 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Cargo.toml
View File

@ -59,7 +59,6 @@ pretty_assertions = "1.0.0"
[features]
plugin = ["nu-plugin", "nu-parser/plugin", "nu-command/plugin", "nu-protocol/plugin", "nu-engine/plugin"]
fetch-support = ["nu-command/fetch"]
default = [
"plugin",
"inc",

3
crates/nu-command/Cargo.toml
View File

@ -68,7 +68,7 @@ sha2 = "0.10.0"
base64 = "0.13.0"
encoding_rs = "0.8.30"
num = { version = "0.4.0", optional = true }
reqwest = {version = "0.11", features = ["blocking"], optional = true }
reqwest = {version = "0.11", features = ["blocking"] }
mime = "0.3.16"
[target.'cfg(unix)'.dependencies]
@ -88,7 +88,6 @@ features = [
trash-support = ["trash"]
plugin = ["nu-parser/plugin"]
dataframe = ["polars", "num"]
fetch = ["reqwest"]
[build-dependencies]
shadow-rs = "0.8.1"

4
crates/nu-command/src/default_context.rs
View File

@ -271,6 +271,7 @@ pub fn create_default_context(cwd: impl AsRef<Path>) -> EngineState {
// Network
bind_command! {
Fetch,
Url,
UrlHost,
UrlPath,
@ -304,9 +305,6 @@ pub fn create_default_context(cwd: impl AsRef<Path>) -> EngineState {
#[cfg(feature = "plugin")]
bind_command!(Register);
#[cfg(feature = "fetch")]
bind_command!(Fetch);
// This is a WIP proof of concept
// bind_command!(ListGitBranches, Git, GitCheckout, Source);

153
crates/nu-command/src/network/fetch.rs
View File

@ -7,6 +7,7 @@ use nu_protocol::ByteStream;
use nu_protocol::{
Category, Example, PipelineData, ShellError, Signature, Span, SyntaxShape, Value,
};
use reqwest::blocking::Response;
use std::io::{BufRead, BufReader, Read};
@ -162,98 +163,61 @@ fn helper(
}
match request.send() {
Ok(resp) => {
// let temp = std::fs::File::create("temp_dwl.txt")?;
// let mut b = BufWriter::new(temp);
// let _bytes = resp.copy_to(&mut b);
// let temp1 = std::fs::File::open("temp_dwl.txt")?;
// let a = BufReader::new(temp1);
// TODO I guess we should check if all bytes were written/read...
match resp.headers().get("content-type") {
Some(content_type) => {
let content_type = content_type.to_str().map_err(|e| {
ShellError::LabeledError(
e.to_string(),
"MIME type were invalid".to_string(),
)
})?;
let content_type = mime::Mime::from_str(content_type).map_err(|_| {
ShellError::LabeledError(
format!("MIME type unknown: {}", content_type),
"given unknown MIME type".to_string(),
)
})?;
let ext = match (content_type.type_(), content_type.subtype()) {
(mime::TEXT, mime::PLAIN) => {
let path_extension = url::Url::parse(&requested_url)
.map_err(|_| {
ShellError::LabeledError(
format!("Cannot parse URL: {}", requested_url),
"cannot parse".to_string(),
)
})?
.path_segments()
.and_then(|segments| segments.last())
.and_then(|name| if name.is_empty() { None } else { Some(name) })
.and_then(|name| {
PathBuf::from(name)
.extension()
.map(|name| name.to_string_lossy().to_string())
});
path_extension
}
_ => Some(content_type.subtype().to_string()),
};
let buffered_input = BufReader::new(resp);
let output = PipelineData::ByteStream(
ByteStream {
stream: Box::new(BufferedReader {
input: buffered_input,
}),
ctrlc: engine_state.ctrlc.clone(),
},
span,
None,
);
if raw {
return Ok(output);
Ok(resp) => match resp.headers().get("content-type") {
Some(content_type) => {
let content_type = content_type.to_str().map_err(|e| {
ShellError::LabeledError(e.to_string(), "MIME type were invalid".to_string())
})?;
let content_type = mime::Mime::from_str(content_type).map_err(|_| {
ShellError::LabeledError(
format!("MIME type unknown: {}", content_type),
"given unknown MIME type".to_string(),
)
})?;
let ext = match (content_type.type_(), content_type.subtype()) {
(mime::TEXT, mime::PLAIN) => {
let path_extension = url::Url::parse(&requested_url)
.map_err(|_| {
ShellError::LabeledError(
format!("Cannot parse URL: {}", requested_url),
"cannot parse".to_string(),
)
})?
.path_segments()
.and_then(|segments| segments.last())
.and_then(|name| if name.is_empty() { None } else { Some(name) })
.and_then(|name| {
PathBuf::from(name)
.extension()
.map(|name| name.to_string_lossy().to_string())
});
path_extension
}
_ => Some(content_type.subtype().to_string()),
};
if let Some(ext) = ext {
match engine_state.find_decl(format!("from {}", ext).as_bytes()) {
Some(converter_id) => engine_state.get_decl(converter_id).run(
engine_state,
stack,
&Call::new(),
output,
),
None => Ok(output),
}
} else {
Ok(output)
}
let output = response_to_buffer(resp, engine_state, span);
if raw {
return Ok(output);
}
None => {
let buffered_input = BufReader::new(resp);
let output = PipelineData::ByteStream(
ByteStream {
stream: Box::new(BufferedReader {
input: buffered_input,
}),
ctrlc: engine_state.ctrlc.clone(),
},
span,
None,
);
if let Some(ext) = ext {
match engine_state.find_decl(format!("from {}", ext).as_bytes()) {
Some(converter_id) => engine_state.get_decl(converter_id).run(
engine_state,
stack,
&Call::new(),
output,
),
None => Ok(output),
}
} else {
Ok(output)
}
}
}
None => Ok(response_to_buffer(resp, engine_state, span)),
},
Err(e) if e.is_timeout() => Err(ShellError::NetworkFailure(
format!("Request to {} has timed out", requested_url),
span,
@ -327,6 +291,25 @@ impl<R: Read> Iterator for BufferedReader<R> {
}
}
fn response_to_buffer(
response: Response,
engine_state: &EngineState,
span: Span,
) -> nu_protocol::PipelineData {
let buffered_input = BufReader::new(response);
PipelineData::ByteStream(
ByteStream {
stream: Box::new(BufferedReader {
input: buffered_input,
}),
ctrlc: engine_state.ctrlc.clone(),
},
span,
None,
)
}
// Only panics if the user agent is invalid but we define it statically so either
// it always or never fails
#[allow(clippy::unwrap_used)]

2
crates/nu-command/src/network/mod.rs
View File

@ -1,7 +1,5 @@
#[cfg(feature = "fetch")]
mod fetch;
mod url;
pub use self::url::*;
#[cfg(feature = "fetch")]
pub use fetch::SubCommand as Fetch;