mirror of
https://github.com/nushell/nushell.git
synced 2024-11-25 01:43:47 +01:00
Port fetch (with fixes) (#660)
* Port fetch to engine-q * Fix check for path as a string * Add a timeout flag and fix some span issues * Add a temporary fetch command that returns byte streams. Got rid of async stuff as we're using the blocking feature of tokio * More tweaks for the bytestream * Rewrite fetch using ByteStreams * buffer read on bytes directly Co-authored-by: Stefan Stanciulescu <contact@stefanstanciulescu.com>
This commit is contained in:
JT
GitHub
parent
b6fcd46075
commit
36079f1a3d
361
Cargo.lock
generated
361
Cargo.lock
generated
@ -354,6 +354,12 @@ version = "1.4.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"
|
||||
|
||||
[[package]]
|
||||
name = "bytes"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8"
|
||||
|
||||
[[package]]
|
||||
name = "bytesize"
|
||||
version = "1.1.0"
|
||||
@ -508,6 +514,16 @@ version = "0.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc"
|
||||
|
||||
[[package]]
|
||||
name = "core-foundation"
|
||||
version = "0.9.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6888e10551bb93e424d8df1d07f1a8b4fceb0001a3a4b048bfc47554946f47b3"
|
||||
dependencies = [
|
||||
"core-foundation-sys",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "core-foundation-sys"
|
||||
version = "0.8.3"
|
||||
@ -635,7 +651,7 @@ checksum = "22813a6dc45b335f9bade10bf7271dc477e81113e89eb251a0bc2a8a81c536e1"
|
||||
dependencies = [
|
||||
"bstr",
|
||||
"csv-core",
|
||||
"itoa",
|
||||
"itoa 0.4.8",
|
||||
"ryu",
|
||||
"serde",
|
||||
]
|
||||
@ -922,6 +938,21 @@ version = "1.0.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
|
||||
|
||||
[[package]]
|
||||
name = "foreign-types"
|
||||
version = "0.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1"
|
||||
dependencies = [
|
||||
"foreign-types-shared",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "foreign-types-shared"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"
|
||||
|
||||
[[package]]
|
||||
name = "form_urlencoded"
|
||||
version = "1.0.1"
|
||||
@ -1109,6 +1140,25 @@ version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574"
|
||||
|
||||
[[package]]
|
||||
name = "h2"
|
||||
version = "0.3.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8f072413d126e57991455e0a922b31e4c8ba7c2ffbebf6b78b4f8521397d65cd"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"fnv",
|
||||
"futures-core",
|
||||
"futures-sink",
|
||||
"futures-util",
|
||||
"http",
|
||||
"indexmap",
|
||||
"slab",
|
||||
"tokio",
|
||||
"tokio-util",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hash32"
|
||||
version = "0.1.1"
|
||||
@ -1181,6 +1231,40 @@ version = "0.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e9025058dae765dee5070ec375f591e2ba14638c63feff74f13805a72e523163"
|
||||
|
||||
[[package]]
|
||||
name = "http"
|
||||
version = "0.2.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "31f4c6746584866f0feabcc69893c5b51beef3831656a968ed7ae254cdc4fd03"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"fnv",
|
||||
"itoa 1.0.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "http-body"
|
||||
version = "0.4.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1ff4f84919677303da5f147645dbea6b1881f368d03ac84e1dc09031ebd7b2c6"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"http",
|
||||
"pin-project-lite",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "httparse"
|
||||
version = "1.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "acd94fdbe1d4ff688b67b04eee2e17bd50995534a61539e45adfefb45e5e5503"
|
||||
|
||||
[[package]]
|
||||
name = "httpdate"
|
||||
version = "1.0.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421"
|
||||
|
||||
[[package]]
|
||||
name = "humantime"
|
||||
version = "1.3.0"
|
||||
@ -1190,6 +1274,43 @@ dependencies = [
|
||||
"quick-error",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hyper"
|
||||
version = "0.14.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b7ec3e62bdc98a2f0393a5048e4c30ef659440ea6e0e572965103e72bd836f55"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"futures-channel",
|
||||
"futures-core",
|
||||
"futures-util",
|
||||
"h2",
|
||||
"http",
|
||||
"http-body",
|
||||
"httparse",
|
||||
"httpdate",
|
||||
"itoa 0.4.8",
|
||||
"pin-project-lite",
|
||||
"socket2",
|
||||
"tokio",
|
||||
"tower-service",
|
||||
"tracing",
|
||||
"want",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hyper-tls"
|
||||
version = "0.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"hyper",
|
||||
"native-tls",
|
||||
"tokio",
|
||||
"tokio-native-tls",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ical"
|
||||
version = "0.7.0"
|
||||
@ -1264,6 +1385,12 @@ dependencies = [
|
||||
"ghost",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ipnet"
|
||||
version = "2.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "68f2d64f2edebec4ce84ad108148e67e1064789bee435edc5b60ad398714a3a9"
|
||||
|
||||
[[package]]
|
||||
name = "is_ci"
|
||||
version = "1.1.1"
|
||||
@ -1291,6 +1418,12 @@ version = "0.4.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4"
|
||||
|
||||
[[package]]
|
||||
name = "itoa"
|
||||
version = "1.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1aab8fc367588b89dcee83ab0fd66b72b50b72fa1904d7095045ace2b0c81c35"
|
||||
|
||||
[[package]]
|
||||
name = "jobserver"
|
||||
version = "0.1.24"
|
||||
@ -1586,6 +1719,12 @@ dependencies = [
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "mime"
|
||||
version = "0.3.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d"
|
||||
|
||||
[[package]]
|
||||
name = "miniz_oxide"
|
||||
version = "0.4.4"
|
||||
@ -1638,6 +1777,24 @@ dependencies = [
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "native-tls"
|
||||
version = "0.2.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "48ba9f7719b5a0f42f338907614285fb5fd70e53858141f69898a1fb7203b24d"
|
||||
dependencies = [
|
||||
"lazy_static",
|
||||
"libc",
|
||||
"log",
|
||||
"openssl",
|
||||
"openssl-probe",
|
||||
"openssl-sys",
|
||||
"schannel",
|
||||
"security-framework",
|
||||
"security-framework-sys",
|
||||
"tempfile",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "nix"
|
||||
version = "0.23.0"
|
||||
@ -1746,6 +1903,7 @@ dependencies = [
|
||||
"lscolors",
|
||||
"md-5",
|
||||
"meval",
|
||||
"mime",
|
||||
"nu-ansi-term",
|
||||
"nu-color-config",
|
||||
"nu-engine",
|
||||
@ -1762,6 +1920,7 @@ dependencies = [
|
||||
"rand",
|
||||
"rayon",
|
||||
"regex",
|
||||
"reqwest",
|
||||
"roxmltree",
|
||||
"rust-embed",
|
||||
"serde",
|
||||
@ -1992,7 +2151,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bafe4179722c2894288ee77a9f044f02811c86af699344c498b0840c698a2465"
|
||||
dependencies = [
|
||||
"arrayvec 0.4.12",
|
||||
"itoa",
|
||||
"itoa 0.4.8",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -2090,6 +2249,20 @@ version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"
|
||||
|
||||
[[package]]
|
||||
name = "openssl"
|
||||
version = "0.10.38"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0c7ae222234c30df141154f159066c5093ff73b63204dcda7121eb082fc56a95"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"cfg-if",
|
||||
"foreign-types",
|
||||
"libc",
|
||||
"once_cell",
|
||||
"openssl-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "openssl-probe"
|
||||
version = "0.1.4"
|
||||
@ -2647,6 +2820,41 @@ dependencies = [
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "reqwest"
|
||||
version = "0.11.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7c4e0a76dc12a116108933f6301b95e83634e0c47b0afbed6abbaa0601e99258"
|
||||
dependencies = [
|
||||
"base64",
|
||||
"bytes",
|
||||
"encoding_rs",
|
||||
"futures-core",
|
||||
"futures-util",
|
||||
"http",
|
||||
"http-body",
|
||||
"hyper",
|
||||
"hyper-tls",
|
||||
"ipnet",
|
||||
"js-sys",
|
||||
"lazy_static",
|
||||
"log",
|
||||
"mime",
|
||||
"native-tls",
|
||||
"percent-encoding",
|
||||
"pin-project-lite",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"serde_urlencoded",
|
||||
"tokio",
|
||||
"tokio-native-tls",
|
||||
"url",
|
||||
"wasm-bindgen",
|
||||
"wasm-bindgen-futures",
|
||||
"web-sys",
|
||||
"winreg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "result"
|
||||
version = "1.0.0"
|
||||
@ -2741,12 +2949,45 @@ dependencies = [
|
||||
"winapi-util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "schannel"
|
||||
version = "0.1.19"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8f05ba609c234e60bee0d547fe94a4c7e9da733d1c962cf6e59efa4cd9c8bc75"
|
||||
dependencies = [
|
||||
"lazy_static",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "scopeguard"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
|
||||
|
||||
[[package]]
|
||||
name = "security-framework"
|
||||
version = "2.4.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "525bc1abfda2e1998d152c45cf13e696f76d0a4972310b22fac1658b05df7c87"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"core-foundation",
|
||||
"core-foundation-sys",
|
||||
"libc",
|
||||
"security-framework-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "security-framework-sys"
|
||||
version = "2.4.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a9dd14d83160b528b7bfd66439110573efcfbe281b17fc2ca9f39f550d619c7e"
|
||||
dependencies = [
|
||||
"core-foundation-sys",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "semver"
|
||||
version = "0.11.0"
|
||||
@ -2803,7 +3044,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d0ffa0837f2dfa6fb90868c2b5468cad482e175f7dad97e7421951e663f2b527"
|
||||
dependencies = [
|
||||
"indexmap",
|
||||
"itoa",
|
||||
"itoa 0.4.8",
|
||||
"ryu",
|
||||
"serde",
|
||||
]
|
||||
@ -2824,7 +3065,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "edfa57a7f8d9c1d260a549e7224100f6c43d43f9103e06dd8b4095a9b2b43ce9"
|
||||
dependencies = [
|
||||
"form_urlencoded",
|
||||
"itoa",
|
||||
"itoa 0.4.8",
|
||||
"ryu",
|
||||
"serde",
|
||||
]
|
||||
@ -2952,6 +3193,16 @@ version = "1.0.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "45456094d1983e2ee2a18fdfebce3189fa451699d0502cb8e3b49dba5ba41451"
|
||||
|
||||
[[package]]
|
||||
name = "socket2"
|
||||
version = "0.4.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5dc90fe6c7be1a323296982db1836d1ea9e47b6839496dde9a541bc496df3516"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "spin"
|
||||
version = "0.9.2"
|
||||
@ -3197,6 +3448,45 @@ dependencies = [
|
||||
"regex",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio"
|
||||
version = "1.15.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fbbf1c778ec206785635ce8ad57fe52b3009ae9e0c9f574a728f3049d3e55838"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"libc",
|
||||
"memchr",
|
||||
"mio",
|
||||
"num_cpus",
|
||||
"pin-project-lite",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio-native-tls"
|
||||
version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b"
|
||||
dependencies = [
|
||||
"native-tls",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio-util"
|
||||
version = "0.6.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9e99e1983e5d376cd8eb4b66604d2e99e79f5bd988c3055891dcd8c9e2604cc0"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"futures-core",
|
||||
"futures-sink",
|
||||
"log",
|
||||
"pin-project-lite",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "toml"
|
||||
version = "0.5.8"
|
||||
@ -3206,6 +3496,32 @@ dependencies = [
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tower-service"
|
||||
version = "0.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6"
|
||||
|
||||
[[package]]
|
||||
name = "tracing"
|
||||
version = "0.1.29"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "375a639232caf30edfc78e8d89b2d4c375515393e7af7e16f01cd96917fb2105"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"pin-project-lite",
|
||||
"tracing-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tracing-core"
|
||||
version = "0.1.21"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1f4ed65637b8390770814083d20756f87bfa2c21bf2f110babdc5438351746e4"
|
||||
dependencies = [
|
||||
"lazy_static",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "trash"
|
||||
version = "2.0.2"
|
||||
@ -3221,6 +3537,12 @@ dependencies = [
|
||||
"windows",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "try-lock"
|
||||
version = "0.2.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642"
|
||||
|
||||
[[package]]
|
||||
name = "typenum"
|
||||
version = "1.14.0"
|
||||
@ -3428,6 +3750,16 @@ dependencies = [
|
||||
"winapi-util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "want"
|
||||
version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0"
|
||||
dependencies = [
|
||||
"log",
|
||||
"try-lock",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasi"
|
||||
version = "0.9.0+wasi-snapshot-preview1"
|
||||
@ -3465,6 +3797,18 @@ dependencies = [
|
||||
"wasm-bindgen-shared",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-futures"
|
||||
version = "0.4.28"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8e8d7523cb1f2a4c96c1317ca690031b714a51cc14e05f712446691f413f5d39"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"js-sys",
|
||||
"wasm-bindgen",
|
||||
"web-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-macro"
|
||||
version = "0.2.78"
|
||||
@ -3565,6 +3909,15 @@ dependencies = [
|
||||
"windows_gen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "winreg"
|
||||
version = "0.7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0120db82e8a1e0b9fb3345a539c478767c0048d842860994d96113d5b667bd69"
|
||||
dependencies = [
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "xmlparser"
|
||||
version = "0.13.3"
|
||||
|
||||
@ -59,7 +59,7 @@ pretty_assertions = "1.0.0"
|
||||
|
||||
[features]
|
||||
plugin = ["nu-plugin", "nu-parser/plugin", "nu-command/plugin", "nu-protocol/plugin", "nu-engine/plugin"]
|
||||
|
||||
fetch-support = ["nu-command/fetch"]
|
||||
default = [
|
||||
"plugin",
|
||||
"inc",
|
||||
|
||||
@ -68,6 +68,8 @@ sha2 = "0.10.0"
|
||||
base64 = "0.13.0"
|
||||
encoding_rs = "0.8.30"
|
||||
num = { version = "0.4.0", optional = true }
|
||||
reqwest = {version = "0.11", features = ["blocking"], optional = true }
|
||||
mime = "0.3.16"
|
||||
|
||||
[target.'cfg(unix)'.dependencies]
|
||||
umask = "1.0.0"
|
||||
@ -86,6 +88,7 @@ features = [
|
||||
trash-support = ["trash"]
|
||||
plugin = ["nu-parser/plugin"]
|
||||
dataframe = ["polars", "num"]
|
||||
fetch = ["reqwest"]
|
||||
|
||||
[build-dependencies]
|
||||
shadow-rs = "0.8.1"
|
||||
|
||||
@ -297,6 +297,9 @@ pub fn create_default_context() -> EngineState {
|
||||
#[cfg(feature = "plugin")]
|
||||
bind_command!(Register);
|
||||
|
||||
#[cfg(feature = "fetch")]
|
||||
bind_command!(Fetch);
|
||||
|
||||
// This is a WIP proof of concept
|
||||
// bind_command!(ListGitBranches, Git, GitCheckout, Source);
|
||||
|
||||
|
||||
338
crates/nu-command/src/network/fetch.rs
Normal file
338
crates/nu-command/src/network/fetch.rs
Normal file
@ -0,0 +1,338 @@
|
||||
use base64::encode;
|
||||
use nu_engine::CallExt;
|
||||
use nu_protocol::ast::Call;
|
||||
use nu_protocol::engine::{Command, EngineState, Stack};
|
||||
use nu_protocol::ByteStream;
|
||||
|
||||
use nu_protocol::{
|
||||
Category, Example, PipelineData, ShellError, Signature, Span, SyntaxShape, Value,
|
||||
};
|
||||
|
||||
use std::io::{BufRead, BufReader, Read};
|
||||
|
||||
use reqwest::StatusCode;
|
||||
use std::path::PathBuf;
|
||||
use std::str::FromStr;
|
||||
use std::time::Duration;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct SubCommand;
|
||||
|
||||
impl Command for SubCommand {
|
||||
fn name(&self) -> &str {
|
||||
"fetch"
|
||||
}
|
||||
|
||||
fn signature(&self) -> Signature {
|
||||
Signature::build("fetch")
|
||||
.desc("Load from a URL into a cell, convert to table if possible (avoid by appending '--raw').")
|
||||
.required(
|
||||
"URL",
|
||||
SyntaxShape::String,
|
||||
"the URL to fetch the contents from",
|
||||
)
|
||||
.named(
|
||||
"user",
|
||||
SyntaxShape::Any,
|
||||
"the username when authenticating",
|
||||
Some('u'),
|
||||
)
|
||||
.named(
|
||||
"password",
|
||||
SyntaxShape::Any,
|
||||
"the password when authenticating",
|
||||
Some('p'),
|
||||
)
|
||||
.named("timeout", SyntaxShape::Int, "timeout period in seconds", Some('t'))
|
||||
.switch("raw", "fetch contents as text rather than a table", Some('r'))
|
||||
.filter()
|
||||
.category(Category::Network)
|
||||
}
|
||||
|
||||
fn usage(&self) -> &str {
|
||||
"Fetch the contents from a URL (HTTP GET operation)."
|
||||
}
|
||||
|
||||
fn run(
|
||||
&self,
|
||||
engine_state: &EngineState,
|
||||
stack: &mut Stack,
|
||||
call: &Call,
|
||||
input: PipelineData,
|
||||
) -> Result<nu_protocol::PipelineData, nu_protocol::ShellError> {
|
||||
run_fetch(engine_state, stack, call, input)
|
||||
}
|
||||
|
||||
fn examples(&self) -> Vec<Example> {
|
||||
vec![
|
||||
Example {
|
||||
description: "Fetch content from url.com",
|
||||
example: "fetch url.com",
|
||||
result: None,
|
||||
},
|
||||
Example {
|
||||
description: "Fetch content from url.com, with username and password",
|
||||
example: "fetch -u myuser -p mypass url.com",
|
||||
result: None,
|
||||
},
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
struct Arguments {
|
||||
url: Option<Value>,
|
||||
raw: bool,
|
||||
user: Option<String>,
|
||||
password: Option<String>,
|
||||
timeout: Option<Value>,
|
||||
}
|
||||
|
||||
fn run_fetch(
|
||||
engine_state: &EngineState,
|
||||
stack: &mut Stack,
|
||||
call: &Call,
|
||||
_input: PipelineData,
|
||||
) -> Result<nu_protocol::PipelineData, nu_protocol::ShellError> {
|
||||
let args = Arguments {
|
||||
url: Some(call.req(engine_state, stack, 0)?),
|
||||
raw: call.has_flag("raw"),
|
||||
user: call.get_flag(engine_state, stack, "user")?,
|
||||
password: call.get_flag(engine_state, stack, "password")?,
|
||||
timeout: call.get_flag(engine_state, stack, "timeout")?,
|
||||
};
|
||||
helper(engine_state, stack, call, args)
|
||||
}
|
||||
|
||||
// Helper function that actually goes to retrieve the resource from the url given
|
||||
// The Option<String> return a possible file extension which can be used in AutoConvert commands
|
||||
fn helper(
|
||||
engine_state: &EngineState,
|
||||
stack: &mut Stack,
|
||||
call: &Call,
|
||||
args: Arguments,
|
||||
) -> std::result::Result<PipelineData, ShellError> {
|
||||
let url_value = if let Some(val) = args.url {
|
||||
val
|
||||
} else {
|
||||
return Err(ShellError::UnsupportedInput(
|
||||
"Expecting a url as a string but got nothing".to_string(),
|
||||
call.head,
|
||||
));
|
||||
};
|
||||
|
||||
let span = url_value.span()?;
|
||||
let requested_url = url_value.as_string()?;
|
||||
let url = match url::Url::parse(&requested_url) {
|
||||
Ok(u) => u,
|
||||
Err(_e) => {
|
||||
return Err(ShellError::UnsupportedInput(
|
||||
"Incomplete or incorrect url. Expected a full url, e.g., https://www.example.com"
|
||||
.to_string(),
|
||||
span,
|
||||
));
|
||||
}
|
||||
};
|
||||
let user = args.user.clone();
|
||||
let password = args.password;
|
||||
let timeout = args.timeout;
|
||||
let raw = args.raw;
|
||||
let login = match (user, password) {
|
||||
(Some(user), Some(password)) => Some(encode(&format!("{}:{}", user, password))),
|
||||
(Some(user), _) => Some(encode(&format!("{}:", user))),
|
||||
_ => None,
|
||||
};
|
||||
|
||||
let client = http_client();
|
||||
let mut request = client.get(url);
|
||||
|
||||
if let Some(timeout) = timeout {
|
||||
let val = timeout.as_i64()?;
|
||||
if val.is_negative() || val < 1 {
|
||||
return Err(ShellError::UnsupportedInput(
|
||||
"Timeout value must be an integer and larger than 0".to_string(),
|
||||
timeout.span().unwrap_or_else(|_| Span::new(0, 0)),
|
||||
));
|
||||
}
|
||||
|
||||
request = request.timeout(Duration::from_secs(val as u64));
|
||||
}
|
||||
|
||||
if let Some(login) = login {
|
||||
request = request.header("Authorization", format!("Basic {}", login));
|
||||
}
|
||||
|
||||
match request.send() {
|
||||
Ok(resp) => {
|
||||
// let temp = std::fs::File::create("temp_dwl.txt")?;
|
||||
// let mut b = BufWriter::new(temp);
|
||||
// let _bytes = resp.copy_to(&mut b);
|
||||
// let temp1 = std::fs::File::open("temp_dwl.txt")?;
|
||||
// let a = BufReader::new(temp1);
|
||||
|
||||
// TODO I guess we should check if all bytes were written/read...
|
||||
match resp.headers().get("content-type") {
|
||||
Some(content_type) => {
|
||||
let content_type = content_type.to_str().map_err(|e| {
|
||||
ShellError::LabeledError(
|
||||
e.to_string(),
|
||||
"MIME type were invalid".to_string(),
|
||||
)
|
||||
})?;
|
||||
let content_type = mime::Mime::from_str(content_type).map_err(|_| {
|
||||
ShellError::LabeledError(
|
||||
format!("MIME type unknown: {}", content_type),
|
||||
"given unknown MIME type".to_string(),
|
||||
)
|
||||
})?;
|
||||
let ext = match (content_type.type_(), content_type.subtype()) {
|
||||
(mime::TEXT, mime::PLAIN) => {
|
||||
let path_extension = url::Url::parse(&requested_url)
|
||||
.map_err(|_| {
|
||||
ShellError::LabeledError(
|
||||
format!("Cannot parse URL: {}", requested_url),
|
||||
"cannot parse".to_string(),
|
||||
)
|
||||
})?
|
||||
.path_segments()
|
||||
.and_then(|segments| segments.last())
|
||||
.and_then(|name| if name.is_empty() { None } else { Some(name) })
|
||||
.and_then(|name| {
|
||||
PathBuf::from(name)
|
||||
.extension()
|
||||
.map(|name| name.to_string_lossy().to_string())
|
||||
});
|
||||
path_extension
|
||||
}
|
||||
_ => Some(content_type.subtype().to_string()),
|
||||
};
|
||||
|
||||
let buffered_input = BufReader::new(resp);
|
||||
|
||||
let output = PipelineData::ByteStream(
|
||||
ByteStream {
|
||||
stream: Box::new(BufferedReader {
|
||||
input: buffered_input,
|
||||
}),
|
||||
ctrlc: engine_state.ctrlc.clone(),
|
||||
},
|
||||
span,
|
||||
None,
|
||||
);
|
||||
|
||||
if raw {
|
||||
return Ok(output);
|
||||
}
|
||||
|
||||
if let Some(ext) = ext {
|
||||
match engine_state.find_decl(format!("from {}", ext).as_bytes()) {
|
||||
Some(converter_id) => engine_state.get_decl(converter_id).run(
|
||||
engine_state,
|
||||
stack,
|
||||
&Call::new(),
|
||||
output,
|
||||
),
|
||||
None => Ok(output),
|
||||
}
|
||||
} else {
|
||||
Ok(output)
|
||||
}
|
||||
}
|
||||
None => {
|
||||
let buffered_input = BufReader::new(resp);
|
||||
|
||||
let output = PipelineData::ByteStream(
|
||||
ByteStream {
|
||||
stream: Box::new(BufferedReader {
|
||||
input: buffered_input,
|
||||
}),
|
||||
ctrlc: engine_state.ctrlc.clone(),
|
||||
},
|
||||
span,
|
||||
None,
|
||||
);
|
||||
Ok(output)
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) if e.is_timeout() => Err(ShellError::NetworkFailure(
|
||||
format!("Request to {} has timed out", requested_url),
|
||||
span,
|
||||
)),
|
||||
Err(e) if e.is_status() => match e.status() {
|
||||
Some(err_code) if err_code == StatusCode::NOT_FOUND => Err(ShellError::NetworkFailure(
|
||||
format!("Requested file not found (404): {:?}", requested_url),
|
||||
span,
|
||||
)),
|
||||
Some(err_code) if err_code == StatusCode::MOVED_PERMANENTLY => {
|
||||
Err(ShellError::NetworkFailure(
|
||||
format!("Resource moved permanently (301): {:?}", requested_url),
|
||||
span,
|
||||
))
|
||||
}
|
||||
Some(err_code) if err_code == StatusCode::BAD_REQUEST => {
|
||||
Err(ShellError::NetworkFailure(
|
||||
format!("Bad request (400) to {:?}", requested_url),
|
||||
span,
|
||||
))
|
||||
}
|
||||
Some(err_code) if err_code == StatusCode::FORBIDDEN => Err(ShellError::NetworkFailure(
|
||||
format!("Access forbidden (403) to {:?}", requested_url),
|
||||
span,
|
||||
)),
|
||||
_ => Err(ShellError::NetworkFailure(
|
||||
format!(
|
||||
"Cannot make request to {:?}. Error is {:?}",
|
||||
requested_url,
|
||||
e.to_string()
|
||||
),
|
||||
span,
|
||||
)),
|
||||
},
|
||||
Err(e) => Err(ShellError::NetworkFailure(
|
||||
format!(
|
||||
"Cannot make request to {:?}. Error is {:?}",
|
||||
requested_url,
|
||||
e.to_string()
|
||||
),
|
||||
span,
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
pub struct BufferedReader<R: Read> {
|
||||
input: BufReader<R>,
|
||||
}
|
||||
|
||||
impl<R: Read> Iterator for BufferedReader<R> {
|
||||
type Item = Result<Vec<u8>, ShellError>;
|
||||
|
||||
fn next(&mut self) -> Option<Self::Item> {
|
||||
let buffer = self.input.fill_buf();
|
||||
match buffer {
|
||||
Ok(s) => {
|
||||
let result = s.to_vec();
|
||||
|
||||
let buffer_len = s.len();
|
||||
|
||||
if buffer_len == 0 {
|
||||
None
|
||||
} else {
|
||||
self.input.consume(buffer_len);
|
||||
|
||||
Some(Ok(result))
|
||||
}
|
||||
}
|
||||
Err(e) => Some(Err(ShellError::IOError(e.to_string()))),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Only panics if the user agent is invalid but we define it statically so either
|
||||
// it always or never fails
|
||||
#[allow(clippy::unwrap_used)]
|
||||
fn http_client() -> reqwest::blocking::Client {
|
||||
reqwest::blocking::Client::builder()
|
||||
.user_agent("nushell")
|
||||
.build()
|
||||
.unwrap()
|
||||
}
|
||||
@ -1,3 +1,7 @@
|
||||
#[cfg(feature = "fetch")]
|
||||
mod fetch;
|
||||
mod url;
|
||||
|
||||
pub use self::url::*;
|
||||
#[cfg(feature = "fetch")]
|
||||
pub use fetch::SubCommand as Fetch;
|
||||
|
||||
@ -152,6 +152,10 @@ pub enum ShellError {
|
||||
#[diagnostic(code(nu::shell::unsupported_input), url(docsrs))]
|
||||
UnsupportedInput(String, #[label("{0}")] Span),
|
||||
|
||||
#[error("Network failure")]
|
||||
#[diagnostic(code(nu::shell::network_failure), url(docsrs))]
|
||||
NetworkFailure(String, #[label("{0}")] Span),
|
||||
|
||||
#[error("Command not found")]
|
||||
#[diagnostic(code(nu::shell::command_not_found), url(docsrs))]
|
||||
CommandNotFound(#[label("command not found")] Span),
|
||||
|
||||
Loading…
Reference in New Issue
Block a user