Url split query (#14211)

Addresses the following points from #14162

> - There is no built-in counterpart to url build-query for splitting a
query string

There is `from url`, which, due to naming, is a little hard to discover
and suffers from the following point

> - url parse can create records with duplicate keys
> - url parse's params should either:
>   - ~group the same keys into a list.~
> - instead of a record, be a key-value table. (table<key: string,
value: string>)

# Description

## `url split-query`

Counterpart to `url build-query`, splits a url encoded query string to
key value pairs, represented as `table<key: string, value: string>`

```
> "a=one&a=two&b=three" | url split-query
╭───┬─────┬───────╮
│ # │ key │ value │
├───┼─────┼───────┤
│ 0 │ a   │ one   │
│ 1 │ a   │ two   │
│ 2 │ b   │ three │
╰───┴─────┴───────╯
```

## `url parse`

The output's `param` field is now a table as well, mirroring the new
`url split-query`

```
> 'http://localhost?a=one&a=two&b=three' | url parse
╭──────────┬─────────────────────╮
│ scheme   │ http                │
│ username │                     │
│ password │                     │
│ host     │ localhost           │
│ port     │                     │
│ path     │ /                   │
│ query    │ a=one&a=two&b=three │
│ fragment │                     │
│          │ ╭───┬─────┬───────╮ │
│ params   │ │ # │ key │ value │ │
│          │ ├───┼─────┼───────┤ │
│          │ │ 0 │ a   │ one   │ │
│          │ │ 1 │ a   │ two   │ │
│          │ │ 2 │ b   │ three │ │
│          │ ╰───┴─────┴───────╯ │
╰──────────┴─────────────────────╯
```

# User-Facing Changes

- `url parse`'s output has the mentioned change, which is backwards
incompatible.
This commit is contained in:
Bahex 2024-11-06 16:35:37 +03:00 committed by GitHub
parent d52ec65f18
commit 3182adb6a0
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 200 additions and 55 deletions

View File

@ -387,6 +387,7 @@ pub fn add_shell_command_context(mut engine_state: EngineState) -> EngineState {
HttpOptions,
Url,
UrlBuildQuery,
UrlSplitQuery,
UrlDecode,
UrlEncode,
UrlJoin,

View File

@ -4,6 +4,7 @@ mod encode;
mod join;
mod parse;
mod query;
mod split_query;
mod url_;
pub use self::parse::SubCommand as UrlParse;
@ -11,4 +12,5 @@ pub use build_query::SubCommand as UrlBuildQuery;
pub use decode::SubCommand as UrlDecode;
pub use encode::SubCommand as UrlEncode;
pub use join::SubCommand as UrlJoin;
pub use split_query::SubCommand as UrlSplitQuery;
pub use url_::Url;

View File

@ -2,6 +2,8 @@ use nu_engine::command_prelude::*;
use nu_protocol::Config;
use url::Url;
use super::query::query_string_to_table;
#[derive(Clone)]
pub struct SubCommand;
@ -53,7 +55,7 @@ impl Command for SubCommand {
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Parses a url",
example: "'http://user123:pass567@www.example.com:8081/foo/bar?param1=section&p2=&f[name]=vldc#hello' | url parse",
example: "'http://user123:pass567@www.example.com:8081/foo/bar?param1=section&p2=&f[name]=vldc&f[no]=42#hello' | url parse",
result: Some(Value::test_record(record! {
"scheme" => Value::test_string("http"),
"username" => Value::test_string("user123"),
@ -61,13 +63,14 @@ impl Command for SubCommand {
"host" => Value::test_string("www.example.com"),
"port" => Value::test_string("8081"),
"path" => Value::test_string("/foo/bar"),
"query" => Value::test_string("param1=section&p2=&f[name]=vldc"),
"query" => Value::test_string("param1=section&p2=&f[name]=vldc&f[no]=42"),
"fragment" => Value::test_string("hello"),
"params" => Value::test_record(record! {
"param1" => Value::test_string("section"),
"p2" => Value::test_string(""),
"f[name]" => Value::test_string("vldc"),
}),
"params" => Value::test_list(vec![
Value::test_record(record! {"key" => Value::test_string("param1"), "value" => Value::test_string("section") }),
Value::test_record(record! {"key" => Value::test_string("p2"), "value" => Value::test_string("") }),
Value::test_record(record! {"key" => Value::test_string("f[name]"), "value" => Value::test_string("vldc") }),
Value::test_record(record! {"key" => Value::test_string("f[no]"), "value" => Value::test_string("42") }),
]),
})),
}]
}
@ -80,54 +83,41 @@ fn get_url_string(value: &Value, config: &Config) -> String {
fn parse(value: Value, head: Span, config: &Config) -> Result<PipelineData, ShellError> {
let url_string = get_url_string(&value, config);
let result_url = Url::parse(url_string.as_str());
// This is the span of the original string, not the call head.
let span = value.span();
match result_url {
Ok(url) => {
let params =
serde_urlencoded::from_str::<Vec<(String, String)>>(url.query().unwrap_or(""));
match params {
Ok(result) => {
let params = result
.into_iter()
.map(|(k, v)| (k, Value::string(v, head)))
.collect();
let url = Url::parse(url_string.as_str()).map_err(|_| ShellError::UnsupportedInput {
msg: "Incomplete or incorrect URL. Expected a full URL, e.g., https://www.example.com"
.to_string(),
input: "value originates from here".into(),
msg_span: head,
input_span: span,
})?;
let port = url.port().map(|p| p.to_string()).unwrap_or_default();
let record = record! {
"scheme" => Value::string(url.scheme(), head),
"username" => Value::string(url.username(), head),
"password" => Value::string(url.password().unwrap_or(""), head),
"host" => Value::string(url.host_str().unwrap_or(""), head),
"port" => Value::string(port, head),
"path" => Value::string(url.path(), head),
"query" => Value::string(url.query().unwrap_or(""), head),
"fragment" => Value::string(url.fragment().unwrap_or(""), head),
"params" => Value::record(params, head),
};
Ok(PipelineData::Value(Value::record(record, head), None))
}
_ => Err(ShellError::UnsupportedInput {
msg: "String not compatible with url-encoding".to_string(),
input: "value originates from here".into(),
msg_span: head,
input_span: span,
}),
}
}
Err(_e) => Err(ShellError::UnsupportedInput {
msg: "Incomplete or incorrect URL. Expected a full URL, e.g., https://www.example.com"
.to_string(),
let params = query_string_to_table(url.query().unwrap_or(""), head, span).map_err(|_| {
ShellError::UnsupportedInput {
msg: "String not compatible with url-encoding".to_string(),
input: "value originates from here".into(),
msg_span: head,
input_span: span,
}),
}
}
})?;
let port = url.port().map(|p| p.to_string()).unwrap_or_default();
let record = record! {
"scheme" => Value::string(url.scheme(), head),
"username" => Value::string(url.username(), head),
"password" => Value::string(url.password().unwrap_or(""), head),
"host" => Value::string(url.host_str().unwrap_or(""), head),
"port" => Value::string(port, head),
"path" => Value::string(url.path(), head),
"query" => Value::string(url.query().unwrap_or(""), head),
"fragment" => Value::string(url.fragment().unwrap_or(""), head),
"params" => params,
};
Ok(PipelineData::Value(Value::record(record, head), None))
}
#[cfg(test)]

View File

@ -1,4 +1,4 @@
use nu_protocol::{Record, ShellError, Span, Type, Value};
use nu_protocol::{IntoValue, Record, ShellError, Span, Type, Value};
pub fn record_to_query_string(
record: &Record,
@ -42,3 +42,26 @@ pub fn record_to_query_string(
help: None,
})
}
pub fn query_string_to_table(query: &str, head: Span, span: Span) -> Result<Value, ShellError> {
let params = serde_urlencoded::from_str::<Vec<(String, String)>>(query)
.map_err(|_| ShellError::UnsupportedInput {
msg: "String not compatible with url-encoding".to_string(),
input: "value originates from here".into(),
msg_span: head,
input_span: span,
})?
.into_iter()
.map(|(key, value)| {
Value::record(
nu_protocol::record! {
"key" => key.into_value(head),
"value" => value.into_value(head)
},
head,
)
})
.collect::<Vec<_>>();
Ok(Value::list(params, head))
}

View File

@ -0,0 +1,106 @@
use nu_engine::command_prelude::*;
use super::query::query_string_to_table;
#[derive(Clone)]
pub struct SubCommand;
impl Command for SubCommand {
fn name(&self) -> &str {
"url split-query"
}
fn signature(&self) -> Signature {
Signature::build("url split-query")
.input_output_types(vec![(
Type::String,
Type::Table([("key".into(), Type::String), ("value".into(), Type::String)].into()),
)])
.category(Category::Network)
}
fn description(&self) -> &str {
"Converts query string into table applying percent-decoding."
}
fn search_terms(&self) -> Vec<&str> {
vec!["convert", "record", "table"]
}
fn examples(&self) -> Vec<Example> {
vec![
Example {
description: "Outputs a table representing the contents of this query string",
example: r#""mode=normal&userid=31415" | url split-query"#,
result: Some(Value::test_list(vec![
Value::test_record(record!{
"key" => Value::test_string("mode"),
"value" => Value::test_string("normal"),
}),
Value::test_record(record!{
"key" => Value::test_string("userid"),
"value" => Value::test_string("31415"),
})
])),
},
Example {
description: "Outputs a table representing the contents of this query string, url-decoding the values",
example: r#""a=AT%26T&b=AT+T" | url split-query"#,
result: Some(Value::test_list(vec![
Value::test_record(record!{
"key" => Value::test_string("a"),
"value" => Value::test_string("AT&T"),
}),
Value::test_record(record!{
"key" => Value::test_string("b"),
"value" => Value::test_string("AT T"),
}),
])),
},
Example {
description: "Outputs a table representing the contents of this query string",
example: r#""a=one&a=two&b=three" | url split-query"#,
result: Some(Value::test_list(vec![
Value::test_record(record!{
"key" => Value::test_string("a"),
"value" => Value::test_string("one"),
}),
Value::test_record(record!{
"key" => Value::test_string("a"),
"value" => Value::test_string("two"),
}),
Value::test_record(record!{
"key" => Value::test_string("b"),
"value" => Value::test_string("three"),
}),
])),
},
]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let value = input.into_value(call.head)?;
let span = value.span();
let query = value.to_expanded_string("", &stack.get_config(engine_state));
let table = query_string_to_table(&query, call.head, span)?;
Ok(PipelineData::Value(table, None))
}
}
#[cfg(test)]
mod test {
use super::*;
#[test]
fn test_examples() {
use crate::test_examples;
test_examples(SubCommand {})
}
}

View File

@ -15,7 +15,7 @@ fn url_parse_simple() {
path: '/',
query: '',
fragment: '',
params: {}
params: []
}
"#
));
@ -37,7 +37,7 @@ fn url_parse_with_port() {
path: '/',
query: '',
fragment: '',
params: {}
params: []
}
"#
));
@ -60,7 +60,7 @@ fn url_parse_with_path() {
path: '/def/ghj',
query: '',
fragment: '',
params: {}
params: []
}
"#
));
@ -83,7 +83,30 @@ fn url_parse_with_params() {
path: '/def/ghj',
query: 'param1=11&param2=',
fragment: '',
params: {param1: '11', param2: ''}
params: [[key, value]; ["param1", "11"], ["param2", ""]]
}
"#
));
assert_eq!(actual.out, "true");
}
#[test]
fn url_parse_with_duplicate_params() {
let actual = nu!(pipeline(
r#"
("http://www.abc.com:8811/def/ghj?param1=11&param2=&param1=22"
| url parse)
== {
scheme: 'http',
username: '',
password: '',
host: 'www.abc.com',
port: '8811',
path: '/def/ghj',
query: 'param1=11&param2=&param1=22',
fragment: '',
params: [[key, value]; ["param1", "11"], ["param2", ""], ["param1", "22"]]
}
"#
));
@ -106,7 +129,7 @@ fn url_parse_with_fragment() {
path: '/def/ghj',
query: 'param1=11&param2=',
fragment: 'hello-fragment',
params: {param1: '11', param2: ''}
params: [[key, value]; ["param1", "11"], ["param2", ""]]
}
"#
));
@ -129,7 +152,7 @@ fn url_parse_with_username_and_password() {
path: '/def/ghj',
query: 'param1=11&param2=',
fragment: 'hello-fragment',
params: {param1: '11', param2: ''}
params: [[key, value]; ["param1", "11"], ["param2", ""]]
}
"#
));