mirror of
https://github.com/httpie/cli.git
synced 2024-11-21 23:33:12 +01:00
Clean up and refactor nested JSON parsing & interpreting (#1440)
This commit is contained in:
Jakub Roztocil
GitHub
parent
a7321d8ac4
commit
0689b55e1d
@ -9,7 +9,7 @@ from httpie.cli.options import ParserSpec
|
||||
from httpie.manager.cli import options as manager_options
|
||||
from httpie.output.ui.rich_help import OptionsHighlighter, to_usage
|
||||
from httpie.output.ui.rich_utils import render_as_string
|
||||
from httpie.utils import split
|
||||
from httpie.utils import split_iterable
|
||||
|
||||
|
||||
# Escape certain characters so they are rendered properly on
|
||||
|
||||
@ -92,7 +92,3 @@ class MultipartRequestDataDict(MultiValueOrderedDict):
|
||||
|
||||
class RequestFilesDict(RequestDataDict):
|
||||
pass
|
||||
|
||||
|
||||
class NestedJSONArray(list):
|
||||
"""Denotes a top-level JSON array."""
|
||||
|
||||
@ -1,404 +0,0 @@
|
||||
from enum import Enum, auto
|
||||
from typing import (
|
||||
Any,
|
||||
Iterator,
|
||||
NamedTuple,
|
||||
Optional,
|
||||
List,
|
||||
NoReturn,
|
||||
Type,
|
||||
Union,
|
||||
)
|
||||
from .dicts import NestedJSONArray
|
||||
|
||||
|
||||
EMPTY_STRING = ''
|
||||
HIGHLIGHTER = '^'
|
||||
OPEN_BRACKET = '['
|
||||
CLOSE_BRACKET = ']'
|
||||
BACKSLASH = '\\'
|
||||
|
||||
|
||||
class HTTPieSyntaxError(ValueError):
|
||||
def __init__(
|
||||
self,
|
||||
source: str,
|
||||
token: Optional['Token'],
|
||||
message: str,
|
||||
message_kind: str = 'Syntax',
|
||||
) -> None:
|
||||
self.source = source
|
||||
self.token = token
|
||||
self.message = message
|
||||
self.message_kind = message_kind
|
||||
|
||||
def __str__(self):
|
||||
lines = [f'HTTPie {self.message_kind} Error: {self.message}']
|
||||
if self.token is not None:
|
||||
lines.append(self.source)
|
||||
lines.append(
|
||||
' ' * self.token.start
|
||||
+ HIGHLIGHTER * (self.token.end - self.token.start)
|
||||
)
|
||||
return '\n'.join(lines)
|
||||
|
||||
|
||||
class TokenKind(Enum):
|
||||
TEXT = auto()
|
||||
NUMBER = auto()
|
||||
LEFT_BRACKET = auto()
|
||||
RIGHT_BRACKET = auto()
|
||||
|
||||
def to_name(self) -> str:
|
||||
for key, value in OPERATORS.items():
|
||||
if value is self:
|
||||
return repr(key)
|
||||
else:
|
||||
return 'a ' + self.name.lower()
|
||||
|
||||
|
||||
OPERATORS = {
|
||||
OPEN_BRACKET: TokenKind.LEFT_BRACKET,
|
||||
CLOSE_BRACKET: TokenKind.RIGHT_BRACKET,
|
||||
}
|
||||
SPECIAL_CHARS = OPERATORS.keys() | {BACKSLASH}
|
||||
LITERAL_TOKENS = [
|
||||
TokenKind.TEXT,
|
||||
TokenKind.NUMBER,
|
||||
]
|
||||
|
||||
|
||||
class Token(NamedTuple):
|
||||
kind: TokenKind
|
||||
value: Union[str, int]
|
||||
start: int
|
||||
end: int
|
||||
|
||||
|
||||
def assert_cant_happen() -> NoReturn:
|
||||
raise ValueError('Unexpected value')
|
||||
|
||||
|
||||
def check_escaped_int(value: str) -> str:
|
||||
if not value.startswith(BACKSLASH):
|
||||
raise ValueError('Not an escaped int')
|
||||
|
||||
try:
|
||||
int(value[1:])
|
||||
except ValueError as exc:
|
||||
raise ValueError('Not an escaped int') from exc
|
||||
else:
|
||||
return value[1:]
|
||||
|
||||
|
||||
def tokenize(source: str) -> Iterator[Token]:
|
||||
cursor = 0
|
||||
backslashes = 0
|
||||
buffer = []
|
||||
|
||||
def send_buffer() -> Iterator[Token]:
|
||||
nonlocal backslashes
|
||||
if not buffer:
|
||||
return None
|
||||
|
||||
value = ''.join(buffer)
|
||||
kind = TokenKind.TEXT
|
||||
if not backslashes:
|
||||
for variation, kind in [
|
||||
(int, TokenKind.NUMBER),
|
||||
(check_escaped_int, TokenKind.TEXT),
|
||||
]:
|
||||
try:
|
||||
value = variation(value)
|
||||
except ValueError:
|
||||
continue
|
||||
else:
|
||||
break
|
||||
|
||||
yield Token(
|
||||
kind, value, start=cursor - (len(buffer) + backslashes), end=cursor
|
||||
)
|
||||
buffer.clear()
|
||||
backslashes = 0
|
||||
|
||||
def can_advance() -> bool:
|
||||
return cursor < len(source)
|
||||
|
||||
while can_advance():
|
||||
index = source[cursor]
|
||||
if index in OPERATORS:
|
||||
yield from send_buffer()
|
||||
yield Token(OPERATORS[index], index, cursor, cursor + 1)
|
||||
elif index == BACKSLASH and can_advance():
|
||||
if source[cursor + 1] in SPECIAL_CHARS:
|
||||
backslashes += 1
|
||||
else:
|
||||
buffer.append(index)
|
||||
|
||||
buffer.append(source[cursor + 1])
|
||||
cursor += 1
|
||||
else:
|
||||
buffer.append(index)
|
||||
|
||||
cursor += 1
|
||||
|
||||
yield from send_buffer()
|
||||
|
||||
|
||||
class PathAction(Enum):
|
||||
KEY = auto()
|
||||
INDEX = auto()
|
||||
APPEND = auto()
|
||||
|
||||
# Pseudo action, used by the interpreter
|
||||
SET = auto()
|
||||
|
||||
def to_string(self) -> str:
|
||||
return self.name.lower()
|
||||
|
||||
|
||||
class Path:
|
||||
def __init__(
|
||||
self,
|
||||
kind: PathAction,
|
||||
accessor: Optional[Union[str, int]] = None,
|
||||
tokens: Optional[List[Token]] = None,
|
||||
is_root: bool = False,
|
||||
):
|
||||
self.kind = kind
|
||||
self.accessor = accessor
|
||||
self.tokens = tokens or []
|
||||
self.is_root = is_root
|
||||
|
||||
def reconstruct(self) -> str:
|
||||
if self.kind is PathAction.KEY:
|
||||
if self.is_root:
|
||||
return str(self.accessor)
|
||||
return OPEN_BRACKET + self.accessor + CLOSE_BRACKET
|
||||
elif self.kind is PathAction.INDEX:
|
||||
return OPEN_BRACKET + str(self.accessor) + CLOSE_BRACKET
|
||||
elif self.kind is PathAction.APPEND:
|
||||
return OPEN_BRACKET + CLOSE_BRACKET
|
||||
else:
|
||||
assert_cant_happen()
|
||||
|
||||
|
||||
def parse(source: str) -> Iterator[Path]:
|
||||
"""
|
||||
start: root_path path*
|
||||
root_path: (literal | index_path | append_path)
|
||||
literal: TEXT | NUMBER
|
||||
|
||||
path:
|
||||
key_path
|
||||
| index_path
|
||||
| append_path
|
||||
key_path: LEFT_BRACKET TEXT RIGHT_BRACKET
|
||||
index_path: LEFT_BRACKET NUMBER RIGHT_BRACKET
|
||||
append_path: LEFT_BRACKET RIGHT_BRACKET
|
||||
"""
|
||||
|
||||
tokens = list(tokenize(source))
|
||||
cursor = 0
|
||||
|
||||
def can_advance():
|
||||
return cursor < len(tokens)
|
||||
|
||||
def expect(*kinds):
|
||||
nonlocal cursor
|
||||
|
||||
assert len(kinds) > 0
|
||||
if can_advance():
|
||||
token = tokens[cursor]
|
||||
cursor += 1
|
||||
if token.kind in kinds:
|
||||
return token
|
||||
elif tokens:
|
||||
token = tokens[-1]._replace(
|
||||
start=tokens[-1].end + 0, end=tokens[-1].end + 1
|
||||
)
|
||||
else:
|
||||
token = None
|
||||
|
||||
if len(kinds) == 1:
|
||||
suffix = kinds[0].to_name()
|
||||
else:
|
||||
suffix = ', '.join(kind.to_name() for kind in kinds[:-1])
|
||||
suffix += ' or ' + kinds[-1].to_name()
|
||||
|
||||
message = f'Expecting {suffix}'
|
||||
raise HTTPieSyntaxError(source, token, message)
|
||||
|
||||
def parse_root():
|
||||
tokens = []
|
||||
if not can_advance():
|
||||
return Path(
|
||||
PathAction.KEY,
|
||||
EMPTY_STRING,
|
||||
is_root=True
|
||||
)
|
||||
|
||||
# (literal | index_path | append_path)?
|
||||
token = expect(*LITERAL_TOKENS, TokenKind.LEFT_BRACKET)
|
||||
tokens.append(token)
|
||||
|
||||
if token.kind in LITERAL_TOKENS:
|
||||
action = PathAction.KEY
|
||||
value = str(token.value)
|
||||
elif token.kind is TokenKind.LEFT_BRACKET:
|
||||
token = expect(TokenKind.NUMBER, TokenKind.RIGHT_BRACKET)
|
||||
tokens.append(token)
|
||||
if token.kind is TokenKind.NUMBER:
|
||||
action = PathAction.INDEX
|
||||
value = token.value
|
||||
tokens.append(expect(TokenKind.RIGHT_BRACKET))
|
||||
elif token.kind is TokenKind.RIGHT_BRACKET:
|
||||
action = PathAction.APPEND
|
||||
value = None
|
||||
else:
|
||||
assert_cant_happen()
|
||||
else:
|
||||
assert_cant_happen()
|
||||
|
||||
return Path(
|
||||
action,
|
||||
value,
|
||||
tokens=tokens,
|
||||
is_root=True
|
||||
)
|
||||
|
||||
yield parse_root()
|
||||
|
||||
# path*
|
||||
while can_advance():
|
||||
path_tokens = []
|
||||
path_tokens.append(expect(TokenKind.LEFT_BRACKET))
|
||||
|
||||
token = expect(
|
||||
TokenKind.TEXT, TokenKind.NUMBER, TokenKind.RIGHT_BRACKET
|
||||
)
|
||||
path_tokens.append(token)
|
||||
if token.kind is TokenKind.RIGHT_BRACKET:
|
||||
path = Path(PathAction.APPEND, tokens=path_tokens)
|
||||
elif token.kind is TokenKind.TEXT:
|
||||
path = Path(PathAction.KEY, token.value, tokens=path_tokens)
|
||||
path_tokens.append(expect(TokenKind.RIGHT_BRACKET))
|
||||
elif token.kind is TokenKind.NUMBER:
|
||||
path = Path(PathAction.INDEX, token.value, tokens=path_tokens)
|
||||
path_tokens.append(expect(TokenKind.RIGHT_BRACKET))
|
||||
else:
|
||||
assert_cant_happen()
|
||||
yield path
|
||||
|
||||
|
||||
JSON_TYPE_MAPPING = {
|
||||
dict: 'object',
|
||||
list: 'array',
|
||||
int: 'number',
|
||||
float: 'number',
|
||||
str: 'string',
|
||||
}
|
||||
|
||||
|
||||
def interpret(context: Any, key: str, value: Any) -> Any:
|
||||
cursor = context
|
||||
|
||||
paths = list(parse(key))
|
||||
paths.append(Path(PathAction.SET, value))
|
||||
|
||||
def type_check(index: int, path: Path, expected_type: Type[Any]) -> None:
|
||||
if not isinstance(cursor, expected_type):
|
||||
if path.tokens:
|
||||
pseudo_token = Token(
|
||||
None, None, path.tokens[0].start, path.tokens[-1].end
|
||||
)
|
||||
else:
|
||||
pseudo_token = None
|
||||
|
||||
cursor_type = JSON_TYPE_MAPPING.get(
|
||||
type(cursor), type(cursor).__name__
|
||||
)
|
||||
required_type = JSON_TYPE_MAPPING[expected_type]
|
||||
|
||||
message = f"Can't perform {path.kind.to_string()!r} based access on "
|
||||
message += repr(
|
||||
''.join(path.reconstruct() for path in paths[:index])
|
||||
)
|
||||
message += (
|
||||
f' which has a type of {cursor_type!r} but this operation'
|
||||
)
|
||||
message += f' requires a type of {required_type!r}.'
|
||||
raise HTTPieSyntaxError(
|
||||
key, pseudo_token, message, message_kind='Type'
|
||||
)
|
||||
|
||||
def object_for(kind: str) -> Any:
|
||||
if kind is PathAction.KEY:
|
||||
return {}
|
||||
elif kind in {PathAction.INDEX, PathAction.APPEND}:
|
||||
return []
|
||||
else:
|
||||
assert_cant_happen()
|
||||
|
||||
for index, (path, next_path) in enumerate(zip(paths, paths[1:])):
|
||||
# If there is no context yet, set it.
|
||||
if cursor is None:
|
||||
context = cursor = object_for(path.kind)
|
||||
|
||||
if path.kind is PathAction.KEY:
|
||||
type_check(index, path, dict)
|
||||
if next_path.kind is PathAction.SET:
|
||||
cursor[path.accessor] = next_path.accessor
|
||||
break
|
||||
|
||||
cursor = cursor.setdefault(
|
||||
path.accessor, object_for(next_path.kind)
|
||||
)
|
||||
elif path.kind is PathAction.INDEX:
|
||||
type_check(index, path, list)
|
||||
if path.accessor < 0:
|
||||
raise HTTPieSyntaxError(
|
||||
key,
|
||||
path.tokens[1],
|
||||
'Negative indexes are not supported.',
|
||||
message_kind='Value',
|
||||
)
|
||||
cursor.extend([None] * (path.accessor - len(cursor) + 1))
|
||||
if next_path.kind is PathAction.SET:
|
||||
cursor[path.accessor] = next_path.accessor
|
||||
break
|
||||
|
||||
if cursor[path.accessor] is None:
|
||||
cursor[path.accessor] = object_for(next_path.kind)
|
||||
|
||||
cursor = cursor[path.accessor]
|
||||
elif path.kind is PathAction.APPEND:
|
||||
type_check(index, path, list)
|
||||
if next_path.kind is PathAction.SET:
|
||||
cursor.append(next_path.accessor)
|
||||
break
|
||||
|
||||
cursor.append(object_for(next_path.kind))
|
||||
cursor = cursor[-1]
|
||||
else:
|
||||
assert_cant_happen()
|
||||
|
||||
return context
|
||||
|
||||
|
||||
def wrap_with_dict(context):
|
||||
if context is None:
|
||||
return {}
|
||||
elif isinstance(context, list):
|
||||
return {EMPTY_STRING: NestedJSONArray(context)}
|
||||
else:
|
||||
assert isinstance(context, dict)
|
||||
return context
|
||||
|
||||
|
||||
def interpret_nested_json(pairs):
|
||||
context = None
|
||||
for key, value in pairs:
|
||||
context = interpret(context, key, value)
|
||||
|
||||
return wrap_with_dict(context)
|
||||
20
httpie/cli/nested_json/__init__.py
Normal file
20
httpie/cli/nested_json/__init__.py
Normal file
@ -0,0 +1,20 @@
|
||||
"""
|
||||
A library for parsing the HTTPie nested JSON key syntax and constructing the resulting objects.
|
||||
|
||||
<https://httpie.io/docs/cli/nested-json>
|
||||
|
||||
It has no dependencies.
|
||||
|
||||
"""
|
||||
from .interpret import interpret_nested_json, unwrap_top_level_list_if_needed
|
||||
from .errors import NestedJSONSyntaxError
|
||||
from .tokens import EMPTY_STRING, NestedJSONArray
|
||||
|
||||
|
||||
__all__ = [
|
||||
'interpret_nested_json',
|
||||
'unwrap_top_level_list_if_needed',
|
||||
'EMPTY_STRING',
|
||||
'NestedJSONArray',
|
||||
'NestedJSONSyntaxError'
|
||||
]
|
||||
27
httpie/cli/nested_json/errors.py
Normal file
27
httpie/cli/nested_json/errors.py
Normal file
@ -0,0 +1,27 @@
|
||||
from typing import Optional
|
||||
|
||||
from .tokens import Token, HIGHLIGHTER
|
||||
|
||||
|
||||
class NestedJSONSyntaxError(ValueError):
|
||||
def __init__(
|
||||
self,
|
||||
source: str,
|
||||
token: Optional[Token],
|
||||
message: str,
|
||||
message_kind: str = 'Syntax',
|
||||
) -> None:
|
||||
self.source = source
|
||||
self.token = token
|
||||
self.message = message
|
||||
self.message_kind = message_kind
|
||||
|
||||
def __str__(self):
|
||||
lines = [f'HTTPie {self.message_kind} Error: {self.message}']
|
||||
if self.token is not None:
|
||||
lines.append(self.source)
|
||||
lines.append(
|
||||
' ' * self.token.start
|
||||
+ HIGHLIGHTER * (self.token.end - self.token.start)
|
||||
)
|
||||
return '\n'.join(lines)
|
||||
129
httpie/cli/nested_json/interpret.py
Normal file
129
httpie/cli/nested_json/interpret.py
Normal file
@ -0,0 +1,129 @@
|
||||
from typing import Type, Union, Any, Iterable, Tuple
|
||||
|
||||
from .parse import parse, assert_cant_happen
|
||||
from .errors import NestedJSONSyntaxError
|
||||
from .tokens import EMPTY_STRING, TokenKind, Token, PathAction, Path, NestedJSONArray
|
||||
|
||||
|
||||
__all__ = [
|
||||
'interpret_nested_json',
|
||||
'unwrap_top_level_list_if_needed',
|
||||
]
|
||||
|
||||
JSONType = Type[Union[dict, list, int, float, str]]
|
||||
JSON_TYPE_MAPPING = {
|
||||
dict: 'object',
|
||||
list: 'array',
|
||||
int: 'number',
|
||||
float: 'number',
|
||||
str: 'string',
|
||||
}
|
||||
|
||||
|
||||
def interpret_nested_json(pairs: Iterable[Tuple[str, str]]) -> dict:
|
||||
context = None
|
||||
for key, value in pairs:
|
||||
context = interpret(context, key, value)
|
||||
return wrap_with_dict(context)
|
||||
|
||||
|
||||
def interpret(context: Any, key: str, value: Any) -> Any:
|
||||
cursor = context
|
||||
paths = list(parse(key))
|
||||
paths.append(Path(PathAction.SET, value))
|
||||
|
||||
# noinspection PyShadowingNames
|
||||
def type_check(index: int, path: Path, expected_type: JSONType):
|
||||
if not isinstance(cursor, expected_type):
|
||||
if path.tokens:
|
||||
pseudo_token = Token(
|
||||
kind=TokenKind.PSEUDO,
|
||||
value='',
|
||||
start=path.tokens[0].start,
|
||||
end=path.tokens[-1].end,
|
||||
)
|
||||
else:
|
||||
pseudo_token = None
|
||||
cursor_type = JSON_TYPE_MAPPING.get(type(cursor), type(cursor).__name__)
|
||||
required_type = JSON_TYPE_MAPPING[expected_type]
|
||||
message = f'Cannot perform {path.kind.to_string()!r} based access on '
|
||||
message += repr(''.join(path.reconstruct() for path in paths[:index]))
|
||||
message += f' which has a type of {cursor_type!r} but this operation'
|
||||
message += f' requires a type of {required_type!r}.'
|
||||
raise NestedJSONSyntaxError(
|
||||
source=key,
|
||||
token=pseudo_token,
|
||||
message=message,
|
||||
message_kind='Type',
|
||||
)
|
||||
|
||||
def object_for(kind: PathAction) -> Any:
|
||||
if kind is PathAction.KEY:
|
||||
return {}
|
||||
elif kind in {PathAction.INDEX, PathAction.APPEND}:
|
||||
return []
|
||||
else:
|
||||
assert_cant_happen()
|
||||
|
||||
for index, (path, next_path) in enumerate(zip(paths, paths[1:])):
|
||||
# If there is no context yet, set it.
|
||||
if cursor is None:
|
||||
context = cursor = object_for(path.kind)
|
||||
if path.kind is PathAction.KEY:
|
||||
type_check(index, path, dict)
|
||||
if next_path.kind is PathAction.SET:
|
||||
cursor[path.accessor] = next_path.accessor
|
||||
break
|
||||
cursor = cursor.setdefault(path.accessor, object_for(next_path.kind))
|
||||
elif path.kind is PathAction.INDEX:
|
||||
type_check(index, path, list)
|
||||
if path.accessor < 0:
|
||||
raise NestedJSONSyntaxError(
|
||||
source=key,
|
||||
token=path.tokens[1],
|
||||
message='Negative indexes are not supported.',
|
||||
message_kind='Value',
|
||||
)
|
||||
cursor.extend([None] * (path.accessor - len(cursor) + 1))
|
||||
if next_path.kind is PathAction.SET:
|
||||
cursor[path.accessor] = next_path.accessor
|
||||
break
|
||||
if cursor[path.accessor] is None:
|
||||
cursor[path.accessor] = object_for(next_path.kind)
|
||||
cursor = cursor[path.accessor]
|
||||
elif path.kind is PathAction.APPEND:
|
||||
type_check(index, path, list)
|
||||
if next_path.kind is PathAction.SET:
|
||||
cursor.append(next_path.accessor)
|
||||
break
|
||||
cursor.append(object_for(next_path.kind))
|
||||
cursor = cursor[-1]
|
||||
else:
|
||||
assert_cant_happen()
|
||||
|
||||
return context
|
||||
|
||||
|
||||
def wrap_with_dict(context):
|
||||
if context is None:
|
||||
return {}
|
||||
elif isinstance(context, list):
|
||||
return {
|
||||
EMPTY_STRING: NestedJSONArray(context),
|
||||
}
|
||||
else:
|
||||
assert isinstance(context, dict)
|
||||
return context
|
||||
|
||||
|
||||
def unwrap_top_level_list_if_needed(data: dict):
|
||||
"""
|
||||
Propagate the top-level list, if that’s what we got.
|
||||
|
||||
"""
|
||||
if len(data) == 1:
|
||||
key, value = list(data.items())[0]
|
||||
if isinstance(value, NestedJSONArray):
|
||||
assert key == EMPTY_STRING
|
||||
return value
|
||||
return data
|
||||
193
httpie/cli/nested_json/parse.py
Normal file
193
httpie/cli/nested_json/parse.py
Normal file
@ -0,0 +1,193 @@
|
||||
from typing import Iterator
|
||||
|
||||
from .errors import NestedJSONSyntaxError
|
||||
from .tokens import (
|
||||
EMPTY_STRING,
|
||||
BACKSLASH,
|
||||
TokenKind,
|
||||
OPERATORS,
|
||||
SPECIAL_CHARS,
|
||||
LITERAL_TOKENS,
|
||||
Token,
|
||||
PathAction,
|
||||
Path,
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
'parse',
|
||||
'assert_cant_happen',
|
||||
]
|
||||
|
||||
|
||||
def parse(source: str) -> Iterator[Path]:
|
||||
"""
|
||||
start: root_path path*
|
||||
root_path: (literal | index_path | append_path)
|
||||
literal: TEXT | NUMBER
|
||||
|
||||
path:
|
||||
key_path
|
||||
| index_path
|
||||
| append_path
|
||||
key_path: LEFT_BRACKET TEXT RIGHT_BRACKET
|
||||
index_path: LEFT_BRACKET NUMBER RIGHT_BRACKET
|
||||
append_path: LEFT_BRACKET RIGHT_BRACKET
|
||||
|
||||
"""
|
||||
|
||||
tokens = list(tokenize(source))
|
||||
cursor = 0
|
||||
|
||||
def can_advance():
|
||||
return cursor < len(tokens)
|
||||
|
||||
# noinspection PyShadowingNames
|
||||
def expect(*kinds):
|
||||
nonlocal cursor
|
||||
assert kinds
|
||||
if can_advance():
|
||||
token = tokens[cursor]
|
||||
cursor += 1
|
||||
if token.kind in kinds:
|
||||
return token
|
||||
elif tokens:
|
||||
token = tokens[-1]._replace(
|
||||
start=tokens[-1].end + 0,
|
||||
end=tokens[-1].end + 1,
|
||||
)
|
||||
else:
|
||||
token = None
|
||||
if len(kinds) == 1:
|
||||
suffix = kinds[0].to_name()
|
||||
else:
|
||||
suffix = ', '.join(kind.to_name() for kind in kinds[:-1])
|
||||
suffix += ' or ' + kinds[-1].to_name()
|
||||
message = f'Expecting {suffix}'
|
||||
raise NestedJSONSyntaxError(source, token, message)
|
||||
|
||||
# noinspection PyShadowingNames
|
||||
def parse_root():
|
||||
tokens = []
|
||||
if not can_advance():
|
||||
return Path(
|
||||
kind=PathAction.KEY,
|
||||
accessor=EMPTY_STRING,
|
||||
is_root=True
|
||||
)
|
||||
# (literal | index_path | append_path)?
|
||||
token = expect(*LITERAL_TOKENS, TokenKind.LEFT_BRACKET)
|
||||
tokens.append(token)
|
||||
if token.kind in LITERAL_TOKENS:
|
||||
action = PathAction.KEY
|
||||
value = str(token.value)
|
||||
elif token.kind is TokenKind.LEFT_BRACKET:
|
||||
token = expect(TokenKind.NUMBER, TokenKind.RIGHT_BRACKET)
|
||||
tokens.append(token)
|
||||
if token.kind is TokenKind.NUMBER:
|
||||
action = PathAction.INDEX
|
||||
value = token.value
|
||||
tokens.append(expect(TokenKind.RIGHT_BRACKET))
|
||||
elif token.kind is TokenKind.RIGHT_BRACKET:
|
||||
action = PathAction.APPEND
|
||||
value = None
|
||||
else:
|
||||
assert_cant_happen()
|
||||
else:
|
||||
assert_cant_happen()
|
||||
# noinspection PyUnboundLocalVariable
|
||||
return Path(
|
||||
kind=action,
|
||||
accessor=value,
|
||||
tokens=tokens,
|
||||
is_root=True
|
||||
)
|
||||
|
||||
yield parse_root()
|
||||
|
||||
# path*
|
||||
while can_advance():
|
||||
path_tokens = [expect(TokenKind.LEFT_BRACKET)]
|
||||
token = expect(TokenKind.TEXT, TokenKind.NUMBER, TokenKind.RIGHT_BRACKET)
|
||||
path_tokens.append(token)
|
||||
if token.kind is TokenKind.RIGHT_BRACKET:
|
||||
path = Path(PathAction.APPEND, tokens=path_tokens)
|
||||
elif token.kind is TokenKind.TEXT:
|
||||
path = Path(PathAction.KEY, token.value, tokens=path_tokens)
|
||||
path_tokens.append(expect(TokenKind.RIGHT_BRACKET))
|
||||
elif token.kind is TokenKind.NUMBER:
|
||||
path = Path(PathAction.INDEX, token.value, tokens=path_tokens)
|
||||
path_tokens.append(expect(TokenKind.RIGHT_BRACKET))
|
||||
else:
|
||||
assert_cant_happen()
|
||||
# noinspection PyUnboundLocalVariable
|
||||
yield path
|
||||
|
||||
|
||||
def tokenize(source: str) -> Iterator[Token]:
|
||||
cursor = 0
|
||||
backslashes = 0
|
||||
buffer = []
|
||||
|
||||
def send_buffer() -> Iterator[Token]:
|
||||
nonlocal backslashes
|
||||
if not buffer:
|
||||
return None
|
||||
|
||||
value = ''.join(buffer)
|
||||
kind = TokenKind.TEXT
|
||||
if not backslashes:
|
||||
for variation, kind in [
|
||||
(int, TokenKind.NUMBER),
|
||||
(check_escaped_int, TokenKind.TEXT),
|
||||
]:
|
||||
try:
|
||||
value = variation(value)
|
||||
except ValueError:
|
||||
continue
|
||||
else:
|
||||
break
|
||||
yield Token(
|
||||
kind=kind,
|
||||
value=value,
|
||||
start=cursor - (len(buffer) + backslashes),
|
||||
end=cursor,
|
||||
)
|
||||
buffer.clear()
|
||||
backslashes = 0
|
||||
|
||||
def can_advance() -> bool:
|
||||
return cursor < len(source)
|
||||
|
||||
while can_advance():
|
||||
index = source[cursor]
|
||||
if index in OPERATORS:
|
||||
yield from send_buffer()
|
||||
yield Token(OPERATORS[index], index, cursor, cursor + 1)
|
||||
elif index == BACKSLASH and can_advance():
|
||||
if source[cursor + 1] in SPECIAL_CHARS:
|
||||
backslashes += 1
|
||||
else:
|
||||
buffer.append(index)
|
||||
buffer.append(source[cursor + 1])
|
||||
cursor += 1
|
||||
else:
|
||||
buffer.append(index)
|
||||
cursor += 1
|
||||
|
||||
yield from send_buffer()
|
||||
|
||||
|
||||
def check_escaped_int(value: str) -> str:
|
||||
if not value.startswith(BACKSLASH):
|
||||
raise ValueError('Not an escaped int')
|
||||
try:
|
||||
int(value[1:])
|
||||
except ValueError as exc:
|
||||
raise ValueError('Not an escaped int') from exc
|
||||
else:
|
||||
return value[1:]
|
||||
|
||||
|
||||
def assert_cant_happen():
|
||||
raise ValueError('Unexpected value')
|
||||
80
httpie/cli/nested_json/tokens.py
Normal file
80
httpie/cli/nested_json/tokens.py
Normal file
@ -0,0 +1,80 @@
|
||||
from enum import Enum, auto
|
||||
from typing import NamedTuple, Union, Optional, List
|
||||
|
||||
EMPTY_STRING = ''
|
||||
HIGHLIGHTER = '^'
|
||||
OPEN_BRACKET = '['
|
||||
CLOSE_BRACKET = ']'
|
||||
BACKSLASH = '\\'
|
||||
|
||||
|
||||
class TokenKind(Enum):
|
||||
TEXT = auto()
|
||||
NUMBER = auto()
|
||||
LEFT_BRACKET = auto()
|
||||
RIGHT_BRACKET = auto()
|
||||
PSEUDO = auto() # Not a real token, use when representing location only.
|
||||
|
||||
def to_name(self) -> str:
|
||||
for key, value in OPERATORS.items():
|
||||
if value is self:
|
||||
return repr(key)
|
||||
else:
|
||||
return 'a ' + self.name.lower()
|
||||
|
||||
|
||||
OPERATORS = {
|
||||
OPEN_BRACKET: TokenKind.LEFT_BRACKET,
|
||||
CLOSE_BRACKET: TokenKind.RIGHT_BRACKET,
|
||||
}
|
||||
SPECIAL_CHARS = OPERATORS.keys() | {BACKSLASH}
|
||||
LITERAL_TOKENS = [
|
||||
TokenKind.TEXT,
|
||||
TokenKind.NUMBER,
|
||||
]
|
||||
|
||||
|
||||
class Token(NamedTuple):
|
||||
kind: TokenKind
|
||||
value: Union[str, int]
|
||||
start: int
|
||||
end: int
|
||||
|
||||
|
||||
class PathAction(Enum):
|
||||
KEY = auto()
|
||||
INDEX = auto()
|
||||
APPEND = auto()
|
||||
# Pseudo action, used by the interpreter
|
||||
SET = auto()
|
||||
|
||||
def to_string(self) -> str:
|
||||
return self.name.lower()
|
||||
|
||||
|
||||
class Path:
|
||||
def __init__(
|
||||
self,
|
||||
kind: PathAction,
|
||||
accessor: Optional[Union[str, int]] = None,
|
||||
tokens: Optional[List[Token]] = None,
|
||||
is_root: bool = False,
|
||||
):
|
||||
self.kind = kind
|
||||
self.accessor = accessor
|
||||
self.tokens = tokens or []
|
||||
self.is_root = is_root
|
||||
|
||||
def reconstruct(self) -> str:
|
||||
if self.kind is PathAction.KEY:
|
||||
if self.is_root:
|
||||
return str(self.accessor)
|
||||
return OPEN_BRACKET + self.accessor + CLOSE_BRACKET
|
||||
elif self.kind is PathAction.INDEX:
|
||||
return OPEN_BRACKET + str(self.accessor) + CLOSE_BRACKET
|
||||
elif self.kind is PathAction.APPEND:
|
||||
return OPEN_BRACKET + CLOSE_BRACKET
|
||||
|
||||
|
||||
class NestedJSONArray(list):
|
||||
"""Denotes a top-level JSON array."""
|
||||
@ -18,7 +18,7 @@ from .dicts import (
|
||||
)
|
||||
from .exceptions import ParseError
|
||||
from .nested_json import interpret_nested_json
|
||||
from ..utils import get_content_type, load_json_preserve_order_and_dupe_keys, split
|
||||
from ..utils import get_content_type, load_json_preserve_order_and_dupe_keys, split_iterable
|
||||
|
||||
|
||||
class RequestItems:
|
||||
@ -78,25 +78,28 @@ class RequestItems:
|
||||
instance.data,
|
||||
),
|
||||
SEPARATOR_DATA_RAW_JSON: (
|
||||
json_only(instance, process_data_raw_json_embed_arg),
|
||||
convert_json_value_to_form_if_needed(
|
||||
in_json_mode=instance.is_json,
|
||||
processor=process_data_raw_json_embed_arg
|
||||
),
|
||||
instance.data,
|
||||
),
|
||||
SEPARATOR_DATA_EMBED_RAW_JSON_FILE: (
|
||||
json_only(instance, process_data_embed_raw_json_file_arg),
|
||||
convert_json_value_to_form_if_needed(
|
||||
in_json_mode=instance.is_json,
|
||||
processor=process_data_embed_raw_json_file_arg,
|
||||
),
|
||||
instance.data,
|
||||
),
|
||||
}
|
||||
|
||||
if instance.is_json:
|
||||
json_item_args, request_item_args = split(
|
||||
request_item_args,
|
||||
lambda arg: arg.sep in SEPARATOR_GROUP_NESTED_JSON_ITEMS
|
||||
json_item_args, request_item_args = split_iterable(
|
||||
iterable=request_item_args,
|
||||
key=lambda arg: arg.sep in SEPARATOR_GROUP_NESTED_JSON_ITEMS
|
||||
)
|
||||
if json_item_args:
|
||||
pairs = [
|
||||
(arg.key, rules[arg.sep][0](arg))
|
||||
for arg in json_item_args
|
||||
]
|
||||
pairs = [(arg.key, rules[arg.sep][0](arg)) for arg in json_item_args]
|
||||
processor_func, target_dict = rules[SEPARATOR_GROUP_NESTED_JSON_ITEMS]
|
||||
value = processor_func(pairs)
|
||||
target_dict.update(value)
|
||||
@ -159,6 +162,30 @@ def process_file_upload_arg(arg: KeyValueArg) -> Tuple[str, IO, str]:
|
||||
)
|
||||
|
||||
|
||||
def convert_json_value_to_form_if_needed(in_json_mode: bool, processor: Callable[[KeyValueArg], JSONType]) -> Callable[[], str]:
|
||||
"""
|
||||
We allow primitive values to be passed to forms via JSON key/value syntax.
|
||||
|
||||
But complex values lead to an error because there’s no clear way to serialize them.
|
||||
|
||||
"""
|
||||
if in_json_mode:
|
||||
return processor
|
||||
|
||||
@functools.wraps(processor)
|
||||
def wrapper(*args, **kwargs) -> str:
|
||||
try:
|
||||
output = processor(*args, **kwargs)
|
||||
except ParseError:
|
||||
output = None
|
||||
if isinstance(output, (str, int, float)):
|
||||
return str(output)
|
||||
else:
|
||||
raise ParseError('Cannot use complex JSON value types with --form/--multipart.')
|
||||
|
||||
return wrapper
|
||||
|
||||
|
||||
def process_data_item_arg(arg: KeyValueArg) -> str:
|
||||
return arg.value
|
||||
|
||||
@ -167,29 +194,6 @@ def process_data_embed_file_contents_arg(arg: KeyValueArg) -> str:
|
||||
return load_text_file(arg)
|
||||
|
||||
|
||||
def json_only(items: RequestItems, func: Callable[[KeyValueArg], JSONType]) -> str:
|
||||
if items.is_json:
|
||||
return func
|
||||
|
||||
@functools.wraps(func)
|
||||
def wrapper(*args, **kwargs) -> str:
|
||||
try:
|
||||
ret = func(*args, **kwargs)
|
||||
except ParseError:
|
||||
ret = None
|
||||
|
||||
# If it is a basic type, then allow it
|
||||
if isinstance(ret, (str, int, float)):
|
||||
return str(ret)
|
||||
else:
|
||||
raise ParseError(
|
||||
'Can\'t use complex JSON value types with '
|
||||
'--form/--multipart.'
|
||||
)
|
||||
|
||||
return wrapper
|
||||
|
||||
|
||||
def process_data_embed_raw_json_file_arg(arg: KeyValueArg) -> JSONType:
|
||||
contents = load_text_file(arg)
|
||||
value = load_json(arg, contents)
|
||||
|
||||
@ -10,12 +10,13 @@ from urllib.parse import urlparse, urlunparse
|
||||
import requests
|
||||
# noinspection PyPackageRequirements
|
||||
import urllib3
|
||||
|
||||
from . import __version__
|
||||
from .adapters import HTTPieHTTPAdapter
|
||||
from .context import Environment
|
||||
from .cli.constants import HTTP_OPTIONS
|
||||
from .cli.nested_json import EMPTY_STRING
|
||||
from .cli.dicts import HTTPHeadersDict, NestedJSONArray
|
||||
from .cli.dicts import HTTPHeadersDict
|
||||
from .cli.nested_json import unwrap_top_level_list_if_needed
|
||||
from .context import Environment
|
||||
from .encoding import UTF8
|
||||
from .models import RequestsMessage
|
||||
from .plugins.registry import plugin_manager
|
||||
@ -306,21 +307,13 @@ def make_send_kwargs_mergeable_from_env(args: argparse.Namespace) -> dict:
|
||||
|
||||
|
||||
def json_dict_to_request_body(data: Dict[str, Any]) -> str:
|
||||
# Propagate the top-level list if there is only one
|
||||
# item in the object, with an en empty key.
|
||||
if len(data) == 1:
|
||||
[(key, value)] = data.items()
|
||||
if isinstance(value, NestedJSONArray):
|
||||
assert key == EMPTY_STRING
|
||||
data = value
|
||||
|
||||
data = unwrap_top_level_list_if_needed(data)
|
||||
if data:
|
||||
data = json.dumps(data)
|
||||
else:
|
||||
# We need to set data to an empty string to prevent requests
|
||||
# from assigning an empty list to `response.request.data`.
|
||||
data = ''
|
||||
|
||||
return data
|
||||
|
||||
|
||||
|
||||
@ -11,7 +11,7 @@ from requests import __version__ as requests_version
|
||||
|
||||
from . import __version__ as httpie_version
|
||||
from .cli.constants import OUT_REQ_BODY
|
||||
from .cli.nested_json import HTTPieSyntaxError
|
||||
from .cli.nested_json import NestedJSONSyntaxError
|
||||
from .client import collect_messages
|
||||
from .context import Environment, LogLevel
|
||||
from .downloads import Downloader
|
||||
@ -78,7 +78,7 @@ def raw_main(
|
||||
args=args,
|
||||
env=env,
|
||||
)
|
||||
except HTTPieSyntaxError as exc:
|
||||
except NestedJSONSyntaxError as exc:
|
||||
env.stderr.write(str(exc) + "\n")
|
||||
if include_traceback:
|
||||
raise
|
||||
|
||||
@ -245,7 +245,7 @@ def get_site_paths(path: Path) -> Iterable[Path]:
|
||||
yield as_site(path)
|
||||
|
||||
|
||||
def split(iterable: Iterable[T], key: Callable[[T], bool]) -> Tuple[List[T], List[T]]:
|
||||
def split_iterable(iterable: Iterable[T], key: Callable[[T], bool]) -> Tuple[List[T], List[T]]:
|
||||
left, right = [], []
|
||||
for item in iterable:
|
||||
if key(item):
|
||||
|
||||
@ -5,7 +5,7 @@ import responses
|
||||
|
||||
from httpie.cli.constants import PRETTY_MAP
|
||||
from httpie.cli.exceptions import ParseError
|
||||
from httpie.cli.nested_json import HTTPieSyntaxError
|
||||
from httpie.cli.nested_json import NestedJSONSyntaxError
|
||||
from httpie.output.formatters.colors import ColorFormatter
|
||||
from httpie.utils import JsonDictPreservingDuplicateKeys
|
||||
|
||||
@ -157,7 +157,7 @@ def test_complex_json_arguments_with_non_json(httpbin, request_type, value):
|
||||
f'option:={json.dumps(value)}',
|
||||
)
|
||||
|
||||
cm.match("Can't use complex JSON value types")
|
||||
cm.match('Cannot use complex JSON value types')
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
@ -508,23 +508,23 @@ def test_nested_json_syntax(input_json, expected_json, httpbin):
|
||||
),
|
||||
(
|
||||
['foo=1', 'foo[key]:=2'],
|
||||
"HTTPie Type Error: Can't perform 'key' based access on 'foo' which has a type of 'string' but this operation requires a type of 'object'.\nfoo[key]\n ^^^^^",
|
||||
"HTTPie Type Error: Cannot perform 'key' based access on 'foo' which has a type of 'string' but this operation requires a type of 'object'.\nfoo[key]\n ^^^^^",
|
||||
),
|
||||
(
|
||||
['foo=1', 'foo[0]:=2'],
|
||||
"HTTPie Type Error: Can't perform 'index' based access on 'foo' which has a type of 'string' but this operation requires a type of 'array'.\nfoo[0]\n ^^^",
|
||||
"HTTPie Type Error: Cannot perform 'index' based access on 'foo' which has a type of 'string' but this operation requires a type of 'array'.\nfoo[0]\n ^^^",
|
||||
),
|
||||
(
|
||||
['foo=1', 'foo[]:=2'],
|
||||
"HTTPie Type Error: Can't perform 'append' based access on 'foo' which has a type of 'string' but this operation requires a type of 'array'.\nfoo[]\n ^^",
|
||||
"HTTPie Type Error: Cannot perform 'append' based access on 'foo' which has a type of 'string' but this operation requires a type of 'array'.\nfoo[]\n ^^",
|
||||
),
|
||||
(
|
||||
['data[key]=value', 'data[key 2]=value 2', 'data[0]=value'],
|
||||
"HTTPie Type Error: Can't perform 'index' based access on 'data' which has a type of 'object' but this operation requires a type of 'array'.\ndata[0]\n ^^^",
|
||||
"HTTPie Type Error: Cannot perform 'index' based access on 'data' which has a type of 'object' but this operation requires a type of 'array'.\ndata[0]\n ^^^",
|
||||
),
|
||||
(
|
||||
['data[key]=value', 'data[key 2]=value 2', 'data[]=value'],
|
||||
"HTTPie Type Error: Can't perform 'append' based access on 'data' which has a type of 'object' but this operation requires a type of 'array'.\ndata[]\n ^^",
|
||||
"HTTPie Type Error: Cannot perform 'append' based access on 'data' which has a type of 'object' but this operation requires a type of 'array'.\ndata[]\n ^^",
|
||||
),
|
||||
(
|
||||
[
|
||||
@ -532,7 +532,7 @@ def test_nested_json_syntax(input_json, expected_json, httpbin):
|
||||
'foo[bar][baz][5][]:=4',
|
||||
'foo[bar][baz][key][]:=5',
|
||||
],
|
||||
"HTTPie Type Error: Can't perform 'key' based access on 'foo[bar][baz]' which has a type of 'array' but this operation requires a type of 'object'.\nfoo[bar][baz][key][]\n ^^^^^",
|
||||
"HTTPie Type Error: Cannot perform 'key' based access on 'foo[bar][baz]' which has a type of 'array' but this operation requires a type of 'object'.\nfoo[bar][baz][key][]\n ^^^^^",
|
||||
),
|
||||
(
|
||||
['foo[-10]:=[1,2]'],
|
||||
@ -540,32 +540,32 @@ def test_nested_json_syntax(input_json, expected_json, httpbin):
|
||||
),
|
||||
(
|
||||
['foo[0]:=1', 'foo[]:=2', 'foo[\\2]:=3'],
|
||||
"HTTPie Type Error: Can't perform 'key' based access on 'foo' which has a type of 'array' but this operation requires a type of 'object'.\nfoo[\\2]\n ^^^^",
|
||||
"HTTPie Type Error: Cannot perform 'key' based access on 'foo' which has a type of 'array' but this operation requires a type of 'object'.\nfoo[\\2]\n ^^^^",
|
||||
),
|
||||
(
|
||||
['foo[\\1]:=2', 'foo[5]:=3'],
|
||||
"HTTPie Type Error: Can't perform 'index' based access on 'foo' which has a type of 'object' but this operation requires a type of 'array'.\nfoo[5]\n ^^^",
|
||||
"HTTPie Type Error: Cannot perform 'index' based access on 'foo' which has a type of 'object' but this operation requires a type of 'array'.\nfoo[5]\n ^^^",
|
||||
),
|
||||
(
|
||||
['x=y', '[]:=2'],
|
||||
"HTTPie Type Error: Can't perform 'append' based access on '' which has a type of 'object' but this operation requires a type of 'array'.",
|
||||
"HTTPie Type Error: Cannot perform 'append' based access on '' which has a type of 'object' but this operation requires a type of 'array'.",
|
||||
),
|
||||
(
|
||||
['[]:=2', 'x=y'],
|
||||
"HTTPie Type Error: Can't perform 'key' based access on '' which has a type of 'array' but this operation requires a type of 'object'.",
|
||||
"HTTPie Type Error: Cannot perform 'key' based access on '' which has a type of 'array' but this operation requires a type of 'object'.",
|
||||
),
|
||||
(
|
||||
[':=[1,2,3]', '[]:=4'],
|
||||
"HTTPie Type Error: Can't perform 'append' based access on '' which has a type of 'object' but this operation requires a type of 'array'.",
|
||||
"HTTPie Type Error: Cannot perform 'append' based access on '' which has a type of 'object' but this operation requires a type of 'array'.",
|
||||
),
|
||||
(
|
||||
['[]:=4', ':=[1,2,3]'],
|
||||
"HTTPie Type Error: Can't perform 'key' based access on '' which has a type of 'array' but this operation requires a type of 'object'.",
|
||||
"HTTPie Type Error: Cannot perform 'key' based access on '' which has a type of 'array' but this operation requires a type of 'object'.",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_nested_json_errors(input_json, expected_error, httpbin):
|
||||
with pytest.raises(HTTPieSyntaxError) as exc:
|
||||
with pytest.raises(NestedJSONSyntaxError) as exc:
|
||||
http(httpbin + '/post', *input_json)
|
||||
|
||||
exc_lines = str(exc.value).splitlines()
|
||||
|
||||
Loading…
Reference in New Issue
Block a user