From 0689b55e1d0a16385b7e275db4458c155d746b6a Mon Sep 17 00:00:00 2001 From: Jakub Roztocil Date: Sat, 1 Oct 2022 12:38:19 +0200 Subject: [PATCH] Clean up and refactor nested JSON parsing & interpreting (#1440) --- extras/scripts/generate_man_pages.py | 2 +- httpie/cli/dicts.py | 4 - httpie/cli/nested_json.py | 404 --------------------------- httpie/cli/nested_json/__init__.py | 20 ++ httpie/cli/nested_json/errors.py | 27 ++ httpie/cli/nested_json/interpret.py | 129 +++++++++ httpie/cli/nested_json/parse.py | 193 +++++++++++++ httpie/cli/nested_json/tokens.py | 80 ++++++ httpie/cli/requestitems.py | 70 ++--- httpie/client.py | 17 +- httpie/core.py | 4 +- httpie/utils.py | 2 +- tests/test_json.py | 30 +- 13 files changed, 510 insertions(+), 472 deletions(-) delete mode 100644 httpie/cli/nested_json.py create mode 100644 httpie/cli/nested_json/__init__.py create mode 100644 httpie/cli/nested_json/errors.py create mode 100644 httpie/cli/nested_json/interpret.py create mode 100644 httpie/cli/nested_json/parse.py create mode 100644 httpie/cli/nested_json/tokens.py diff --git a/extras/scripts/generate_man_pages.py b/extras/scripts/generate_man_pages.py index 53034886..71e0100f 100644 --- a/extras/scripts/generate_man_pages.py +++ b/extras/scripts/generate_man_pages.py @@ -9,7 +9,7 @@ from httpie.cli.options import ParserSpec from httpie.manager.cli import options as manager_options from httpie.output.ui.rich_help import OptionsHighlighter, to_usage from httpie.output.ui.rich_utils import render_as_string -from httpie.utils import split +from httpie.utils import split_iterable # Escape certain characters so they are rendered properly on diff --git a/httpie/cli/dicts.py b/httpie/cli/dicts.py index 074cd9e7..6b6d4736 100644 --- a/httpie/cli/dicts.py +++ b/httpie/cli/dicts.py @@ -92,7 +92,3 @@ class MultipartRequestDataDict(MultiValueOrderedDict): class RequestFilesDict(RequestDataDict): pass - - -class NestedJSONArray(list): - """Denotes a top-level JSON array.""" diff --git a/httpie/cli/nested_json.py b/httpie/cli/nested_json.py deleted file mode 100644 index 6616e9ac..00000000 --- a/httpie/cli/nested_json.py +++ /dev/null @@ -1,404 +0,0 @@ -from enum import Enum, auto -from typing import ( - Any, - Iterator, - NamedTuple, - Optional, - List, - NoReturn, - Type, - Union, -) -from .dicts import NestedJSONArray - - -EMPTY_STRING = '' -HIGHLIGHTER = '^' -OPEN_BRACKET = '[' -CLOSE_BRACKET = ']' -BACKSLASH = '\\' - - -class HTTPieSyntaxError(ValueError): - def __init__( - self, - source: str, - token: Optional['Token'], - message: str, - message_kind: str = 'Syntax', - ) -> None: - self.source = source - self.token = token - self.message = message - self.message_kind = message_kind - - def __str__(self): - lines = [f'HTTPie {self.message_kind} Error: {self.message}'] - if self.token is not None: - lines.append(self.source) - lines.append( - ' ' * self.token.start - + HIGHLIGHTER * (self.token.end - self.token.start) - ) - return '\n'.join(lines) - - -class TokenKind(Enum): - TEXT = auto() - NUMBER = auto() - LEFT_BRACKET = auto() - RIGHT_BRACKET = auto() - - def to_name(self) -> str: - for key, value in OPERATORS.items(): - if value is self: - return repr(key) - else: - return 'a ' + self.name.lower() - - -OPERATORS = { - OPEN_BRACKET: TokenKind.LEFT_BRACKET, - CLOSE_BRACKET: TokenKind.RIGHT_BRACKET, -} -SPECIAL_CHARS = OPERATORS.keys() | {BACKSLASH} -LITERAL_TOKENS = [ - TokenKind.TEXT, - TokenKind.NUMBER, -] - - -class Token(NamedTuple): - kind: TokenKind - value: Union[str, int] - start: int - end: int - - -def assert_cant_happen() -> NoReturn: - raise ValueError('Unexpected value') - - -def check_escaped_int(value: str) -> str: - if not value.startswith(BACKSLASH): - raise ValueError('Not an escaped int') - - try: - int(value[1:]) - except ValueError as exc: - raise ValueError('Not an escaped int') from exc - else: - return value[1:] - - -def tokenize(source: str) -> Iterator[Token]: - cursor = 0 - backslashes = 0 - buffer = [] - - def send_buffer() -> Iterator[Token]: - nonlocal backslashes - if not buffer: - return None - - value = ''.join(buffer) - kind = TokenKind.TEXT - if not backslashes: - for variation, kind in [ - (int, TokenKind.NUMBER), - (check_escaped_int, TokenKind.TEXT), - ]: - try: - value = variation(value) - except ValueError: - continue - else: - break - - yield Token( - kind, value, start=cursor - (len(buffer) + backslashes), end=cursor - ) - buffer.clear() - backslashes = 0 - - def can_advance() -> bool: - return cursor < len(source) - - while can_advance(): - index = source[cursor] - if index in OPERATORS: - yield from send_buffer() - yield Token(OPERATORS[index], index, cursor, cursor + 1) - elif index == BACKSLASH and can_advance(): - if source[cursor + 1] in SPECIAL_CHARS: - backslashes += 1 - else: - buffer.append(index) - - buffer.append(source[cursor + 1]) - cursor += 1 - else: - buffer.append(index) - - cursor += 1 - - yield from send_buffer() - - -class PathAction(Enum): - KEY = auto() - INDEX = auto() - APPEND = auto() - - # Pseudo action, used by the interpreter - SET = auto() - - def to_string(self) -> str: - return self.name.lower() - - -class Path: - def __init__( - self, - kind: PathAction, - accessor: Optional[Union[str, int]] = None, - tokens: Optional[List[Token]] = None, - is_root: bool = False, - ): - self.kind = kind - self.accessor = accessor - self.tokens = tokens or [] - self.is_root = is_root - - def reconstruct(self) -> str: - if self.kind is PathAction.KEY: - if self.is_root: - return str(self.accessor) - return OPEN_BRACKET + self.accessor + CLOSE_BRACKET - elif self.kind is PathAction.INDEX: - return OPEN_BRACKET + str(self.accessor) + CLOSE_BRACKET - elif self.kind is PathAction.APPEND: - return OPEN_BRACKET + CLOSE_BRACKET - else: - assert_cant_happen() - - -def parse(source: str) -> Iterator[Path]: - """ - start: root_path path* - root_path: (literal | index_path | append_path) - literal: TEXT | NUMBER - - path: - key_path - | index_path - | append_path - key_path: LEFT_BRACKET TEXT RIGHT_BRACKET - index_path: LEFT_BRACKET NUMBER RIGHT_BRACKET - append_path: LEFT_BRACKET RIGHT_BRACKET - """ - - tokens = list(tokenize(source)) - cursor = 0 - - def can_advance(): - return cursor < len(tokens) - - def expect(*kinds): - nonlocal cursor - - assert len(kinds) > 0 - if can_advance(): - token = tokens[cursor] - cursor += 1 - if token.kind in kinds: - return token - elif tokens: - token = tokens[-1]._replace( - start=tokens[-1].end + 0, end=tokens[-1].end + 1 - ) - else: - token = None - - if len(kinds) == 1: - suffix = kinds[0].to_name() - else: - suffix = ', '.join(kind.to_name() for kind in kinds[:-1]) - suffix += ' or ' + kinds[-1].to_name() - - message = f'Expecting {suffix}' - raise HTTPieSyntaxError(source, token, message) - - def parse_root(): - tokens = [] - if not can_advance(): - return Path( - PathAction.KEY, - EMPTY_STRING, - is_root=True - ) - - # (literal | index_path | append_path)? - token = expect(*LITERAL_TOKENS, TokenKind.LEFT_BRACKET) - tokens.append(token) - - if token.kind in LITERAL_TOKENS: - action = PathAction.KEY - value = str(token.value) - elif token.kind is TokenKind.LEFT_BRACKET: - token = expect(TokenKind.NUMBER, TokenKind.RIGHT_BRACKET) - tokens.append(token) - if token.kind is TokenKind.NUMBER: - action = PathAction.INDEX - value = token.value - tokens.append(expect(TokenKind.RIGHT_BRACKET)) - elif token.kind is TokenKind.RIGHT_BRACKET: - action = PathAction.APPEND - value = None - else: - assert_cant_happen() - else: - assert_cant_happen() - - return Path( - action, - value, - tokens=tokens, - is_root=True - ) - - yield parse_root() - - # path* - while can_advance(): - path_tokens = [] - path_tokens.append(expect(TokenKind.LEFT_BRACKET)) - - token = expect( - TokenKind.TEXT, TokenKind.NUMBER, TokenKind.RIGHT_BRACKET - ) - path_tokens.append(token) - if token.kind is TokenKind.RIGHT_BRACKET: - path = Path(PathAction.APPEND, tokens=path_tokens) - elif token.kind is TokenKind.TEXT: - path = Path(PathAction.KEY, token.value, tokens=path_tokens) - path_tokens.append(expect(TokenKind.RIGHT_BRACKET)) - elif token.kind is TokenKind.NUMBER: - path = Path(PathAction.INDEX, token.value, tokens=path_tokens) - path_tokens.append(expect(TokenKind.RIGHT_BRACKET)) - else: - assert_cant_happen() - yield path - - -JSON_TYPE_MAPPING = { - dict: 'object', - list: 'array', - int: 'number', - float: 'number', - str: 'string', -} - - -def interpret(context: Any, key: str, value: Any) -> Any: - cursor = context - - paths = list(parse(key)) - paths.append(Path(PathAction.SET, value)) - - def type_check(index: int, path: Path, expected_type: Type[Any]) -> None: - if not isinstance(cursor, expected_type): - if path.tokens: - pseudo_token = Token( - None, None, path.tokens[0].start, path.tokens[-1].end - ) - else: - pseudo_token = None - - cursor_type = JSON_TYPE_MAPPING.get( - type(cursor), type(cursor).__name__ - ) - required_type = JSON_TYPE_MAPPING[expected_type] - - message = f"Can't perform {path.kind.to_string()!r} based access on " - message += repr( - ''.join(path.reconstruct() for path in paths[:index]) - ) - message += ( - f' which has a type of {cursor_type!r} but this operation' - ) - message += f' requires a type of {required_type!r}.' - raise HTTPieSyntaxError( - key, pseudo_token, message, message_kind='Type' - ) - - def object_for(kind: str) -> Any: - if kind is PathAction.KEY: - return {} - elif kind in {PathAction.INDEX, PathAction.APPEND}: - return [] - else: - assert_cant_happen() - - for index, (path, next_path) in enumerate(zip(paths, paths[1:])): - # If there is no context yet, set it. - if cursor is None: - context = cursor = object_for(path.kind) - - if path.kind is PathAction.KEY: - type_check(index, path, dict) - if next_path.kind is PathAction.SET: - cursor[path.accessor] = next_path.accessor - break - - cursor = cursor.setdefault( - path.accessor, object_for(next_path.kind) - ) - elif path.kind is PathAction.INDEX: - type_check(index, path, list) - if path.accessor < 0: - raise HTTPieSyntaxError( - key, - path.tokens[1], - 'Negative indexes are not supported.', - message_kind='Value', - ) - cursor.extend([None] * (path.accessor - len(cursor) + 1)) - if next_path.kind is PathAction.SET: - cursor[path.accessor] = next_path.accessor - break - - if cursor[path.accessor] is None: - cursor[path.accessor] = object_for(next_path.kind) - - cursor = cursor[path.accessor] - elif path.kind is PathAction.APPEND: - type_check(index, path, list) - if next_path.kind is PathAction.SET: - cursor.append(next_path.accessor) - break - - cursor.append(object_for(next_path.kind)) - cursor = cursor[-1] - else: - assert_cant_happen() - - return context - - -def wrap_with_dict(context): - if context is None: - return {} - elif isinstance(context, list): - return {EMPTY_STRING: NestedJSONArray(context)} - else: - assert isinstance(context, dict) - return context - - -def interpret_nested_json(pairs): - context = None - for key, value in pairs: - context = interpret(context, key, value) - - return wrap_with_dict(context) diff --git a/httpie/cli/nested_json/__init__.py b/httpie/cli/nested_json/__init__.py new file mode 100644 index 00000000..17b129ab --- /dev/null +++ b/httpie/cli/nested_json/__init__.py @@ -0,0 +1,20 @@ +""" +A library for parsing the HTTPie nested JSON key syntax and constructing the resulting objects. + + + +It has no dependencies. + +""" +from .interpret import interpret_nested_json, unwrap_top_level_list_if_needed +from .errors import NestedJSONSyntaxError +from .tokens import EMPTY_STRING, NestedJSONArray + + +__all__ = [ + 'interpret_nested_json', + 'unwrap_top_level_list_if_needed', + 'EMPTY_STRING', + 'NestedJSONArray', + 'NestedJSONSyntaxError' +] diff --git a/httpie/cli/nested_json/errors.py b/httpie/cli/nested_json/errors.py new file mode 100644 index 00000000..f53f8715 --- /dev/null +++ b/httpie/cli/nested_json/errors.py @@ -0,0 +1,27 @@ +from typing import Optional + +from .tokens import Token, HIGHLIGHTER + + +class NestedJSONSyntaxError(ValueError): + def __init__( + self, + source: str, + token: Optional[Token], + message: str, + message_kind: str = 'Syntax', + ) -> None: + self.source = source + self.token = token + self.message = message + self.message_kind = message_kind + + def __str__(self): + lines = [f'HTTPie {self.message_kind} Error: {self.message}'] + if self.token is not None: + lines.append(self.source) + lines.append( + ' ' * self.token.start + + HIGHLIGHTER * (self.token.end - self.token.start) + ) + return '\n'.join(lines) diff --git a/httpie/cli/nested_json/interpret.py b/httpie/cli/nested_json/interpret.py new file mode 100644 index 00000000..71fad98a --- /dev/null +++ b/httpie/cli/nested_json/interpret.py @@ -0,0 +1,129 @@ +from typing import Type, Union, Any, Iterable, Tuple + +from .parse import parse, assert_cant_happen +from .errors import NestedJSONSyntaxError +from .tokens import EMPTY_STRING, TokenKind, Token, PathAction, Path, NestedJSONArray + + +__all__ = [ + 'interpret_nested_json', + 'unwrap_top_level_list_if_needed', +] + +JSONType = Type[Union[dict, list, int, float, str]] +JSON_TYPE_MAPPING = { + dict: 'object', + list: 'array', + int: 'number', + float: 'number', + str: 'string', +} + + +def interpret_nested_json(pairs: Iterable[Tuple[str, str]]) -> dict: + context = None + for key, value in pairs: + context = interpret(context, key, value) + return wrap_with_dict(context) + + +def interpret(context: Any, key: str, value: Any) -> Any: + cursor = context + paths = list(parse(key)) + paths.append(Path(PathAction.SET, value)) + + # noinspection PyShadowingNames + def type_check(index: int, path: Path, expected_type: JSONType): + if not isinstance(cursor, expected_type): + if path.tokens: + pseudo_token = Token( + kind=TokenKind.PSEUDO, + value='', + start=path.tokens[0].start, + end=path.tokens[-1].end, + ) + else: + pseudo_token = None + cursor_type = JSON_TYPE_MAPPING.get(type(cursor), type(cursor).__name__) + required_type = JSON_TYPE_MAPPING[expected_type] + message = f'Cannot perform {path.kind.to_string()!r} based access on ' + message += repr(''.join(path.reconstruct() for path in paths[:index])) + message += f' which has a type of {cursor_type!r} but this operation' + message += f' requires a type of {required_type!r}.' + raise NestedJSONSyntaxError( + source=key, + token=pseudo_token, + message=message, + message_kind='Type', + ) + + def object_for(kind: PathAction) -> Any: + if kind is PathAction.KEY: + return {} + elif kind in {PathAction.INDEX, PathAction.APPEND}: + return [] + else: + assert_cant_happen() + + for index, (path, next_path) in enumerate(zip(paths, paths[1:])): + # If there is no context yet, set it. + if cursor is None: + context = cursor = object_for(path.kind) + if path.kind is PathAction.KEY: + type_check(index, path, dict) + if next_path.kind is PathAction.SET: + cursor[path.accessor] = next_path.accessor + break + cursor = cursor.setdefault(path.accessor, object_for(next_path.kind)) + elif path.kind is PathAction.INDEX: + type_check(index, path, list) + if path.accessor < 0: + raise NestedJSONSyntaxError( + source=key, + token=path.tokens[1], + message='Negative indexes are not supported.', + message_kind='Value', + ) + cursor.extend([None] * (path.accessor - len(cursor) + 1)) + if next_path.kind is PathAction.SET: + cursor[path.accessor] = next_path.accessor + break + if cursor[path.accessor] is None: + cursor[path.accessor] = object_for(next_path.kind) + cursor = cursor[path.accessor] + elif path.kind is PathAction.APPEND: + type_check(index, path, list) + if next_path.kind is PathAction.SET: + cursor.append(next_path.accessor) + break + cursor.append(object_for(next_path.kind)) + cursor = cursor[-1] + else: + assert_cant_happen() + + return context + + +def wrap_with_dict(context): + if context is None: + return {} + elif isinstance(context, list): + return { + EMPTY_STRING: NestedJSONArray(context), + } + else: + assert isinstance(context, dict) + return context + + +def unwrap_top_level_list_if_needed(data: dict): + """ + Propagate the top-level list, if that’s what we got. + + """ + if len(data) == 1: + key, value = list(data.items())[0] + if isinstance(value, NestedJSONArray): + assert key == EMPTY_STRING + return value + return data diff --git a/httpie/cli/nested_json/parse.py b/httpie/cli/nested_json/parse.py new file mode 100644 index 00000000..323a22ee --- /dev/null +++ b/httpie/cli/nested_json/parse.py @@ -0,0 +1,193 @@ +from typing import Iterator + +from .errors import NestedJSONSyntaxError +from .tokens import ( + EMPTY_STRING, + BACKSLASH, + TokenKind, + OPERATORS, + SPECIAL_CHARS, + LITERAL_TOKENS, + Token, + PathAction, + Path, +) + + +__all__ = [ + 'parse', + 'assert_cant_happen', +] + + +def parse(source: str) -> Iterator[Path]: + """ + start: root_path path* + root_path: (literal | index_path | append_path) + literal: TEXT | NUMBER + + path: + key_path + | index_path + | append_path + key_path: LEFT_BRACKET TEXT RIGHT_BRACKET + index_path: LEFT_BRACKET NUMBER RIGHT_BRACKET + append_path: LEFT_BRACKET RIGHT_BRACKET + + """ + + tokens = list(tokenize(source)) + cursor = 0 + + def can_advance(): + return cursor < len(tokens) + + # noinspection PyShadowingNames + def expect(*kinds): + nonlocal cursor + assert kinds + if can_advance(): + token = tokens[cursor] + cursor += 1 + if token.kind in kinds: + return token + elif tokens: + token = tokens[-1]._replace( + start=tokens[-1].end + 0, + end=tokens[-1].end + 1, + ) + else: + token = None + if len(kinds) == 1: + suffix = kinds[0].to_name() + else: + suffix = ', '.join(kind.to_name() for kind in kinds[:-1]) + suffix += ' or ' + kinds[-1].to_name() + message = f'Expecting {suffix}' + raise NestedJSONSyntaxError(source, token, message) + + # noinspection PyShadowingNames + def parse_root(): + tokens = [] + if not can_advance(): + return Path( + kind=PathAction.KEY, + accessor=EMPTY_STRING, + is_root=True + ) + # (literal | index_path | append_path)? + token = expect(*LITERAL_TOKENS, TokenKind.LEFT_BRACKET) + tokens.append(token) + if token.kind in LITERAL_TOKENS: + action = PathAction.KEY + value = str(token.value) + elif token.kind is TokenKind.LEFT_BRACKET: + token = expect(TokenKind.NUMBER, TokenKind.RIGHT_BRACKET) + tokens.append(token) + if token.kind is TokenKind.NUMBER: + action = PathAction.INDEX + value = token.value + tokens.append(expect(TokenKind.RIGHT_BRACKET)) + elif token.kind is TokenKind.RIGHT_BRACKET: + action = PathAction.APPEND + value = None + else: + assert_cant_happen() + else: + assert_cant_happen() + # noinspection PyUnboundLocalVariable + return Path( + kind=action, + accessor=value, + tokens=tokens, + is_root=True + ) + + yield parse_root() + + # path* + while can_advance(): + path_tokens = [expect(TokenKind.LEFT_BRACKET)] + token = expect(TokenKind.TEXT, TokenKind.NUMBER, TokenKind.RIGHT_BRACKET) + path_tokens.append(token) + if token.kind is TokenKind.RIGHT_BRACKET: + path = Path(PathAction.APPEND, tokens=path_tokens) + elif token.kind is TokenKind.TEXT: + path = Path(PathAction.KEY, token.value, tokens=path_tokens) + path_tokens.append(expect(TokenKind.RIGHT_BRACKET)) + elif token.kind is TokenKind.NUMBER: + path = Path(PathAction.INDEX, token.value, tokens=path_tokens) + path_tokens.append(expect(TokenKind.RIGHT_BRACKET)) + else: + assert_cant_happen() + # noinspection PyUnboundLocalVariable + yield path + + +def tokenize(source: str) -> Iterator[Token]: + cursor = 0 + backslashes = 0 + buffer = [] + + def send_buffer() -> Iterator[Token]: + nonlocal backslashes + if not buffer: + return None + + value = ''.join(buffer) + kind = TokenKind.TEXT + if not backslashes: + for variation, kind in [ + (int, TokenKind.NUMBER), + (check_escaped_int, TokenKind.TEXT), + ]: + try: + value = variation(value) + except ValueError: + continue + else: + break + yield Token( + kind=kind, + value=value, + start=cursor - (len(buffer) + backslashes), + end=cursor, + ) + buffer.clear() + backslashes = 0 + + def can_advance() -> bool: + return cursor < len(source) + + while can_advance(): + index = source[cursor] + if index in OPERATORS: + yield from send_buffer() + yield Token(OPERATORS[index], index, cursor, cursor + 1) + elif index == BACKSLASH and can_advance(): + if source[cursor + 1] in SPECIAL_CHARS: + backslashes += 1 + else: + buffer.append(index) + buffer.append(source[cursor + 1]) + cursor += 1 + else: + buffer.append(index) + cursor += 1 + + yield from send_buffer() + + +def check_escaped_int(value: str) -> str: + if not value.startswith(BACKSLASH): + raise ValueError('Not an escaped int') + try: + int(value[1:]) + except ValueError as exc: + raise ValueError('Not an escaped int') from exc + else: + return value[1:] + + +def assert_cant_happen(): + raise ValueError('Unexpected value') diff --git a/httpie/cli/nested_json/tokens.py b/httpie/cli/nested_json/tokens.py new file mode 100644 index 00000000..e8f3f4c1 --- /dev/null +++ b/httpie/cli/nested_json/tokens.py @@ -0,0 +1,80 @@ +from enum import Enum, auto +from typing import NamedTuple, Union, Optional, List + +EMPTY_STRING = '' +HIGHLIGHTER = '^' +OPEN_BRACKET = '[' +CLOSE_BRACKET = ']' +BACKSLASH = '\\' + + +class TokenKind(Enum): + TEXT = auto() + NUMBER = auto() + LEFT_BRACKET = auto() + RIGHT_BRACKET = auto() + PSEUDO = auto() # Not a real token, use when representing location only. + + def to_name(self) -> str: + for key, value in OPERATORS.items(): + if value is self: + return repr(key) + else: + return 'a ' + self.name.lower() + + +OPERATORS = { + OPEN_BRACKET: TokenKind.LEFT_BRACKET, + CLOSE_BRACKET: TokenKind.RIGHT_BRACKET, +} +SPECIAL_CHARS = OPERATORS.keys() | {BACKSLASH} +LITERAL_TOKENS = [ + TokenKind.TEXT, + TokenKind.NUMBER, +] + + +class Token(NamedTuple): + kind: TokenKind + value: Union[str, int] + start: int + end: int + + +class PathAction(Enum): + KEY = auto() + INDEX = auto() + APPEND = auto() + # Pseudo action, used by the interpreter + SET = auto() + + def to_string(self) -> str: + return self.name.lower() + + +class Path: + def __init__( + self, + kind: PathAction, + accessor: Optional[Union[str, int]] = None, + tokens: Optional[List[Token]] = None, + is_root: bool = False, + ): + self.kind = kind + self.accessor = accessor + self.tokens = tokens or [] + self.is_root = is_root + + def reconstruct(self) -> str: + if self.kind is PathAction.KEY: + if self.is_root: + return str(self.accessor) + return OPEN_BRACKET + self.accessor + CLOSE_BRACKET + elif self.kind is PathAction.INDEX: + return OPEN_BRACKET + str(self.accessor) + CLOSE_BRACKET + elif self.kind is PathAction.APPEND: + return OPEN_BRACKET + CLOSE_BRACKET + + +class NestedJSONArray(list): + """Denotes a top-level JSON array.""" diff --git a/httpie/cli/requestitems.py b/httpie/cli/requestitems.py index 96731b59..8931b88a 100644 --- a/httpie/cli/requestitems.py +++ b/httpie/cli/requestitems.py @@ -18,7 +18,7 @@ from .dicts import ( ) from .exceptions import ParseError from .nested_json import interpret_nested_json -from ..utils import get_content_type, load_json_preserve_order_and_dupe_keys, split +from ..utils import get_content_type, load_json_preserve_order_and_dupe_keys, split_iterable class RequestItems: @@ -78,25 +78,28 @@ class RequestItems: instance.data, ), SEPARATOR_DATA_RAW_JSON: ( - json_only(instance, process_data_raw_json_embed_arg), + convert_json_value_to_form_if_needed( + in_json_mode=instance.is_json, + processor=process_data_raw_json_embed_arg + ), instance.data, ), SEPARATOR_DATA_EMBED_RAW_JSON_FILE: ( - json_only(instance, process_data_embed_raw_json_file_arg), + convert_json_value_to_form_if_needed( + in_json_mode=instance.is_json, + processor=process_data_embed_raw_json_file_arg, + ), instance.data, ), } if instance.is_json: - json_item_args, request_item_args = split( - request_item_args, - lambda arg: arg.sep in SEPARATOR_GROUP_NESTED_JSON_ITEMS + json_item_args, request_item_args = split_iterable( + iterable=request_item_args, + key=lambda arg: arg.sep in SEPARATOR_GROUP_NESTED_JSON_ITEMS ) if json_item_args: - pairs = [ - (arg.key, rules[arg.sep][0](arg)) - for arg in json_item_args - ] + pairs = [(arg.key, rules[arg.sep][0](arg)) for arg in json_item_args] processor_func, target_dict = rules[SEPARATOR_GROUP_NESTED_JSON_ITEMS] value = processor_func(pairs) target_dict.update(value) @@ -159,6 +162,30 @@ def process_file_upload_arg(arg: KeyValueArg) -> Tuple[str, IO, str]: ) +def convert_json_value_to_form_if_needed(in_json_mode: bool, processor: Callable[[KeyValueArg], JSONType]) -> Callable[[], str]: + """ + We allow primitive values to be passed to forms via JSON key/value syntax. + + But complex values lead to an error because there’s no clear way to serialize them. + + """ + if in_json_mode: + return processor + + @functools.wraps(processor) + def wrapper(*args, **kwargs) -> str: + try: + output = processor(*args, **kwargs) + except ParseError: + output = None + if isinstance(output, (str, int, float)): + return str(output) + else: + raise ParseError('Cannot use complex JSON value types with --form/--multipart.') + + return wrapper + + def process_data_item_arg(arg: KeyValueArg) -> str: return arg.value @@ -167,29 +194,6 @@ def process_data_embed_file_contents_arg(arg: KeyValueArg) -> str: return load_text_file(arg) -def json_only(items: RequestItems, func: Callable[[KeyValueArg], JSONType]) -> str: - if items.is_json: - return func - - @functools.wraps(func) - def wrapper(*args, **kwargs) -> str: - try: - ret = func(*args, **kwargs) - except ParseError: - ret = None - - # If it is a basic type, then allow it - if isinstance(ret, (str, int, float)): - return str(ret) - else: - raise ParseError( - 'Can\'t use complex JSON value types with ' - '--form/--multipart.' - ) - - return wrapper - - def process_data_embed_raw_json_file_arg(arg: KeyValueArg) -> JSONType: contents = load_text_file(arg) value = load_json(arg, contents) diff --git a/httpie/client.py b/httpie/client.py index 97608e78..815ec559 100644 --- a/httpie/client.py +++ b/httpie/client.py @@ -10,12 +10,13 @@ from urllib.parse import urlparse, urlunparse import requests # noinspection PyPackageRequirements import urllib3 + from . import __version__ from .adapters import HTTPieHTTPAdapter -from .context import Environment from .cli.constants import HTTP_OPTIONS -from .cli.nested_json import EMPTY_STRING -from .cli.dicts import HTTPHeadersDict, NestedJSONArray +from .cli.dicts import HTTPHeadersDict +from .cli.nested_json import unwrap_top_level_list_if_needed +from .context import Environment from .encoding import UTF8 from .models import RequestsMessage from .plugins.registry import plugin_manager @@ -306,21 +307,13 @@ def make_send_kwargs_mergeable_from_env(args: argparse.Namespace) -> dict: def json_dict_to_request_body(data: Dict[str, Any]) -> str: - # Propagate the top-level list if there is only one - # item in the object, with an en empty key. - if len(data) == 1: - [(key, value)] = data.items() - if isinstance(value, NestedJSONArray): - assert key == EMPTY_STRING - data = value - + data = unwrap_top_level_list_if_needed(data) if data: data = json.dumps(data) else: # We need to set data to an empty string to prevent requests # from assigning an empty list to `response.request.data`. data = '' - return data diff --git a/httpie/core.py b/httpie/core.py index c90452a0..d0c26dcb 100644 --- a/httpie/core.py +++ b/httpie/core.py @@ -11,7 +11,7 @@ from requests import __version__ as requests_version from . import __version__ as httpie_version from .cli.constants import OUT_REQ_BODY -from .cli.nested_json import HTTPieSyntaxError +from .cli.nested_json import NestedJSONSyntaxError from .client import collect_messages from .context import Environment, LogLevel from .downloads import Downloader @@ -78,7 +78,7 @@ def raw_main( args=args, env=env, ) - except HTTPieSyntaxError as exc: + except NestedJSONSyntaxError as exc: env.stderr.write(str(exc) + "\n") if include_traceback: raise diff --git a/httpie/utils.py b/httpie/utils.py index 0c4e0048..4735b2be 100644 --- a/httpie/utils.py +++ b/httpie/utils.py @@ -245,7 +245,7 @@ def get_site_paths(path: Path) -> Iterable[Path]: yield as_site(path) -def split(iterable: Iterable[T], key: Callable[[T], bool]) -> Tuple[List[T], List[T]]: +def split_iterable(iterable: Iterable[T], key: Callable[[T], bool]) -> Tuple[List[T], List[T]]: left, right = [], [] for item in iterable: if key(item): diff --git a/tests/test_json.py b/tests/test_json.py index 2ba603a6..e758ebe7 100644 --- a/tests/test_json.py +++ b/tests/test_json.py @@ -5,7 +5,7 @@ import responses from httpie.cli.constants import PRETTY_MAP from httpie.cli.exceptions import ParseError -from httpie.cli.nested_json import HTTPieSyntaxError +from httpie.cli.nested_json import NestedJSONSyntaxError from httpie.output.formatters.colors import ColorFormatter from httpie.utils import JsonDictPreservingDuplicateKeys @@ -157,7 +157,7 @@ def test_complex_json_arguments_with_non_json(httpbin, request_type, value): f'option:={json.dumps(value)}', ) - cm.match("Can't use complex JSON value types") + cm.match('Cannot use complex JSON value types') @pytest.mark.parametrize( @@ -508,23 +508,23 @@ def test_nested_json_syntax(input_json, expected_json, httpbin): ), ( ['foo=1', 'foo[key]:=2'], - "HTTPie Type Error: Can't perform 'key' based access on 'foo' which has a type of 'string' but this operation requires a type of 'object'.\nfoo[key]\n ^^^^^", + "HTTPie Type Error: Cannot perform 'key' based access on 'foo' which has a type of 'string' but this operation requires a type of 'object'.\nfoo[key]\n ^^^^^", ), ( ['foo=1', 'foo[0]:=2'], - "HTTPie Type Error: Can't perform 'index' based access on 'foo' which has a type of 'string' but this operation requires a type of 'array'.\nfoo[0]\n ^^^", + "HTTPie Type Error: Cannot perform 'index' based access on 'foo' which has a type of 'string' but this operation requires a type of 'array'.\nfoo[0]\n ^^^", ), ( ['foo=1', 'foo[]:=2'], - "HTTPie Type Error: Can't perform 'append' based access on 'foo' which has a type of 'string' but this operation requires a type of 'array'.\nfoo[]\n ^^", + "HTTPie Type Error: Cannot perform 'append' based access on 'foo' which has a type of 'string' but this operation requires a type of 'array'.\nfoo[]\n ^^", ), ( ['data[key]=value', 'data[key 2]=value 2', 'data[0]=value'], - "HTTPie Type Error: Can't perform 'index' based access on 'data' which has a type of 'object' but this operation requires a type of 'array'.\ndata[0]\n ^^^", + "HTTPie Type Error: Cannot perform 'index' based access on 'data' which has a type of 'object' but this operation requires a type of 'array'.\ndata[0]\n ^^^", ), ( ['data[key]=value', 'data[key 2]=value 2', 'data[]=value'], - "HTTPie Type Error: Can't perform 'append' based access on 'data' which has a type of 'object' but this operation requires a type of 'array'.\ndata[]\n ^^", + "HTTPie Type Error: Cannot perform 'append' based access on 'data' which has a type of 'object' but this operation requires a type of 'array'.\ndata[]\n ^^", ), ( [ @@ -532,7 +532,7 @@ def test_nested_json_syntax(input_json, expected_json, httpbin): 'foo[bar][baz][5][]:=4', 'foo[bar][baz][key][]:=5', ], - "HTTPie Type Error: Can't perform 'key' based access on 'foo[bar][baz]' which has a type of 'array' but this operation requires a type of 'object'.\nfoo[bar][baz][key][]\n ^^^^^", + "HTTPie Type Error: Cannot perform 'key' based access on 'foo[bar][baz]' which has a type of 'array' but this operation requires a type of 'object'.\nfoo[bar][baz][key][]\n ^^^^^", ), ( ['foo[-10]:=[1,2]'], @@ -540,32 +540,32 @@ def test_nested_json_syntax(input_json, expected_json, httpbin): ), ( ['foo[0]:=1', 'foo[]:=2', 'foo[\\2]:=3'], - "HTTPie Type Error: Can't perform 'key' based access on 'foo' which has a type of 'array' but this operation requires a type of 'object'.\nfoo[\\2]\n ^^^^", + "HTTPie Type Error: Cannot perform 'key' based access on 'foo' which has a type of 'array' but this operation requires a type of 'object'.\nfoo[\\2]\n ^^^^", ), ( ['foo[\\1]:=2', 'foo[5]:=3'], - "HTTPie Type Error: Can't perform 'index' based access on 'foo' which has a type of 'object' but this operation requires a type of 'array'.\nfoo[5]\n ^^^", + "HTTPie Type Error: Cannot perform 'index' based access on 'foo' which has a type of 'object' but this operation requires a type of 'array'.\nfoo[5]\n ^^^", ), ( ['x=y', '[]:=2'], - "HTTPie Type Error: Can't perform 'append' based access on '' which has a type of 'object' but this operation requires a type of 'array'.", + "HTTPie Type Error: Cannot perform 'append' based access on '' which has a type of 'object' but this operation requires a type of 'array'.", ), ( ['[]:=2', 'x=y'], - "HTTPie Type Error: Can't perform 'key' based access on '' which has a type of 'array' but this operation requires a type of 'object'.", + "HTTPie Type Error: Cannot perform 'key' based access on '' which has a type of 'array' but this operation requires a type of 'object'.", ), ( [':=[1,2,3]', '[]:=4'], - "HTTPie Type Error: Can't perform 'append' based access on '' which has a type of 'object' but this operation requires a type of 'array'.", + "HTTPie Type Error: Cannot perform 'append' based access on '' which has a type of 'object' but this operation requires a type of 'array'.", ), ( ['[]:=4', ':=[1,2,3]'], - "HTTPie Type Error: Can't perform 'key' based access on '' which has a type of 'array' but this operation requires a type of 'object'.", + "HTTPie Type Error: Cannot perform 'key' based access on '' which has a type of 'array' but this operation requires a type of 'object'.", ), ], ) def test_nested_json_errors(input_json, expected_error, httpbin): - with pytest.raises(HTTPieSyntaxError) as exc: + with pytest.raises(NestedJSONSyntaxError) as exc: http(httpbin + '/post', *input_json) exc_lines = str(exc.value).splitlines()