Clean up and refactor nested JSON parsing & interpreting (#1440)

extras/scripts/generate_man_pages.py

@@ -9,7 +9,7 @@ from httpie.cli.options import ParserSpec
 from httpie.manager.cli import options as manager_options
 from httpie.output.ui.rich_help import OptionsHighlighter, to_usage
 from httpie.output.ui.rich_utils import render_as_string
-from httpie.utils import split
+from httpie.utils import split_iterable
 
 
 # Escape certain characters so they are rendered properly on

httpie/cli/dicts.py

@@ -92,7 +92,3 @@ class MultipartRequestDataDict(MultiValueOrderedDict):
 
 class RequestFilesDict(RequestDataDict):
     pass
-
-
-class NestedJSONArray(list):
-    """Denotes a top-level JSON array."""

httpie/cli/nested_json.py

@@ -1,404 +0,0 @@
-from enum import Enum, auto
-from typing import (
-    Any,
-    Iterator,
-    NamedTuple,
-    Optional,
-    List,
-    NoReturn,
-    Type,
-    Union,
-)
-from .dicts import NestedJSONArray
-
-
-EMPTY_STRING = ''
-HIGHLIGHTER = '^'
-OPEN_BRACKET = '['
-CLOSE_BRACKET = ']'
-BACKSLASH = '\\'
-
-
-class HTTPieSyntaxError(ValueError):
-    def __init__(
-        self,
-        source: str,
-        token: Optional['Token'],
-        message: str,
-        message_kind: str = 'Syntax',
-    ) -> None:
-        self.source = source
-        self.token = token
-        self.message = message
-        self.message_kind = message_kind
-
-    def __str__(self):
-        lines = [f'HTTPie {self.message_kind} Error: {self.message}']
-        if self.token is not None:
-            lines.append(self.source)
-            lines.append(
-                ' ' * self.token.start
-                + HIGHLIGHTER * (self.token.end - self.token.start)
-            )
-        return '\n'.join(lines)
-
-
-class TokenKind(Enum):
-    TEXT = auto()
-    NUMBER = auto()
-    LEFT_BRACKET = auto()
-    RIGHT_BRACKET = auto()
-
-    def to_name(self) -> str:
-        for key, value in OPERATORS.items():
-            if value is self:
-                return repr(key)
-        else:
-            return 'a ' + self.name.lower()
-
-
-OPERATORS = {
-    OPEN_BRACKET: TokenKind.LEFT_BRACKET,
-    CLOSE_BRACKET: TokenKind.RIGHT_BRACKET,
-}
-SPECIAL_CHARS = OPERATORS.keys() | {BACKSLASH}
-LITERAL_TOKENS = [
-    TokenKind.TEXT,
-    TokenKind.NUMBER,
-]
-
-
-class Token(NamedTuple):
-    kind: TokenKind
-    value: Union[str, int]
-    start: int
-    end: int
-
-
-def assert_cant_happen() -> NoReturn:
-    raise ValueError('Unexpected value')
-
-
-def check_escaped_int(value: str) -> str:
-    if not value.startswith(BACKSLASH):
-        raise ValueError('Not an escaped int')
-
-    try:
-        int(value[1:])
-    except ValueError as exc:
-        raise ValueError('Not an escaped int') from exc
-    else:
-        return value[1:]
-
-
-def tokenize(source: str) -> Iterator[Token]:
-    cursor = 0
-    backslashes = 0
-    buffer = []
-
-    def send_buffer() -> Iterator[Token]:
-        nonlocal backslashes
-        if not buffer:
-            return None
-
-        value = ''.join(buffer)
-        kind = TokenKind.TEXT
-        if not backslashes:
-            for variation, kind in [
-                (int, TokenKind.NUMBER),
-                (check_escaped_int, TokenKind.TEXT),
-            ]:
-                try:
-                    value = variation(value)
-                except ValueError:
-                    continue
-                else:
-                    break
-
-        yield Token(
-            kind, value, start=cursor - (len(buffer) + backslashes), end=cursor
-        )
-        buffer.clear()
-        backslashes = 0
-
-    def can_advance() -> bool:
-        return cursor < len(source)
-
-    while can_advance():
-        index = source[cursor]
-        if index in OPERATORS:
-            yield from send_buffer()
-            yield Token(OPERATORS[index], index, cursor, cursor + 1)
-        elif index == BACKSLASH and can_advance():
-            if source[cursor + 1] in SPECIAL_CHARS:
-                backslashes += 1
-            else:
-                buffer.append(index)
-
-            buffer.append(source[cursor + 1])
-            cursor += 1
-        else:
-            buffer.append(index)
-
-        cursor += 1
-
-    yield from send_buffer()
-
-
-class PathAction(Enum):
-    KEY = auto()
-    INDEX = auto()
-    APPEND = auto()
-
-    # Pseudo action, used by the interpreter
-    SET = auto()
-
-    def to_string(self) -> str:
-        return self.name.lower()
-
-
-class Path:
-    def __init__(
-        self,
-        kind: PathAction,
-        accessor: Optional[Union[str, int]] = None,
-        tokens: Optional[List[Token]] = None,
-        is_root: bool = False,
-    ):
-        self.kind = kind
-        self.accessor = accessor
-        self.tokens = tokens or []
-        self.is_root = is_root
-
-    def reconstruct(self) -> str:
-        if self.kind is PathAction.KEY:
-            if self.is_root:
-                return str(self.accessor)
-            return OPEN_BRACKET + self.accessor + CLOSE_BRACKET
-        elif self.kind is PathAction.INDEX:
-            return OPEN_BRACKET + str(self.accessor) + CLOSE_BRACKET
-        elif self.kind is PathAction.APPEND:
-            return OPEN_BRACKET + CLOSE_BRACKET
-        else:
-            assert_cant_happen()
-
-
-def parse(source: str) -> Iterator[Path]:
-    """
-    start: root_path path*
-    root_path: (literal | index_path | append_path)
-    literal: TEXT | NUMBER
-
-    path:
-        key_path
-        | index_path
-        | append_path
-    key_path: LEFT_BRACKET TEXT RIGHT_BRACKET
-    index_path: LEFT_BRACKET NUMBER RIGHT_BRACKET
-    append_path: LEFT_BRACKET RIGHT_BRACKET
-    """
-
-    tokens = list(tokenize(source))
-    cursor = 0
-
-    def can_advance():
-        return cursor < len(tokens)
-
-    def expect(*kinds):
-        nonlocal cursor
-
-        assert len(kinds) > 0
-        if can_advance():
-            token = tokens[cursor]
-            cursor += 1
-            if token.kind in kinds:
-                return token
-        elif tokens:
-            token = tokens[-1]._replace(
-                start=tokens[-1].end + 0, end=tokens[-1].end + 1
-            )
-        else:
-            token = None
-
-        if len(kinds) == 1:
-            suffix = kinds[0].to_name()
-        else:
-            suffix = ', '.join(kind.to_name() for kind in kinds[:-1])
-            suffix += ' or ' + kinds[-1].to_name()
-
-        message = f'Expecting {suffix}'
-        raise HTTPieSyntaxError(source, token, message)
-
-    def parse_root():
-        tokens = []
-        if not can_advance():
-            return Path(
-                PathAction.KEY,
-                EMPTY_STRING,
-                is_root=True
-            )
-
-        # (literal | index_path | append_path)?
-        token = expect(*LITERAL_TOKENS, TokenKind.LEFT_BRACKET)
-        tokens.append(token)
-
-        if token.kind in LITERAL_TOKENS:
-            action = PathAction.KEY
-            value = str(token.value)
-        elif token.kind is TokenKind.LEFT_BRACKET:
-            token = expect(TokenKind.NUMBER, TokenKind.RIGHT_BRACKET)
-            tokens.append(token)
-            if token.kind is TokenKind.NUMBER:
-                action = PathAction.INDEX
-                value = token.value
-                tokens.append(expect(TokenKind.RIGHT_BRACKET))
-            elif token.kind is TokenKind.RIGHT_BRACKET:
-                action = PathAction.APPEND
-                value = None
-            else:
-                assert_cant_happen()
-        else:
-            assert_cant_happen()
-
-        return Path(
-            action,
-            value,
-            tokens=tokens,
-            is_root=True
-        )
-
-    yield parse_root()
-
-    # path*
-    while can_advance():
-        path_tokens = []
-        path_tokens.append(expect(TokenKind.LEFT_BRACKET))
-
-        token = expect(
-            TokenKind.TEXT, TokenKind.NUMBER, TokenKind.RIGHT_BRACKET
-        )
-        path_tokens.append(token)
-        if token.kind is TokenKind.RIGHT_BRACKET:
-            path = Path(PathAction.APPEND, tokens=path_tokens)
-        elif token.kind is TokenKind.TEXT:
-            path = Path(PathAction.KEY, token.value, tokens=path_tokens)
-            path_tokens.append(expect(TokenKind.RIGHT_BRACKET))
-        elif token.kind is TokenKind.NUMBER:
-            path = Path(PathAction.INDEX, token.value, tokens=path_tokens)
-            path_tokens.append(expect(TokenKind.RIGHT_BRACKET))
-        else:
-            assert_cant_happen()
-        yield path
-
-
-JSON_TYPE_MAPPING = {
-    dict: 'object',
-    list: 'array',
-    int: 'number',
-    float: 'number',
-    str: 'string',
-}
-
-
-def interpret(context: Any, key: str, value: Any) -> Any:
-    cursor = context
-
-    paths = list(parse(key))
-    paths.append(Path(PathAction.SET, value))
-
-    def type_check(index: int, path: Path, expected_type: Type[Any]) -> None:
-        if not isinstance(cursor, expected_type):
-            if path.tokens:
-                pseudo_token = Token(
-                    None, None, path.tokens[0].start, path.tokens[-1].end
-                )
-            else:
-                pseudo_token = None
-
-            cursor_type = JSON_TYPE_MAPPING.get(
-                type(cursor), type(cursor).__name__
-            )
-            required_type = JSON_TYPE_MAPPING[expected_type]
-
-            message = f"Can't perform {path.kind.to_string()!r} based access on "
-            message += repr(
-                ''.join(path.reconstruct() for path in paths[:index])
-            )
-            message += (
-                f' which has a type of {cursor_type!r} but this operation'
-            )
-            message += f' requires a type of {required_type!r}.'
-            raise HTTPieSyntaxError(
-                key, pseudo_token, message, message_kind='Type'
-            )
-
-    def object_for(kind: str) -> Any:
-        if kind is PathAction.KEY:
-            return {}
-        elif kind in {PathAction.INDEX, PathAction.APPEND}:
-            return []
-        else:
-            assert_cant_happen()
-
-    for index, (path, next_path) in enumerate(zip(paths, paths[1:])):
-        # If there is no context yet, set it.
-        if cursor is None:
-            context = cursor = object_for(path.kind)
-
-        if path.kind is PathAction.KEY:
-            type_check(index, path, dict)
-            if next_path.kind is PathAction.SET:
-                cursor[path.accessor] = next_path.accessor
-                break
-
-            cursor = cursor.setdefault(
-                path.accessor, object_for(next_path.kind)
-            )
-        elif path.kind is PathAction.INDEX:
-            type_check(index, path, list)
-            if path.accessor < 0:
-                raise HTTPieSyntaxError(
-                    key,
-                    path.tokens[1],
-                    'Negative indexes are not supported.',
-                    message_kind='Value',
-                )
-            cursor.extend([None] * (path.accessor - len(cursor) + 1))
-            if next_path.kind is PathAction.SET:
-                cursor[path.accessor] = next_path.accessor
-                break
-
-            if cursor[path.accessor] is None:
-                cursor[path.accessor] = object_for(next_path.kind)
-
-            cursor = cursor[path.accessor]
-        elif path.kind is PathAction.APPEND:
-            type_check(index, path, list)
-            if next_path.kind is PathAction.SET:
-                cursor.append(next_path.accessor)
-                break
-
-            cursor.append(object_for(next_path.kind))
-            cursor = cursor[-1]
-        else:
-            assert_cant_happen()
-
-    return context
-
-
-def wrap_with_dict(context):
-    if context is None:
-        return {}
-    elif isinstance(context, list):
-        return {EMPTY_STRING: NestedJSONArray(context)}
-    else:
-        assert isinstance(context, dict)
-        return context
-
-
-def interpret_nested_json(pairs):
-    context = None
-    for key, value in pairs:
-        context = interpret(context, key, value)
-
-    return wrap_with_dict(context)

httpie/cli/nested_json/__init__.py

@@ -0,0 +1,20 @@
+"""
+A library for parsing the HTTPie nested JSON key syntax and constructing the resulting objects.
+
+<https://httpie.io/docs/cli/nested-json>
+
+It has no dependencies.
+
+"""
+from .interpret import interpret_nested_json, unwrap_top_level_list_if_needed
+from .errors import NestedJSONSyntaxError
+from .tokens import EMPTY_STRING, NestedJSONArray
+
+
+__all__ = [
+    'interpret_nested_json',
+    'unwrap_top_level_list_if_needed',
+    'EMPTY_STRING',
+    'NestedJSONArray',
+    'NestedJSONSyntaxError'
+]

httpie/cli/nested_json/errors.py

@@ -0,0 +1,27 @@
+from typing import Optional
+
+from .tokens import Token, HIGHLIGHTER
+
+
+class NestedJSONSyntaxError(ValueError):
+    def __init__(
+        self,
+        source: str,
+        token: Optional[Token],
+        message: str,
+        message_kind: str = 'Syntax',
+    ) -> None:
+        self.source = source
+        self.token = token
+        self.message = message
+        self.message_kind = message_kind
+
+    def __str__(self):
+        lines = [f'HTTPie {self.message_kind} Error: {self.message}']
+        if self.token is not None:
+            lines.append(self.source)
+            lines.append(
+                ' ' * self.token.start
+                + HIGHLIGHTER * (self.token.end - self.token.start)
+            )
+        return '\n'.join(lines)

httpie/cli/nested_json/interpret.py

@@ -0,0 +1,129 @@
+from typing import Type, Union, Any, Iterable, Tuple
+
+from .parse import parse, assert_cant_happen
+from .errors import NestedJSONSyntaxError
+from .tokens import EMPTY_STRING, TokenKind, Token, PathAction, Path, NestedJSONArray
+
+
+__all__ = [
+    'interpret_nested_json',
+    'unwrap_top_level_list_if_needed',
+]
+
+JSONType = Type[Union[dict, list, int, float, str]]
+JSON_TYPE_MAPPING = {
+    dict: 'object',
+    list: 'array',
+    int: 'number',
+    float: 'number',
+    str: 'string',
+}
+
+
+def interpret_nested_json(pairs: Iterable[Tuple[str, str]]) -> dict:
+    context = None
+    for key, value in pairs:
+        context = interpret(context, key, value)
+    return wrap_with_dict(context)
+
+
+def interpret(context: Any, key: str, value: Any) -> Any:
+    cursor = context
+    paths = list(parse(key))
+    paths.append(Path(PathAction.SET, value))
+
+    # noinspection PyShadowingNames
+    def type_check(index: int, path: Path, expected_type: JSONType):
+        if not isinstance(cursor, expected_type):
+            if path.tokens:
+                pseudo_token = Token(
+                    kind=TokenKind.PSEUDO,
+                    value='',
+                    start=path.tokens[0].start,
+                    end=path.tokens[-1].end,
+                )
+            else:
+                pseudo_token = None
+            cursor_type = JSON_TYPE_MAPPING.get(type(cursor), type(cursor).__name__)
+            required_type = JSON_TYPE_MAPPING[expected_type]
+            message = f'Cannot perform {path.kind.to_string()!r} based access on '
+            message += repr(''.join(path.reconstruct() for path in paths[:index]))
+            message += f' which has a type of {cursor_type!r} but this operation'
+            message += f' requires a type of {required_type!r}.'
+            raise NestedJSONSyntaxError(
+                source=key,
+                token=pseudo_token,
+                message=message,
+                message_kind='Type',
+            )
+
+    def object_for(kind: PathAction) -> Any:
+        if kind is PathAction.KEY:
+            return {}
+        elif kind in {PathAction.INDEX, PathAction.APPEND}:
+            return []
+        else:
+            assert_cant_happen()
+
+    for index, (path, next_path) in enumerate(zip(paths, paths[1:])):
+        # If there is no context yet, set it.
+        if cursor is None:
+            context = cursor = object_for(path.kind)
+        if path.kind is PathAction.KEY:
+            type_check(index, path, dict)
+            if next_path.kind is PathAction.SET:
+                cursor[path.accessor] = next_path.accessor
+                break
+            cursor = cursor.setdefault(path.accessor, object_for(next_path.kind))
+        elif path.kind is PathAction.INDEX:
+            type_check(index, path, list)
+            if path.accessor < 0:
+                raise NestedJSONSyntaxError(
+                    source=key,
+                    token=path.tokens[1],
+                    message='Negative indexes are not supported.',
+                    message_kind='Value',
+                )
+            cursor.extend([None] * (path.accessor - len(cursor) + 1))
+            if next_path.kind is PathAction.SET:
+                cursor[path.accessor] = next_path.accessor
+                break
+            if cursor[path.accessor] is None:
+                cursor[path.accessor] = object_for(next_path.kind)
+            cursor = cursor[path.accessor]
+        elif path.kind is PathAction.APPEND:
+            type_check(index, path, list)
+            if next_path.kind is PathAction.SET:
+                cursor.append(next_path.accessor)
+                break
+            cursor.append(object_for(next_path.kind))
+            cursor = cursor[-1]
+        else:
+            assert_cant_happen()
+
+    return context
+
+
+def wrap_with_dict(context):
+    if context is None:
+        return {}
+    elif isinstance(context, list):
+        return {
+            EMPTY_STRING: NestedJSONArray(context),
+        }
+    else:
+        assert isinstance(context, dict)
+        return context
+
+
+def unwrap_top_level_list_if_needed(data: dict):
+    """
+    Propagate the top-level list, if that’s what we got.
+
+    """
+    if len(data) == 1:
+        key, value = list(data.items())[0]
+        if isinstance(value, NestedJSONArray):
+            assert key == EMPTY_STRING
+            return value
+    return data

httpie/cli/nested_json/parse.py

@@ -0,0 +1,193 @@
+from typing import Iterator
+
+from .errors import NestedJSONSyntaxError
+from .tokens import (
+    EMPTY_STRING,
+    BACKSLASH,
+    TokenKind,
+    OPERATORS,
+    SPECIAL_CHARS,
+    LITERAL_TOKENS,
+    Token,
+    PathAction,
+    Path,
+)
+
+
+__all__ = [
+    'parse',
+    'assert_cant_happen',
+]
+
+
+def parse(source: str) -> Iterator[Path]:
+    """
+    start: root_path path*
+    root_path: (literal | index_path | append_path)
+    literal: TEXT | NUMBER
+
+    path:
+        key_path
+        | index_path
+        | append_path
+    key_path: LEFT_BRACKET TEXT RIGHT_BRACKET
+    index_path: LEFT_BRACKET NUMBER RIGHT_BRACKET
+    append_path: LEFT_BRACKET RIGHT_BRACKET
+
+    """
+
+    tokens = list(tokenize(source))
+    cursor = 0
+
+    def can_advance():
+        return cursor < len(tokens)
+
+    # noinspection PyShadowingNames
+    def expect(*kinds):
+        nonlocal cursor
+        assert kinds
+        if can_advance():
+            token = tokens[cursor]
+            cursor += 1
+            if token.kind in kinds:
+                return token
+        elif tokens:
+            token = tokens[-1]._replace(
+                start=tokens[-1].end + 0,
+                end=tokens[-1].end + 1,
+            )
+        else:
+            token = None
+        if len(kinds) == 1:
+            suffix = kinds[0].to_name()
+        else:
+            suffix = ', '.join(kind.to_name() for kind in kinds[:-1])
+            suffix += ' or ' + kinds[-1].to_name()
+        message = f'Expecting {suffix}'
+        raise NestedJSONSyntaxError(source, token, message)
+
+    # noinspection PyShadowingNames
+    def parse_root():
+        tokens = []
+        if not can_advance():
+            return Path(
+                kind=PathAction.KEY,
+                accessor=EMPTY_STRING,
+                is_root=True
+            )
+        # (literal | index_path | append_path)?
+        token = expect(*LITERAL_TOKENS, TokenKind.LEFT_BRACKET)
+        tokens.append(token)
+        if token.kind in LITERAL_TOKENS:
+            action = PathAction.KEY
+            value = str(token.value)
+        elif token.kind is TokenKind.LEFT_BRACKET:
+            token = expect(TokenKind.NUMBER, TokenKind.RIGHT_BRACKET)
+            tokens.append(token)
+            if token.kind is TokenKind.NUMBER:
+                action = PathAction.INDEX
+                value = token.value
+                tokens.append(expect(TokenKind.RIGHT_BRACKET))
+            elif token.kind is TokenKind.RIGHT_BRACKET:
+                action = PathAction.APPEND
+                value = None
+            else:
+                assert_cant_happen()
+        else:
+            assert_cant_happen()
+        # noinspection PyUnboundLocalVariable
+        return Path(
+            kind=action,
+            accessor=value,
+            tokens=tokens,
+            is_root=True
+        )
+
+    yield parse_root()
+
+    # path*
+    while can_advance():
+        path_tokens = [expect(TokenKind.LEFT_BRACKET)]
+        token = expect(TokenKind.TEXT, TokenKind.NUMBER, TokenKind.RIGHT_BRACKET)
+        path_tokens.append(token)
+        if token.kind is TokenKind.RIGHT_BRACKET:
+            path = Path(PathAction.APPEND, tokens=path_tokens)
+        elif token.kind is TokenKind.TEXT:
+            path = Path(PathAction.KEY, token.value, tokens=path_tokens)
+            path_tokens.append(expect(TokenKind.RIGHT_BRACKET))
+        elif token.kind is TokenKind.NUMBER:
+            path = Path(PathAction.INDEX, token.value, tokens=path_tokens)
+            path_tokens.append(expect(TokenKind.RIGHT_BRACKET))
+        else:
+            assert_cant_happen()
+        # noinspection PyUnboundLocalVariable
+        yield path
+
+
+def tokenize(source: str) -> Iterator[Token]:
+    cursor = 0
+    backslashes = 0
+    buffer = []
+
+    def send_buffer() -> Iterator[Token]:
+        nonlocal backslashes
+        if not buffer:
+            return None
+
+        value = ''.join(buffer)
+        kind = TokenKind.TEXT
+        if not backslashes:
+            for variation, kind in [
+                (int, TokenKind.NUMBER),
+                (check_escaped_int, TokenKind.TEXT),
+            ]:
+                try:
+                    value = variation(value)
+                except ValueError:
+                    continue
+                else:
+                    break
+        yield Token(
+            kind=kind,
+            value=value,
+            start=cursor - (len(buffer) + backslashes),
+            end=cursor,
+        )
+        buffer.clear()
+        backslashes = 0
+
+    def can_advance() -> bool:
+        return cursor < len(source)
+
+    while can_advance():
+        index = source[cursor]
+        if index in OPERATORS:
+            yield from send_buffer()
+            yield Token(OPERATORS[index], index, cursor, cursor + 1)
+        elif index == BACKSLASH and can_advance():
+            if source[cursor + 1] in SPECIAL_CHARS:
+                backslashes += 1
+            else:
+                buffer.append(index)
+            buffer.append(source[cursor + 1])
+            cursor += 1
+        else:
+            buffer.append(index)
+        cursor += 1
+
+    yield from send_buffer()
+
+
+def check_escaped_int(value: str) -> str:
+    if not value.startswith(BACKSLASH):
+        raise ValueError('Not an escaped int')
+    try:
+        int(value[1:])
+    except ValueError as exc:
+        raise ValueError('Not an escaped int') from exc
+    else:
+        return value[1:]
+
+
+def assert_cant_happen():
+    raise ValueError('Unexpected value')

httpie/cli/nested_json/tokens.py

@@ -0,0 +1,80 @@
+from enum import Enum, auto
+from typing import NamedTuple, Union, Optional, List
+
+EMPTY_STRING = ''
+HIGHLIGHTER = '^'
+OPEN_BRACKET = '['
+CLOSE_BRACKET = ']'
+BACKSLASH = '\\'
+
+
+class TokenKind(Enum):
+    TEXT = auto()
+    NUMBER = auto()
+    LEFT_BRACKET = auto()
+    RIGHT_BRACKET = auto()
+    PSEUDO = auto()  # Not a real token, use when representing location only.
+
+    def to_name(self) -> str:
+        for key, value in OPERATORS.items():
+            if value is self:
+                return repr(key)
+        else:
+            return 'a ' + self.name.lower()
+
+
+OPERATORS = {
+    OPEN_BRACKET: TokenKind.LEFT_BRACKET,
+    CLOSE_BRACKET: TokenKind.RIGHT_BRACKET,
+}
+SPECIAL_CHARS = OPERATORS.keys() | {BACKSLASH}
+LITERAL_TOKENS = [
+    TokenKind.TEXT,
+    TokenKind.NUMBER,
+]
+
+
+class Token(NamedTuple):
+    kind: TokenKind
+    value: Union[str, int]
+    start: int
+    end: int
+
+
+class PathAction(Enum):
+    KEY = auto()
+    INDEX = auto()
+    APPEND = auto()
+    # Pseudo action, used by the interpreter
+    SET = auto()
+
+    def to_string(self) -> str:
+        return self.name.lower()
+
+
+class Path:
+    def __init__(
+        self,
+        kind: PathAction,
+        accessor: Optional[Union[str, int]] = None,
+        tokens: Optional[List[Token]] = None,
+        is_root: bool = False,
+    ):
+        self.kind = kind
+        self.accessor = accessor
+        self.tokens = tokens or []
+        self.is_root = is_root
+
+    def reconstruct(self) -> str:
+        if self.kind is PathAction.KEY:
+            if self.is_root:
+                return str(self.accessor)
+            return OPEN_BRACKET + self.accessor + CLOSE_BRACKET
+        elif self.kind is PathAction.INDEX:
+            return OPEN_BRACKET + str(self.accessor) + CLOSE_BRACKET
+        elif self.kind is PathAction.APPEND:
+            return OPEN_BRACKET + CLOSE_BRACKET
+
+
+class NestedJSONArray(list):
+    """Denotes a top-level JSON array."""

httpie/cli/requestitems.py

@@ -18,7 +18,7 @@ from .dicts import (
 )
 from .exceptions import ParseError
 from .nested_json import interpret_nested_json
-from ..utils import get_content_type, load_json_preserve_order_and_dupe_keys, split
+from ..utils import get_content_type, load_json_preserve_order_and_dupe_keys, split_iterable
 
 
 class RequestItems:
@@ -78,25 +78,28 @@ class RequestItems:
                 instance.data,
             ),
             SEPARATOR_DATA_RAW_JSON: (
-                json_only(instance, process_data_raw_json_embed_arg),
+                convert_json_value_to_form_if_needed(
+                    in_json_mode=instance.is_json,
+                    processor=process_data_raw_json_embed_arg
+                ),
                 instance.data,
             ),
             SEPARATOR_DATA_EMBED_RAW_JSON_FILE: (
-                json_only(instance, process_data_embed_raw_json_file_arg),
+                convert_json_value_to_form_if_needed(
+                    in_json_mode=instance.is_json,
+                    processor=process_data_embed_raw_json_file_arg,
+                ),
                 instance.data,
             ),
         }
 
         if instance.is_json:
-            json_item_args, request_item_args = split(
+            json_item_args, request_item_args = split_iterable(
-                request_item_args,
+                iterable=request_item_args,
-                lambda arg: arg.sep in SEPARATOR_GROUP_NESTED_JSON_ITEMS
+                key=lambda arg: arg.sep in SEPARATOR_GROUP_NESTED_JSON_ITEMS
             )
             if json_item_args:
-                pairs = [
+                pairs = [(arg.key, rules[arg.sep][0](arg)) for arg in json_item_args]
-                    (arg.key, rules[arg.sep][0](arg))
-                    for arg in json_item_args
-                ]
                 processor_func, target_dict = rules[SEPARATOR_GROUP_NESTED_JSON_ITEMS]
                 value = processor_func(pairs)
                 target_dict.update(value)
@@ -159,6 +162,30 @@ def process_file_upload_arg(arg: KeyValueArg) -> Tuple[str, IO, str]:
     )
 
 
+def convert_json_value_to_form_if_needed(in_json_mode: bool, processor: Callable[[KeyValueArg], JSONType]) -> Callable[[], str]:
+    """
+    We allow primitive values to be passed to forms via JSON key/value syntax.
+
+    But complex values lead to an error because there’s no clear way to serialize them.
+
+    """
+    if in_json_mode:
+        return processor
+
+    @functools.wraps(processor)
+    def wrapper(*args, **kwargs) -> str:
+        try:
+            output = processor(*args, **kwargs)
+        except ParseError:
+            output = None
+        if isinstance(output, (str, int, float)):
+            return str(output)
+        else:
+            raise ParseError('Cannot use complex JSON value types with --form/--multipart.')
+
+    return wrapper
+
+
 def process_data_item_arg(arg: KeyValueArg) -> str:
     return arg.value
 
@@ -167,29 +194,6 @@ def process_data_embed_file_contents_arg(arg: KeyValueArg) -> str:
     return load_text_file(arg)
 
 
-def json_only(items: RequestItems, func: Callable[[KeyValueArg], JSONType]) -> str:
-    if items.is_json:
-        return func
-
-    @functools.wraps(func)
-    def wrapper(*args, **kwargs) -> str:
-        try:
-            ret = func(*args, **kwargs)
-        except ParseError:
-            ret = None
-
-        # If it is a basic type, then allow it
-        if isinstance(ret, (str, int, float)):
-            return str(ret)
-        else:
-            raise ParseError(
-                'Can\'t use complex JSON value types with '
-                '--form/--multipart.'
-            )
-
-    return wrapper
-
-
 def process_data_embed_raw_json_file_arg(arg: KeyValueArg) -> JSONType:
     contents = load_text_file(arg)
     value = load_json(arg, contents)

httpie/client.py

@@ -10,12 +10,13 @@ from urllib.parse import urlparse, urlunparse
 import requests
 # noinspection PyPackageRequirements
 import urllib3
+
 from . import __version__
 from .adapters import HTTPieHTTPAdapter
-from .context import Environment
 from .cli.constants import HTTP_OPTIONS
-from .cli.nested_json import EMPTY_STRING
+from .cli.dicts import HTTPHeadersDict
-from .cli.dicts import HTTPHeadersDict, NestedJSONArray
+from .cli.nested_json import unwrap_top_level_list_if_needed
+from .context import Environment
 from .encoding import UTF8
 from .models import RequestsMessage
 from .plugins.registry import plugin_manager
@@ -306,21 +307,13 @@ def make_send_kwargs_mergeable_from_env(args: argparse.Namespace) -> dict:
 
 
 def json_dict_to_request_body(data: Dict[str, Any]) -> str:
-    # Propagate the top-level list if there is only one
+    data = unwrap_top_level_list_if_needed(data)
-    # item in the object, with an en empty key.
-    if len(data) == 1:
-        [(key, value)] = data.items()
-        if isinstance(value, NestedJSONArray):
-            assert key == EMPTY_STRING
-            data = value
-
     if data:
         data = json.dumps(data)
     else:
         # We need to set data to an empty string to prevent requests
         # from assigning an empty list to `response.request.data`.
         data = ''
-
     return data
 
 

httpie/core.py

@@ -11,7 +11,7 @@ from requests import __version__ as requests_version
 
 from . import __version__ as httpie_version
 from .cli.constants import OUT_REQ_BODY
-from .cli.nested_json import HTTPieSyntaxError
+from .cli.nested_json import NestedJSONSyntaxError
 from .client import collect_messages
 from .context import Environment, LogLevel
 from .downloads import Downloader
@@ -78,7 +78,7 @@ def raw_main(
             args=args,
             env=env,
         )
-    except HTTPieSyntaxError as exc:
+    except NestedJSONSyntaxError as exc:
         env.stderr.write(str(exc) + "\n")
         if include_traceback:
             raise

httpie/utils.py

@@ -245,7 +245,7 @@ def get_site_paths(path: Path) -> Iterable[Path]:
         yield as_site(path)
 
 
-def split(iterable: Iterable[T], key: Callable[[T], bool]) -> Tuple[List[T], List[T]]:
+def split_iterable(iterable: Iterable[T], key: Callable[[T], bool]) -> Tuple[List[T], List[T]]:
     left, right = [], []
     for item in iterable:
         if key(item):

tests/test_json.py

@@ -5,7 +5,7 @@ import responses
 
 from httpie.cli.constants import PRETTY_MAP
 from httpie.cli.exceptions import ParseError
-from httpie.cli.nested_json import HTTPieSyntaxError
+from httpie.cli.nested_json import NestedJSONSyntaxError
 from httpie.output.formatters.colors import ColorFormatter
 from httpie.utils import JsonDictPreservingDuplicateKeys
 
@@ -157,7 +157,7 @@ def test_complex_json_arguments_with_non_json(httpbin, request_type, value):
             f'option:={json.dumps(value)}',
         )
 
-    cm.match("Can't use complex JSON value types")
+    cm.match('Cannot use complex JSON value types')
 
 
 @pytest.mark.parametrize(
@@ -508,23 +508,23 @@ def test_nested_json_syntax(input_json, expected_json, httpbin):
         ),
         (
             ['foo=1', 'foo[key]:=2'],
-            "HTTPie Type Error: Can't perform 'key' based access on 'foo' which has a type of 'string' but this operation requires a type of 'object'.\nfoo[key]\n   ^^^^^",
+            "HTTPie Type Error: Cannot perform 'key' based access on 'foo' which has a type of 'string' but this operation requires a type of 'object'.\nfoo[key]\n   ^^^^^",
         ),
         (
             ['foo=1', 'foo[0]:=2'],
-            "HTTPie Type Error: Can't perform 'index' based access on 'foo' which has a type of 'string' but this operation requires a type of 'array'.\nfoo[0]\n   ^^^",
+            "HTTPie Type Error: Cannot perform 'index' based access on 'foo' which has a type of 'string' but this operation requires a type of 'array'.\nfoo[0]\n   ^^^",
         ),
         (
             ['foo=1', 'foo[]:=2'],
-            "HTTPie Type Error: Can't perform 'append' based access on 'foo' which has a type of 'string' but this operation requires a type of 'array'.\nfoo[]\n   ^^",
+            "HTTPie Type Error: Cannot perform 'append' based access on 'foo' which has a type of 'string' but this operation requires a type of 'array'.\nfoo[]\n   ^^",
         ),
         (
             ['data[key]=value', 'data[key 2]=value 2', 'data[0]=value'],
-            "HTTPie Type Error: Can't perform 'index' based access on 'data' which has a type of 'object' but this operation requires a type of 'array'.\ndata[0]\n    ^^^",
+            "HTTPie Type Error: Cannot perform 'index' based access on 'data' which has a type of 'object' but this operation requires a type of 'array'.\ndata[0]\n    ^^^",
         ),
         (
             ['data[key]=value', 'data[key 2]=value 2', 'data[]=value'],
-            "HTTPie Type Error: Can't perform 'append' based access on 'data' which has a type of 'object' but this operation requires a type of 'array'.\ndata[]\n    ^^",
+            "HTTPie Type Error: Cannot perform 'append' based access on 'data' which has a type of 'object' but this operation requires a type of 'array'.\ndata[]\n    ^^",
         ),
         (
             [
@@ -532,7 +532,7 @@ def test_nested_json_syntax(input_json, expected_json, httpbin):
                 'foo[bar][baz][5][]:=4',
                 'foo[bar][baz][key][]:=5',
             ],
-            "HTTPie Type Error: Can't perform 'key' based access on 'foo[bar][baz]' which has a type of 'array' but this operation requires a type of 'object'.\nfoo[bar][baz][key][]\n             ^^^^^",
+            "HTTPie Type Error: Cannot perform 'key' based access on 'foo[bar][baz]' which has a type of 'array' but this operation requires a type of 'object'.\nfoo[bar][baz][key][]\n             ^^^^^",
         ),
         (
             ['foo[-10]:=[1,2]'],
@@ -540,32 +540,32 @@ def test_nested_json_syntax(input_json, expected_json, httpbin):
         ),
         (
             ['foo[0]:=1', 'foo[]:=2', 'foo[\\2]:=3'],
-            "HTTPie Type Error: Can't perform 'key' based access on 'foo' which has a type of 'array' but this operation requires a type of 'object'.\nfoo[\\2]\n   ^^^^",
+            "HTTPie Type Error: Cannot perform 'key' based access on 'foo' which has a type of 'array' but this operation requires a type of 'object'.\nfoo[\\2]\n   ^^^^",
         ),
         (
             ['foo[\\1]:=2', 'foo[5]:=3'],
-            "HTTPie Type Error: Can't perform 'index' based access on 'foo' which has a type of 'object' but this operation requires a type of 'array'.\nfoo[5]\n   ^^^",
+            "HTTPie Type Error: Cannot perform 'index' based access on 'foo' which has a type of 'object' but this operation requires a type of 'array'.\nfoo[5]\n   ^^^",
         ),
         (
             ['x=y', '[]:=2'],
-            "HTTPie Type Error: Can't perform 'append' based access on '' which has a type of 'object' but this operation requires a type of 'array'.",
+            "HTTPie Type Error: Cannot perform 'append' based access on '' which has a type of 'object' but this operation requires a type of 'array'.",
         ),
         (
             ['[]:=2', 'x=y'],
-            "HTTPie Type Error: Can't perform 'key' based access on '' which has a type of 'array' but this operation requires a type of 'object'.",
+            "HTTPie Type Error: Cannot perform 'key' based access on '' which has a type of 'array' but this operation requires a type of 'object'.",
         ),
         (
             [':=[1,2,3]', '[]:=4'],
-            "HTTPie Type Error: Can't perform 'append' based access on '' which has a type of 'object' but this operation requires a type of 'array'.",
+            "HTTPie Type Error: Cannot perform 'append' based access on '' which has a type of 'object' but this operation requires a type of 'array'.",
         ),
         (
             ['[]:=4', ':=[1,2,3]'],
-            "HTTPie Type Error: Can't perform 'key' based access on '' which has a type of 'array' but this operation requires a type of 'object'.",
+            "HTTPie Type Error: Cannot perform 'key' based access on '' which has a type of 'array' but this operation requires a type of 'object'.",
         ),
     ],
 )
 def test_nested_json_errors(input_json, expected_error, httpbin):
-    with pytest.raises(HTTPieSyntaxError) as exc:
+    with pytest.raises(NestedJSONSyntaxError) as exc:
         http(httpbin + '/post', *input_json)
 
     exc_lines = str(exc.value).splitlines()