Decouple parser definition from argparse (#1293)

docs/README.md

@@ -2402,6 +2402,27 @@ For managing these plugins; starting with 3.0, we are offering a new plugin mana
 
 This command is currently in beta.
 
+### `httpie cli`
+
+#### `httpie cli export-args`
+
+`httpie cli export-args` command can expose the parser specification of `http`/`https` commands
+(like an API definition) to outside tools so that they can use this to build better interactions
+over them (e.g offer auto-complete).
+
+
+Available formats to export in include:
+| format | Description                                                                                                                                       |
+|--------|---------------------------------------------------------------------------------------------------------------------------------------------------|
+| `json` | Export the parser spec in JSON. The schema includes a top-level `version` parameter which should be interpreted in [semver](https://semver.org/). |
+
+You can use any of these formats with `--format` parameter, but the default one is `json`.
+
+```bash
+$ httpie cli export-args | jq '"Program: " + .spec.name + ", Version: " +  .version'
+"Program: http, Version: 0.0.1a0"
+```
+
 ### `httpie plugins`
 
 `plugins` interface is a very simple plugin manager for installing, listing and uninstalling HTTPie plugins.

httpie/cli/constants.py

@@ -90,13 +90,19 @@ OUTPUT_OPTIONS = frozenset({
 })
 
 # Pretty
+
+
+class PrettyOptions(enum.Enum):
+    STDOUT_TTY_ONLY = enum.auto()
+
+
 PRETTY_MAP = {
     'all': ['format', 'colors'],
     'colors': ['colors'],
     'format': ['format'],
     'none': []
 }
-PRETTY_STDOUT_TTY_ONLY = object()
+PRETTY_STDOUT_TTY_ONLY = PrettyOptions.STDOUT_TTY_ONLY
 
 
 DEFAULT_FORMAT_OPTIONS = [

httpie/cli/options.py

@@ -0,0 +1,189 @@
+import argparse
+import textwrap
+import typing
+from dataclasses import dataclass, field
+from enum import Enum, auto
+from typing import Any, Optional, Dict, List, Type, TypeVar
+
+from httpie.cli.argparser import HTTPieArgumentParser
+from httpie.cli.utils import LazyChoices
+
+
+class Qualifiers(Enum):
+    OPTIONAL = auto()
+    ZERO_OR_MORE = auto()
+    SUPPRESS = auto()
+
+
+def map_qualifiers(
+    configuration: Dict[str, Any], qualifier_map: Dict[Qualifiers, Any]
+) -> Dict[str, Any]:
+    return {
+        key: qualifier_map[value] if isinstance(value, Qualifiers) else value
+        for key, value in configuration.items()
+    }
+
+
+PARSER_SPEC_VERSION = '0.0.1a0'
+
+
+@dataclass
+class ParserSpec:
+    program: str
+    description: Optional[str] = None
+    epilog: Optional[str] = None
+    groups: List['Group'] = field(default_factory=list)
+
+    def finalize(self) -> 'ParserSpec':
+        if self.description:
+            self.description = textwrap.dedent(self.description)
+        if self.epilog:
+            self.epilog = textwrap.dedent(self.epilog)
+        for group in self.groups:
+            group.finalize()
+        return self
+
+    def add_group(self, name: str, **kwargs) -> 'Group':
+        group = Group(name, **kwargs)
+        self.groups.append(group)
+        return group
+
+    def serialize(self) -> Dict[str, Any]:
+        return {
+            'name': self.program,
+            'description': self.description,
+            'groups': [group.serialize() for group in self.groups],
+        }
+
+
+@dataclass
+class Group:
+    name: str
+    description: str = ''
+    is_mutually_exclusive: bool = False
+    arguments: List['Argument'] = field(default_factory=list)
+
+    def finalize(self) -> None:
+        if self.description:
+            self.description = textwrap.dedent(self.description)
+
+    def add_argument(self, *args, **kwargs):
+        argument = Argument(list(args), kwargs.copy())
+        self.arguments.append(argument)
+        return argument
+
+    def serialize(self) -> Dict[str, Any]:
+        return {
+            'name': self.name,
+            'description': self.description or None,
+            'is_mutually_exclusive': self.is_mutually_exclusive,
+            'args': [argument.serialize() for argument in self.arguments],
+        }
+
+
+class Argument(typing.NamedTuple):
+    aliases: List[str]
+    configuration: Dict[str, Any]
+
+    def serialize(self) -> Dict[str, Any]:
+        configuration = self.configuration.copy()
+
+        # Unpack the dynamically computed choices, since we
+        # will need to store the actual values somewhere.
+        action = configuration.pop('action', None)
+        if action == 'lazy_choices':
+            choices = LazyChoices(self.aliases, **{'dest': None, **configuration})
+            configuration['choices'] = list(choices.load())
+            configuration['help'] = choices.help
+
+        result = {}
+        if self.aliases:
+            result['options'] = self.aliases.copy()
+        else:
+            result['options'] = [configuration['metavar']]
+            result['is_positional'] = True
+
+        qualifiers = JSON_QUALIFIER_TO_OPTIONS[configuration.get('nargs', Qualifiers.SUPPRESS)]
+        result.update(qualifiers)
+
+        help_msg = configuration.get('help')
+        if help_msg and help_msg is not Qualifiers.SUPPRESS:
+            result['description'] = help_msg.strip()
+
+        python_type = configuration.get('type')
+        if python_type is not None:
+            if hasattr(python_type, '__name__'):
+                type_name = python_type.__name__
+            else:
+                type_name = type(python_type).__name__
+
+            result['python_type_name'] = type_name
+
+        result.update({
+            key: value
+            for key, value in configuration.items()
+            if key in JSON_DIRECT_MIRROR_OPTIONS
+        })
+
+        return result
+
+    def __getattr__(self, attribute_name):
+        if attribute_name in self.configuration:
+            return self.configuration[attribute_name]
+        else:
+            raise AttributeError(attribute_name)
+
+
+ParserType = TypeVar('ParserType', bound=Type[argparse.ArgumentParser])
+
+ARGPARSE_QUALIFIER_MAP = {
+    Qualifiers.OPTIONAL: argparse.OPTIONAL,
+    Qualifiers.SUPPRESS: argparse.SUPPRESS,
+    Qualifiers.ZERO_OR_MORE: argparse.ZERO_OR_MORE,
+}
+
+
+def to_argparse(
+    abstract_options: ParserSpec,
+    parser_type: ParserType = HTTPieArgumentParser,
+) -> ParserType:
+    concrete_parser = parser_type(
+        prog=abstract_options.program,
+        description=abstract_options.description,
+        epilog=abstract_options.epilog,
+    )
+    concrete_parser.register('action', 'lazy_choices', LazyChoices)
+
+    for abstract_group in abstract_options.groups:
+        concrete_group = concrete_parser.add_argument_group(
+            title=abstract_group.name, description=abstract_group.description
+        )
+        if abstract_group.is_mutually_exclusive:
+            concrete_group = concrete_group.add_mutually_exclusive_group(required=False)
+
+        for abstract_argument in abstract_group.arguments:
+            concrete_group.add_argument(
+                *abstract_argument.aliases,
+                **map_qualifiers(
+                    abstract_argument.configuration, ARGPARSE_QUALIFIER_MAP
+                )
+            )
+
+    return concrete_parser
+
+
+JSON_DIRECT_MIRROR_OPTIONS = (
+    'choices',
+    'metavar'
+)
+
+
+JSON_QUALIFIER_TO_OPTIONS = {
+    Qualifiers.OPTIONAL: {'is_optional': True},
+    Qualifiers.ZERO_OR_MORE: {'is_optional': True, 'is_variadic': True},
+    Qualifiers.SUPPRESS: {}
+}
+
+
+def to_data(abstract_options: ParserSpec) -> Dict[str, Any]:
+    return {'version': PARSER_SPEC_VERSION, 'spec': abstract_options.serialize()}

httpie/core.py

@@ -17,9 +17,10 @@ from .context import Environment, Levels
 from .downloads import Downloader
 from .models import (
     RequestsMessageKind,
-    OutputOptions,
+    OutputOptions
 )
-from .output.writer import write_message, write_stream, MESSAGE_SEPARATOR_BYTES
+from .output.models import ProcessingOptions
+from .output.writer import write_message, write_stream, write_raw_data, MESSAGE_SEPARATOR_BYTES
 from .plugins.registry import plugin_manager
 from .status import ExitStatus, http_status_to_exit_status
 from .utils import unwrap_context
@@ -169,6 +170,7 @@ def program(args: argparse.Namespace, env: Environment) -> ExitStatus:
     downloader = None
     initial_request: Optional[requests.PreparedRequest] = None
     final_response: Optional[requests.Response] = None
+    processing_options = ProcessingOptions.from_raw_args(args)
 
     def separate():
         getattr(env.stdout, 'buffer', env.stdout).write(MESSAGE_SEPARATOR_BYTES)
@@ -183,12 +185,12 @@ def program(args: argparse.Namespace, env: Environment) -> ExitStatus:
             and chunk
         )
         if should_pipe_to_stdout:
-            msg = requests.PreparedRequest()
+            return write_raw_data(
-            msg.is_body_upload_chunk = True
+                env,
-            msg.body = chunk
+                chunk,
-            msg.headers = initial_request.headers
+                processing_options=processing_options,
-            msg_output_options = OutputOptions.from_message(msg, body=True, headers=False)
+                headers=initial_request.headers
-            write_message(requests_message=msg, env=env, args=args, output_options=msg_output_options)
+            )
 
     try:
         if args.download:
@@ -222,9 +224,14 @@ def program(args: argparse.Namespace, env: Environment) -> ExitStatus:
                     exit_status = http_status_to_exit_status(http_status=message.status_code, follow=args.follow)
                     if exit_status != ExitStatus.SUCCESS and (not env.stdout_isatty or args.quiet == 1):
                         env.log_error(f'HTTP {message.raw.status} {message.raw.reason}', level=Levels.WARNING)
-            write_message(requests_message=message, env=env, args=args, output_options=output_options._replace(
+            write_message(
+                requests_message=message,
+                env=env,
+                output_options=output_options._replace(
                     body=do_write_body
-            ))
+                ),
+                processing_options=processing_options
+            )
             prev_with_body = output_options.body
 
         # Cleanup

httpie/manager/cli.py

@@ -45,6 +45,14 @@ COMMANDS = {
     },
     'cli': {
         'help': 'Manage HTTPie for Terminal',
+        'export-args': [
+            'Export available options for the CLI',
+            {
+                'flags': ['-f', '--format'],
+                'choices': ['json'],
+                'default': 'json'
+            }
+        ],
         'sessions': {
             'help': 'Manage HTTPie sessions',
             'upgrade': [

httpie/manager/tasks.py

@@ -114,3 +114,28 @@ def cli_upgrade_all_sessions(env: Environment, args: argparse.Namespace) -> Exit
                 session_name=session_name
             )
     return status
+
+
+FORMAT_TO_CONTENT_TYPE = {
+    'json': 'application/json'
+}
+
+
+@task('export-args')
+def cli_export(env: Environment, args: argparse.Namespace) -> ExitStatus:
+    import json
+    from httpie.cli.definition import options
+    from httpie.cli.options import to_data
+    from httpie.output.writer import write_raw_data
+
+    if args.format == 'json':
+        data = json.dumps(to_data(options))
+    else:
+        raise NotImplementedError(f'Unexpected format value: {args.format}')
+
+    write_raw_data(
+        env,
+        data,
+        stream_kwargs={'mime_overwrite': FORMAT_TO_CONTENT_TYPE[args.format]},
+    )
+    return ExitStatus.SUCCESS

httpie/output/models.py

@@ -0,0 +1,44 @@
+import argparse
+from typing import Any, Dict, Union, List, NamedTuple, Optional
+
+from httpie.context import Environment
+from httpie.cli.constants import PrettyOptions, PRETTY_MAP, PRETTY_STDOUT_TTY_ONLY
+from httpie.cli.argtypes import PARSED_DEFAULT_FORMAT_OPTIONS
+from httpie.output.formatters.colors import AUTO_STYLE
+
+
+class ProcessingOptions(NamedTuple):
+    """Represents a set of stylistic options
+    that are used when deciding which stream
+    should be used."""
+
+    debug: bool = False
+    traceback: bool = False
+
+    stream: bool = False
+    style: str = AUTO_STYLE
+    prettify: Union[List[str], PrettyOptions] = PRETTY_STDOUT_TTY_ONLY
+
+    response_mime: Optional[str] = None
+    response_charset: Optional[str] = None
+
+    json: bool = False
+    format_options: Dict[str, Any] = PARSED_DEFAULT_FORMAT_OPTIONS
+
+    def get_prettify(self, env: Environment) -> List[str]:
+        if self.prettify is PRETTY_STDOUT_TTY_ONLY:
+            return PRETTY_MAP['all' if env.stdout_isatty else 'none']
+        else:
+            return self.prettify
+
+    @classmethod
+    def from_raw_args(cls, options: argparse.Namespace) -> 'ProcessingOptions':
+        fetched_options = {
+            option: getattr(options, option)
+            for option in cls._fields
+        }
+        return cls(**fetched_options)
+
+    @property
+    def show_traceback(self):
+        return self.debug or self.traceback

httpie/output/streams.py

@@ -34,7 +34,8 @@ class BaseStream(metaclass=ABCMeta):
         self,
         msg: HTTPMessage,
         output_options: OutputOptions,
-        on_body_chunk_downloaded: Callable[[bytes], None] = None
+        on_body_chunk_downloaded: Callable[[bytes], None] = None,
+        **kwargs
     ):
         """
         :param msg: a :class:`models.HTTPMessage` subclass
@@ -45,6 +46,7 @@ class BaseStream(metaclass=ABCMeta):
         self.msg = msg
         self.output_options = output_options
         self.on_body_chunk_downloaded = on_body_chunk_downloaded
+        self.extra_options = kwargs
 
     def get_headers(self) -> bytes:
         """Return the headers' bytes."""

httpie/output/writer.py

@@ -1,6 +1,6 @@
-import argparse
 import errno
-from typing import IO, TextIO, Tuple, Type, Union
+import requests
+from typing import Any, Dict, IO, Optional, TextIO, Tuple, Type, Union
 
 from ..cli.dicts import HTTPHeadersDict
 from ..context import Environment
@@ -10,8 +10,9 @@ from ..models import (
     HTTPMessage,
     RequestsMessage,
     RequestsMessageKind,
-    OutputOptions
+    OutputOptions,
 )
+from .models import ProcessingOptions
 from .processing import Conversion, Formatting
 from .streams import (
     BaseStream, BufferedPrettyStream, EncodedStream, PrettyStream, RawStream,
@@ -25,30 +26,31 @@ MESSAGE_SEPARATOR_BYTES = MESSAGE_SEPARATOR.encode()
 def write_message(
     requests_message: RequestsMessage,
     env: Environment,
-    args: argparse.Namespace,
     output_options: OutputOptions,
+    processing_options: ProcessingOptions,
+    extra_stream_kwargs: Optional[Dict[str, Any]] = None
 ):
     if not output_options.any():
         return
     write_stream_kwargs = {
         'stream': build_output_stream_for_message(
-            args=args,
             env=env,
             requests_message=requests_message,
             output_options=output_options,
+            processing_options=processing_options,
+            extra_stream_kwargs=extra_stream_kwargs
         ),
         # NOTE: `env.stdout` will in fact be `stderr` with `--download`
         'outfile': env.stdout,
-        'flush': env.stdout_isatty or args.stream
+        'flush': env.stdout_isatty or processing_options.stream
     }
     try:
-        if env.is_windows and 'colors' in args.prettify:
+        if env.is_windows and 'colors' in processing_options.get_prettify(env):
             write_stream_with_colors_win(**write_stream_kwargs)
         else:
             write_stream(**write_stream_kwargs)
     except OSError as e:
-        show_traceback = args.debug or args.traceback
+        if processing_options.show_traceback and e.errno == errno.EPIPE:
-        if not show_traceback and e.errno == errno.EPIPE:
             # Ignore broken pipes unless --traceback.
             env.stderr.write('\n')
         else:
@@ -94,11 +96,34 @@ def write_stream_with_colors_win(
             outfile.flush()
 
 
+def write_raw_data(
+    env: Environment,
+    data: Any,
+    *,
+    processing_options: Optional[ProcessingOptions] = None,
+    headers: Optional[HTTPHeadersDict] = None,
+    stream_kwargs: Optional[Dict[str, Any]] = None
+):
+    msg = requests.PreparedRequest()
+    msg.is_body_upload_chunk = True
+    msg.body = data
+    msg.headers = headers or HTTPHeadersDict()
+    msg_output_options = OutputOptions.from_message(msg, body=True, headers=False)
+    return write_message(
+        requests_message=msg,
+        env=env,
+        output_options=msg_output_options,
+        processing_options=processing_options or ProcessingOptions(),
+        extra_stream_kwargs=stream_kwargs
+    )
+
+
 def build_output_stream_for_message(
-    args: argparse.Namespace,
     env: Environment,
     requests_message: RequestsMessage,
     output_options: OutputOptions,
+    processing_options: ProcessingOptions,
+    extra_stream_kwargs: Optional[Dict[str, Any]] = None
 ):
     message_type = {
         RequestsMessageKind.REQUEST: HTTPRequest,
@@ -106,10 +131,12 @@ def build_output_stream_for_message(
     }[output_options.kind]
     stream_class, stream_kwargs = get_stream_type_and_kwargs(
         env=env,
-        args=args,
+        processing_options=processing_options,
         message_type=message_type,
         headers=requests_message.headers
     )
+    if extra_stream_kwargs:
+        stream_kwargs.update(extra_stream_kwargs)
     yield from stream_class(
         msg=message_type(requests_message),
         output_options=output_options,
@@ -124,20 +151,21 @@ def build_output_stream_for_message(
 
 def get_stream_type_and_kwargs(
     env: Environment,
-    args: argparse.Namespace,
+    processing_options: ProcessingOptions,
     message_type: Type[HTTPMessage],
     headers: HTTPHeadersDict,
 ) -> Tuple[Type['BaseStream'], dict]:
     """Pick the right stream type and kwargs for it based on `env` and `args`.
 
     """
-    is_stream = args.stream
+    is_stream = processing_options.stream
+    prettify_groups = processing_options.get_prettify(env)
     if not is_stream and message_type is HTTPResponse:
         # If this is a response, then check the headers for determining
         # auto-streaming.
         is_stream = headers.get('Content-Type') == 'text/event-stream'
 
-    if not env.stdout_isatty and not args.prettify:
+    if not env.stdout_isatty and not prettify_groups:
         stream_class = RawStream
         stream_kwargs = {
             'chunk_size': (
@@ -153,19 +181,19 @@ def get_stream_type_and_kwargs(
         }
         if message_type is HTTPResponse:
             stream_kwargs.update({
-                'mime_overwrite': args.response_mime,
+                'mime_overwrite': processing_options.response_mime,
-                'encoding_overwrite': args.response_charset,
+                'encoding_overwrite': processing_options.response_charset,
             })
-        if args.prettify:
+        if prettify_groups:
             stream_class = PrettyStream if is_stream else BufferedPrettyStream
             stream_kwargs.update({
                 'conversion': Conversion(),
                 'formatting': Formatting(
                     env=env,
-                    groups=args.prettify,
+                    groups=prettify_groups,
-                    color_scheme=args.style,
+                    color_scheme=processing_options.style,
-                    explicit_json=args.json,
+                    explicit_json=processing_options.json,
-                    format_options=args.format_options,
+                    format_options=processing_options.format_options,
                 )
             })
 

tests/test_httpie_cli.py

@@ -3,6 +3,7 @@ import shutil
 import json
 from httpie.sessions import SESSIONS_DIR_NAME
 from httpie.status import ExitStatus
+from httpie.cli.options import PARSER_SPEC_VERSION
 from tests.utils import DUMMY_HOST, httpie
 from tests.fixtures import SESSION_FILES_PATH, SESSION_FILES_NEW, SESSION_FILES_OLD, read_session_file
 
@@ -123,3 +124,15 @@ def test_httpie_sessions_upgrade_all(tmp_path, mock_env, extra_args, extra_varia
         assert read_session_file(refactored_session_file) == read_session_file(
             expected_session_file, extra_variables=extra_variables
         )
+
+
+@pytest.mark.parametrize(
+    'load_func, extra_options', [
+        (json.loads, []),
+        (json.loads, ['--format=json'])
+    ]
+)
+def test_cli_export(load_func, extra_options):
+    response = httpie('cli', 'export-args', *extra_options)
+    assert response.exit_status == ExitStatus.SUCCESS
+    assert load_func(response)['version'] == PARSER_SPEC_VERSION

tests/test_parser_schema.py

@@ -0,0 +1,60 @@
+from httpie.cli.options import ParserSpec, Qualifiers
+
+
+def test_parser_serialization():
+    small_parser = ParserSpec("test_parser")
+
+    group_1 = small_parser.add_group("group_1")
+    group_1.add_argument("regular_arg", help="regular arg")
+    group_1.add_argument(
+        "variadic_arg",
+        metavar="META",
+        help=Qualifiers.SUPPRESS,
+        nargs=Qualifiers.ZERO_OR_MORE,
+    )
+    group_1.add_argument(
+        "-O",
+        "--opt-arg",
+        action="lazy_choices",
+        getter=lambda: ["opt_1", "opt_2"],
+        help_formatter=lambda state: ", ".join(state),
+    )
+
+    group_2 = small_parser.add_group("group_2")
+    group_2.add_argument("--typed", action="store_true", type=int)
+
+    definition = small_parser.finalize()
+    assert definition.serialize() == {
+        "name": "test_parser",
+        "description": None,
+        "groups": [
+            {
+                "name": "group_1",
+                "description": None,
+                "is_mutually_exclusive": False,
+                "args": [
+                    {
+                        "options": ["regular_arg"],
+                        "description": "regular arg",
+                    },
+                    {
+                        "options": ["variadic_arg"],
+                        "is_optional": True,
+                        "is_variadic": True,
+                        "metavar": "META",
+                    },
+                    {
+                        "options": ["-O", "--opt-arg"],
+                        "description": "opt_1, opt_2",
+                        "choices": ["opt_1", "opt_2"],
+                    },
+                ],
+            },
+            {
+                "name": "group_2",
+                "description": None,
+                "is_mutually_exclusive": False,
+                "args": [{"options": ["--typed"], "python_type_name": "int"}],
+            },
+        ],
+    }