holm/httpie-cli
1
1
Fork 0
forked from extern/httpie-cli
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Decouple parser definition from argparse (#1293)

Browse Source
This commit is contained in:
Batuhan Taskaya 2022-03-08 01:34:04 +03:00 committed by GitHub
parent 7509dd4e6c
commit 77af4c7a5c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 708 additions and 294 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
docs/README.md
View File

@ -2402,6 +2402,27 @@ For managing these plugins; starting with 3.0, we are offering a new plugin mana
This command is currently in beta.
### `httpie cli`
#### `httpie cli export-args`
`httpie cli export-args` command can expose the parser specification of `http`/`https` commands
(like an API definition) to outside tools so that they can use this to build better interactions
over them (e.g offer auto-complete).
Available formats to export in include:
| format | Description |
|--------|---------------------------------------------------------------------------------------------------------------------------------------------------|
| `json` | Export the parser spec in JSON. The schema includes a top-level `version` parameter which should be interpreted in [semver](https://semver.org/). |
You can use any of these formats with `--format` parameter, but the default one is `json`.
```bash
$ httpie cli export-args | jq '"Program: " + .spec.name + ", Version: " + .version'
"Program: http, Version: 0.0.1a0"
```
### `httpie plugins`
`plugins` interface is a very simple plugin manager for installing, listing and uninstalling HTTPie plugins.

8
httpie/cli/constants.py
View File

@ -90,13 +90,19 @@ OUTPUT_OPTIONS = frozenset({
})
# Pretty
class PrettyOptions(enum.Enum):
STDOUT_TTY_ONLY = enum.auto()
PRETTY_MAP = {
'all': ['format', 'colors'],
'colors': ['colors'],
'format': ['format'],
'none': []
}
PRETTY_STDOUT_TTY_ONLY = object()
PRETTY_STDOUT_TTY_ONLY = PrettyOptions.STDOUT_TTY_ONLY
DEFAULT_FORMAT_OPTIONS = [

531
httpie/cli/definition.py
View File

File diff suppressed because it is too large Load Diff

189
httpie/cli/options.py Normal file
View File

@ -0,0 +1,189 @@
import argparse
import textwrap
import typing
from dataclasses import dataclass, field
from enum import Enum, auto
from typing import Any, Optional, Dict, List, Type, TypeVar
from httpie.cli.argparser import HTTPieArgumentParser
from httpie.cli.utils import LazyChoices
class Qualifiers(Enum):
OPTIONAL = auto()
ZERO_OR_MORE = auto()
SUPPRESS = auto()
def map_qualifiers(
configuration: Dict[str, Any], qualifier_map: Dict[Qualifiers, Any]
) -> Dict[str, Any]:
return {
key: qualifier_map[value] if isinstance(value, Qualifiers) else value
for key, value in configuration.items()
}
PARSER_SPEC_VERSION = '0.0.1a0'
@dataclass
class ParserSpec:
program: str
description: Optional[str] = None
epilog: Optional[str] = None
groups: List['Group'] = field(default_factory=list)
def finalize(self) -> 'ParserSpec':
if self.description:
self.description = textwrap.dedent(self.description)
if self.epilog:
self.epilog = textwrap.dedent(self.epilog)
for group in self.groups:
group.finalize()
return self
def add_group(self, name: str, **kwargs) -> 'Group':
group = Group(name, **kwargs)
self.groups.append(group)
return group
def serialize(self) -> Dict[str, Any]:
return {
'name': self.program,
'description': self.description,
'groups': [group.serialize() for group in self.groups],
}
@dataclass
class Group:
name: str
description: str = ''
is_mutually_exclusive: bool = False
arguments: List['Argument'] = field(default_factory=list)
def finalize(self) -> None:
if self.description:
self.description = textwrap.dedent(self.description)
def add_argument(self, *args, **kwargs):
argument = Argument(list(args), kwargs.copy())
self.arguments.append(argument)
return argument
def serialize(self) -> Dict[str, Any]:
return {
'name': self.name,
'description': self.description or None,
'is_mutually_exclusive': self.is_mutually_exclusive,
'args': [argument.serialize() for argument in self.arguments],
}
class Argument(typing.NamedTuple):
aliases: List[str]
configuration: Dict[str, Any]
def serialize(self) -> Dict[str, Any]:
configuration = self.configuration.copy()
# Unpack the dynamically computed choices, since we
# will need to store the actual values somewhere.
action = configuration.pop('action', None)
if action == 'lazy_choices':
choices = LazyChoices(self.aliases, **{'dest': None, **configuration})
configuration['choices'] = list(choices.load())
configuration['help'] = choices.help
result = {}
if self.aliases:
result['options'] = self.aliases.copy()
else:
result['options'] = [configuration['metavar']]
result['is_positional'] = True
qualifiers = JSON_QUALIFIER_TO_OPTIONS[configuration.get('nargs', Qualifiers.SUPPRESS)]
result.update(qualifiers)
help_msg = configuration.get('help')
if help_msg and help_msg is not Qualifiers.SUPPRESS:
result['description'] = help_msg.strip()
python_type = configuration.get('type')
if python_type is not None:
if hasattr(python_type, '__name__'):
type_name = python_type.__name__
else:
type_name = type(python_type).__name__
result['python_type_name'] = type_name
result.update({
key: value
for key, value in configuration.items()
if key in JSON_DIRECT_MIRROR_OPTIONS
})
return result
def __getattr__(self, attribute_name):
if attribute_name in self.configuration:
return self.configuration[attribute_name]
else:
raise AttributeError(attribute_name)
ParserType = TypeVar('ParserType', bound=Type[argparse.ArgumentParser])
ARGPARSE_QUALIFIER_MAP = {
Qualifiers.OPTIONAL: argparse.OPTIONAL,
Qualifiers.SUPPRESS: argparse.SUPPRESS,
Qualifiers.ZERO_OR_MORE: argparse.ZERO_OR_MORE,
}
def to_argparse(
abstract_options: ParserSpec,
parser_type: ParserType = HTTPieArgumentParser,
) -> ParserType:
concrete_parser = parser_type(
prog=abstract_options.program,
description=abstract_options.description,
epilog=abstract_options.epilog,
)
concrete_parser.register('action', 'lazy_choices', LazyChoices)
for abstract_group in abstract_options.groups:
concrete_group = concrete_parser.add_argument_group(
title=abstract_group.name, description=abstract_group.description
)
if abstract_group.is_mutually_exclusive:
concrete_group = concrete_group.add_mutually_exclusive_group(required=False)
for abstract_argument in abstract_group.arguments:
concrete_group.add_argument(
*abstract_argument.aliases,
**map_qualifiers(
abstract_argument.configuration, ARGPARSE_QUALIFIER_MAP
)
)
return concrete_parser
JSON_DIRECT_MIRROR_OPTIONS = (
'choices',
'metavar'
)
JSON_QUALIFIER_TO_OPTIONS = {
Qualifiers.OPTIONAL: {'is_optional': True},
Qualifiers.ZERO_OR_MORE: {'is_optional': True, 'is_variadic': True},
Qualifiers.SUPPRESS: {}
}
def to_data(abstract_options: ParserSpec) -> Dict[str, Any]:
return {'version': PARSER_SPEC_VERSION, 'spec': abstract_options.serialize()}

29
httpie/core.py
View File

@ -17,9 +17,10 @@ from .context import Environment, Levels
from .downloads import Downloader
from .models import (
RequestsMessageKind,
OutputOptions,
OutputOptions
)
from .output.writer import write_message, write_stream, MESSAGE_SEPARATOR_BYTES
from .output.models import ProcessingOptions
from .output.writer import write_message, write_stream, write_raw_data, MESSAGE_SEPARATOR_BYTES
from .plugins.registry import plugin_manager
from .status import ExitStatus, http_status_to_exit_status
from .utils import unwrap_context
@ -169,6 +170,7 @@ def program(args: argparse.Namespace, env: Environment) -> ExitStatus:
downloader = None
initial_request: Optional[requests.PreparedRequest] = None
final_response: Optional[requests.Response] = None
processing_options = ProcessingOptions.from_raw_args(args)
def separate():
getattr(env.stdout, 'buffer', env.stdout).write(MESSAGE_SEPARATOR_BYTES)
@ -183,12 +185,12 @@ def program(args: argparse.Namespace, env: Environment) -> ExitStatus:
and chunk
)
if should_pipe_to_stdout:
msg = requests.PreparedRequest()
msg.is_body_upload_chunk = True
msg.body = chunk
msg.headers = initial_request.headers
msg_output_options = OutputOptions.from_message(msg, body=True, headers=False)
write_message(requests_message=msg, env=env, args=args, output_options=msg_output_options)
return write_raw_data(
env,
chunk,
processing_options=processing_options,
headers=initial_request.headers
)
try:
if args.download:
@ -222,9 +224,14 @@ def program(args: argparse.Namespace, env: Environment) -> ExitStatus:
exit_status = http_status_to_exit_status(http_status=message.status_code, follow=args.follow)
if exit_status != ExitStatus.SUCCESS and (not env.stdout_isatty or args.quiet == 1):
env.log_error(f'HTTP {message.raw.status} {message.raw.reason}', level=Levels.WARNING)
write_message(requests_message=message, env=env, args=args, output_options=output_options._replace(
body=do_write_body
))
write_message(
requests_message=message,
env=env,
output_options=output_options._replace(
body=do_write_body
),
processing_options=processing_options
)
prev_with_body = output_options.body
# Cleanup

8
httpie/manager/cli.py
View File

@ -45,6 +45,14 @@ COMMANDS = {
},
'cli': {
'help': 'Manage HTTPie for Terminal',
'export-args': [
'Export available options for the CLI',
{
'flags': ['-f', '--format'],
'choices': ['json'],
'default': 'json'
}
],
'sessions': {
'help': 'Manage HTTPie sessions',
'upgrade': [

25
httpie/manager/tasks.py
View File

@ -114,3 +114,28 @@ def cli_upgrade_all_sessions(env: Environment, args: argparse.Namespace) -> Exit
session_name=session_name
)
return status
FORMAT_TO_CONTENT_TYPE = {
'json': 'application/json'
}
@task('export-args')
def cli_export(env: Environment, args: argparse.Namespace) -> ExitStatus:
import json
from httpie.cli.definition import options
from httpie.cli.options import to_data
from httpie.output.writer import write_raw_data
if args.format == 'json':
data = json.dumps(to_data(options))
else:
raise NotImplementedError(f'Unexpected format value: {args.format}')
write_raw_data(
env,
data,
stream_kwargs={'mime_overwrite': FORMAT_TO_CONTENT_TYPE[args.format]},
)
return ExitStatus.SUCCESS

44
httpie/output/models.py Normal file
View File

@ -0,0 +1,44 @@
import argparse
from typing import Any, Dict, Union, List, NamedTuple, Optional
from httpie.context import Environment
from httpie.cli.constants import PrettyOptions, PRETTY_MAP, PRETTY_STDOUT_TTY_ONLY
from httpie.cli.argtypes import PARSED_DEFAULT_FORMAT_OPTIONS
from httpie.output.formatters.colors import AUTO_STYLE
class ProcessingOptions(NamedTuple):
"""Represents a set of stylistic options
that are used when deciding which stream
should be used."""
debug: bool = False
traceback: bool = False
stream: bool = False
style: str = AUTO_STYLE
prettify: Union[List[str], PrettyOptions] = PRETTY_STDOUT_TTY_ONLY
response_mime: Optional[str] = None
response_charset: Optional[str] = None
json: bool = False
format_options: Dict[str, Any] = PARSED_DEFAULT_FORMAT_OPTIONS
def get_prettify(self, env: Environment) -> List[str]:
if self.prettify is PRETTY_STDOUT_TTY_ONLY:
return PRETTY_MAP['all' if env.stdout_isatty else 'none']
else:
return self.prettify
@classmethod
def from_raw_args(cls, options: argparse.Namespace) -> 'ProcessingOptions':
fetched_options = {
option: getattr(options, option)
for option in cls._fields
}
return cls(**fetched_options)
@property
def show_traceback(self):
return self.debug or self.traceback

4
httpie/output/streams.py
View File

@ -34,7 +34,8 @@ class BaseStream(metaclass=ABCMeta):
self,
msg: HTTPMessage,
output_options: OutputOptions,
on_body_chunk_downloaded: Callable[[bytes], None] = None
on_body_chunk_downloaded: Callable[[bytes], None] = None,
**kwargs
):
"""
:param msg: a :class:`models.HTTPMessage` subclass
@ -45,6 +46,7 @@ class BaseStream(metaclass=ABCMeta):
self.msg = msg
self.output_options = output_options
self.on_body_chunk_downloaded = on_body_chunk_downloaded
self.extra_options = kwargs
def get_headers(self) -> bytes:
"""Return the headers' bytes."""

70
httpie/output/writer.py
View File

@ -1,6 +1,6 @@
import argparse
import errno
from typing import IO, TextIO, Tuple, Type, Union
import requests
from typing import Any, Dict, IO, Optional, TextIO, Tuple, Type, Union
from ..cli.dicts import HTTPHeadersDict
from ..context import Environment
@ -10,8 +10,9 @@ from ..models import (
HTTPMessage,
RequestsMessage,
RequestsMessageKind,
OutputOptions
OutputOptions,
)
from .models import ProcessingOptions
from .processing import Conversion, Formatting
from .streams import (
BaseStream, BufferedPrettyStream, EncodedStream, PrettyStream, RawStream,
@ -25,30 +26,31 @@ MESSAGE_SEPARATOR_BYTES = MESSAGE_SEPARATOR.encode()
def write_message(
requests_message: RequestsMessage,
env: Environment,
args: argparse.Namespace,
output_options: OutputOptions,
processing_options: ProcessingOptions,
extra_stream_kwargs: Optional[Dict[str, Any]] = None
):
if not output_options.any():
return
write_stream_kwargs = {
'stream': build_output_stream_for_message(
args=args,
env=env,
requests_message=requests_message,
output_options=output_options,
processing_options=processing_options,
extra_stream_kwargs=extra_stream_kwargs
),
# NOTE: `env.stdout` will in fact be `stderr` with `--download`
'outfile': env.stdout,
'flush': env.stdout_isatty or args.stream
'flush': env.stdout_isatty or processing_options.stream
}
try:
if env.is_windows and 'colors' in args.prettify:
if env.is_windows and 'colors' in processing_options.get_prettify(env):
write_stream_with_colors_win(**write_stream_kwargs)
else:
write_stream(**write_stream_kwargs)
except OSError as e:
show_traceback = args.debug or args.traceback
if not show_traceback and e.errno == errno.EPIPE:
if processing_options.show_traceback and e.errno == errno.EPIPE:
# Ignore broken pipes unless --traceback.
env.stderr.write('\n')
else:
@ -94,11 +96,34 @@ def write_stream_with_colors_win(
outfile.flush()
def write_raw_data(
env: Environment,
data: Any,
*,
processing_options: Optional[ProcessingOptions] = None,
headers: Optional[HTTPHeadersDict] = None,
stream_kwargs: Optional[Dict[str, Any]] = None
):
msg = requests.PreparedRequest()
msg.is_body_upload_chunk = True
msg.body = data
msg.headers = headers or HTTPHeadersDict()
msg_output_options = OutputOptions.from_message(msg, body=True, headers=False)
return write_message(
requests_message=msg,
env=env,
output_options=msg_output_options,
processing_options=processing_options or ProcessingOptions(),
extra_stream_kwargs=stream_kwargs
)
def build_output_stream_for_message(
args: argparse.Namespace,
env: Environment,
requests_message: RequestsMessage,
output_options: OutputOptions,
processing_options: ProcessingOptions,
extra_stream_kwargs: Optional[Dict[str, Any]] = None
):
message_type = {
RequestsMessageKind.REQUEST: HTTPRequest,
@ -106,10 +131,12 @@ def build_output_stream_for_message(
}[output_options.kind]
stream_class, stream_kwargs = get_stream_type_and_kwargs(
env=env,
args=args,
processing_options=processing_options,
message_type=message_type,
headers=requests_message.headers
)
if extra_stream_kwargs:
stream_kwargs.update(extra_stream_kwargs)
yield from stream_class(
msg=message_type(requests_message),
output_options=output_options,
@ -124,20 +151,21 @@ def build_output_stream_for_message(
def get_stream_type_and_kwargs(
env: Environment,
args: argparse.Namespace,
processing_options: ProcessingOptions,
message_type: Type[HTTPMessage],
headers: HTTPHeadersDict,
) -> Tuple[Type['BaseStream'], dict]:
"""Pick the right stream type and kwargs for it based on `env` and `args`.
"""
is_stream = args.stream
is_stream = processing_options.stream
prettify_groups = processing_options.get_prettify(env)
if not is_stream and message_type is HTTPResponse:
# If this is a response, then check the headers for determining
# auto-streaming.
is_stream = headers.get('Content-Type') == 'text/event-stream'
if not env.stdout_isatty and not args.prettify:
if not env.stdout_isatty and not prettify_groups:
stream_class = RawStream
stream_kwargs = {
'chunk_size': (
@ -153,19 +181,19 @@ def get_stream_type_and_kwargs(
}
if message_type is HTTPResponse:
stream_kwargs.update({
'mime_overwrite': args.response_mime,
'encoding_overwrite': args.response_charset,
'mime_overwrite': processing_options.response_mime,
'encoding_overwrite': processing_options.response_charset,
})
if args.prettify:
if prettify_groups:
stream_class = PrettyStream if is_stream else BufferedPrettyStream
stream_kwargs.update({
'conversion': Conversion(),
'formatting': Formatting(
env=env,
groups=args.prettify,
color_scheme=args.style,
explicit_json=args.json,
format_options=args.format_options,
groups=prettify_groups,
color_scheme=processing_options.style,
explicit_json=processing_options.json,
format_options=processing_options.format_options,
)
})

13
tests/test_httpie_cli.py
View File

@ -3,6 +3,7 @@ import shutil
import json
from httpie.sessions import SESSIONS_DIR_NAME
from httpie.status import ExitStatus
from httpie.cli.options import PARSER_SPEC_VERSION
from tests.utils import DUMMY_HOST, httpie
from tests.fixtures import SESSION_FILES_PATH, SESSION_FILES_NEW, SESSION_FILES_OLD, read_session_file
@ -123,3 +124,15 @@ def test_httpie_sessions_upgrade_all(tmp_path, mock_env, extra_args, extra_varia
assert read_session_file(refactored_session_file) == read_session_file(
expected_session_file, extra_variables=extra_variables
)
@pytest.mark.parametrize(
'load_func, extra_options', [
(json.loads, []),
(json.loads, ['--format=json'])
]
)
def test_cli_export(load_func, extra_options):
response = httpie('cli', 'export-args', *extra_options)
assert response.exit_status == ExitStatus.SUCCESS
assert load_func(response)['version'] == PARSER_SPEC_VERSION

60
tests/test_parser_schema.py Normal file
View File

@ -0,0 +1,60 @@
from httpie.cli.options import ParserSpec, Qualifiers
def test_parser_serialization():
small_parser = ParserSpec("test_parser")
group_1 = small_parser.add_group("group_1")
group_1.add_argument("regular_arg", help="regular arg")
group_1.add_argument(
"variadic_arg",
metavar="META",
help=Qualifiers.SUPPRESS,
nargs=Qualifiers.ZERO_OR_MORE,
)
group_1.add_argument(
"-O",
"--opt-arg",
action="lazy_choices",
getter=lambda: ["opt_1", "opt_2"],
help_formatter=lambda state: ", ".join(state),
)
group_2 = small_parser.add_group("group_2")
group_2.add_argument("--typed", action="store_true", type=int)
definition = small_parser.finalize()
assert definition.serialize() == {
"name": "test_parser",
"description": None,
"groups": [
{
"name": "group_1",
"description": None,
"is_mutually_exclusive": False,
"args": [
{
"options": ["regular_arg"],
"description": "regular arg",
},
{
"options": ["variadic_arg"],
"is_optional": True,
"is_variadic": True,
"metavar": "META",
},
{
"options": ["-O", "--opt-arg"],
"description": "opt_1, opt_2",
"choices": ["opt_1", "opt_2"],
},
],
},
{
"name": "group_2",
"description": None,
"is_mutually_exclusive": False,
"args": [{"options": ["--typed"], "python_type_name": "int"}],
},
],
}