From 245cede2c274bef7b172fe65814bfabc6f87b0b4 Mon Sep 17 00:00:00 2001 From: Batuhan Taskaya Date: Tue, 30 Nov 2021 11:12:51 +0300 Subject: [PATCH] cmd: Implement httpie plugins interface (#1200) --- docs/README.md | 69 ++++++++++++ docs/markdownlint.rb | 3 + httpie/cli/argparser.py | 81 ++++++++++---- httpie/cli/definition.py | 2 +- httpie/client.py | 2 +- httpie/compat.py | 36 ++++++ httpie/config.py | 4 + httpie/context.py | 16 ++- httpie/core.py | 48 +++++--- httpie/manager/__init__.py | 0 httpie/manager/__main__.py | 61 ++++++++++ httpie/manager/cli.py | 93 ++++++++++++++++ httpie/manager/core.py | 33 ++++++ httpie/manager/plugins.py | 188 +++++++++++++++++++++++++++++++ httpie/plugins/manager.py | 73 +++++++++--- httpie/{ssl.py => ssl_.py} | 0 httpie/utils.py | 11 ++ setup.py | 2 + tests/conftest.py | 7 ++ tests/test_plugins_cli.py | 132 ++++++++++++++++++++++ tests/test_ssl.py | 2 +- tests/utils/__init__.py | 52 ++++++++- tests/utils/plugins_cli.py | 222 +++++++++++++++++++++++++++++++++++++ 23 files changed, 1075 insertions(+), 62 deletions(-) create mode 100644 httpie/manager/__init__.py create mode 100644 httpie/manager/__main__.py create mode 100644 httpie/manager/cli.py create mode 100644 httpie/manager/core.py create mode 100644 httpie/manager/plugins.py rename httpie/{ssl.py => ssl_.py} (100%) create mode 100644 tests/test_plugins_cli.py create mode 100644 tests/utils/plugins_cli.py diff --git a/docs/README.md b/docs/README.md index a887e937..de5d28e1 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1880,6 +1880,11 @@ $ cat ~/.config/httpie/config.json Technically, it is possible to include any HTTPie options in there. However, it is not recommended to modify the default behavior in a way that would break your compatibility with the wider world as that may become confusing. +#### `plugins_dir` + +The directory where the plugins will be installed. HTTPie needs to have read/write access on that directory, since +`httpie plugins install` will download new plugins to there. + ### Un-setting previously specified options Default options from the config file, or specified any other way, can be unset for a particular invocation via `--no-OPTION` arguments passed via the command line (e.g., `--no-style` or `--no-session`). @@ -1919,6 +1924,70 @@ And since there’s neither data nor `EOF`, it will get stuck. So unless you’r Also, it might be good to set a connection `--timeout` limit to prevent your program from hanging if the server never responds. +## Plugins Manager + +HTTPie offers extensibility through plugins, and there are over 50+ of them available to try! +They add things like new authentication methods ([akamai/httpie-edgegrid](https://github.com/akamai/httpie-edgegrid)), +transport mechanisms ([httpie/httpie-unixsocket](https://github.com/httpie/httpie-unixsocket)), +message convertors ([banteg/httpie-image](https://github.com/banteg/httpie-image)), or simply +change how a response is formatted. + +> Note: Plugins are usually made by our community members, and thus have no direct relationship with +> the HTTPie project. We do not control / review them at the moment, so use them at your own discretion. + +For managing these plugins; starting with 3.0, we are offering a new plugin manager. + +This command is currently in beta. + +### `httpie plugins` + +`plugins` interface is a very simple plugin manager for installing, listing and uninstalling HTTPie plugins. + +> In the past `pip` was used to install/uninstall plugins, but on some environments (e.g brew installed +packages) it wasn't working properly. The new interface is a very simple overlay on top of `pip` to allow +plugin installations on every installation method. + +> By default the plugins (and their missing dependencies) will be stored under the configuration directory, +but this can be modified through `plugins_dir` variable on the config. + +#### `httpie plugins install` + +For installing plugins from [PyPI](https://pypi.org/) or from local paths, `httpie plugins install` +can be used. + +```bash +$ httpie plugins install httpie-plugin +Installing httpie-plugin... +Successfully installed httpie-plugin-1.0.2 +``` + +> Tip: Generally HTTPie plugins start with `httpie-` prefix. Try searching for it on [PyPI](https://pypi.org/search/?q=httpie-) +> to find out all plugins from the community. + +#### `httpie plugins list` + +List all installed plugins. + +```bash +$ httpie plugins list +httpie_plugin (1.0.2) + httpie_plugin (httpie.plugins.auth.v1) +httpie_plugin_2 (1.0.6) + httpie_plugin_2 (httpie.plugins.auth.v1) +httpie_converter (1.0.0) + httpie_iterm_converter (httpie.plugins.converter.v1) + httpie_konsole_konverter (httpie.plugins.converter.v1) +``` + +#### `httpie plugins uninstall` + +Uninstall plugins from the isolated plugins directory. If the plugin is not installed +through `httpie plugins install`, it won't uninstall it. + +```bash +$ httpie plugins uninstall httpie-plugin +``` + ## Meta ### Interface design diff --git a/docs/markdownlint.rb b/docs/markdownlint.rb index 397f4757..c31232d8 100644 --- a/docs/markdownlint.rb +++ b/docs/markdownlint.rb @@ -17,6 +17,9 @@ exclude_rule 'MD013' # MD014 Dollar signs used before commands without showing output exclude_rule 'MD014' +# MD028 Blank line inside blockquote +exclude_rule 'MD028' + # Tell the linter to use ordered lists: # 1. Foo # 2. Bar diff --git a/httpie/cli/argparser.py b/httpie/cli/argparser.py index 1a3d7589..28dcd96c 100644 --- a/httpie/cli/argparser.py +++ b/httpie/cli/argparser.py @@ -50,7 +50,64 @@ class HTTPieHelpFormatter(RawDescriptionHelpFormatter): # TODO: refactor and design type-annotated data structures # for raw args + parsed args and keep things immutable. -class HTTPieArgumentParser(argparse.ArgumentParser): +class BaseHTTPieArgumentParser(argparse.ArgumentParser): + def __init__(self, *args, formatter_class=HTTPieHelpFormatter, **kwargs): + super().__init__(*args, formatter_class=formatter_class, **kwargs) + self.env = None + self.args = None + self.has_stdin_data = False + self.has_input_data = False + + # noinspection PyMethodOverriding + def parse_args( + self, + env: Environment, + args=None, + namespace=None + ) -> argparse.Namespace: + self.env = env + self.args, no_options = self.parse_known_args(args, namespace) + if self.args.debug: + self.args.traceback = True + self.has_stdin_data = ( + self.env.stdin + and not getattr(self.args, 'ignore_stdin', False) + and not self.env.stdin_isatty + ) + self.has_input_data = self.has_stdin_data or getattr(self.args, 'raw', None) is not None + return self.args + + # noinspection PyShadowingBuiltins + def _print_message(self, message, file=None): + # Sneak in our stderr/stdout. + if hasattr(self, 'root'): + env = self.root.env + else: + env = self.env + + if env is not None: + file = { + sys.stdout: env.stdout, + sys.stderr: env.stderr, + None: env.stderr + }.get(file, file) + + if not hasattr(file, 'buffer') and isinstance(message, str): + message = message.encode(env.stdout_encoding) + super()._print_message(message, file) + + +class HTTPieManagerArgumentParser(BaseHTTPieArgumentParser): + def parse_known_args(self, args=None, namespace=None): + try: + return super().parse_known_args(args, namespace) + except SystemExit as exc: + if not hasattr(self, 'root') and exc.code == 2: # Argument Parser Error + raise argparse.ArgumentError(None, None) + raise + + +class HTTPieArgumentParser(BaseHTTPieArgumentParser): """Adds additional logic to `argparse.ArgumentParser`. Handles all input (CLI args, file args, stdin), applies defaults, @@ -58,13 +115,9 @@ class HTTPieArgumentParser(argparse.ArgumentParser): """ - def __init__(self, *args, formatter_class=HTTPieHelpFormatter, **kwargs): - kwargs['add_help'] = False - super().__init__(*args, formatter_class=formatter_class, **kwargs) - self.env = None - self.args = None - self.has_stdin_data = False - self.has_input_data = False + def __init__(self, *args, **kwargs): + kwargs.setdefault('add_help', False) + super().__init__(*args, **kwargs) # noinspection PyMethodOverriding def parse_args( @@ -141,18 +194,6 @@ class HTTPieArgumentParser(argparse.ArgumentParser): else: self.args.url = scheme + self.args.url - # noinspection PyShadowingBuiltins - def _print_message(self, message, file=None): - # Sneak in our stderr/stdout. - file = { - sys.stdout: self.env.stdout, - sys.stderr: self.env.stderr, - None: self.env.stderr - }.get(file, file) - if not hasattr(file, 'buffer') and isinstance(message, str): - message = message.encode(self.env.stdout_encoding) - super()._print_message(message, file) - def _setup_standard_streams(self): """ Modify `env.stdout` and `env.stdout_isatty` based on args, if needed. diff --git a/httpie/cli/definition.py b/httpie/cli/definition.py index 6ebb50c5..98c45987 100644 --- a/httpie/cli/definition.py +++ b/httpie/cli/definition.py @@ -25,7 +25,7 @@ from ..output.formatters.colors import ( from ..plugins.builtin import BuiltinAuthPlugin from ..plugins.registry import plugin_manager from ..sessions import DEFAULT_SESSIONS_DIR -from ..ssl import AVAILABLE_SSL_VERSION_ARG_MAPPING, DEFAULT_SSL_CIPHERS +from ..ssl_ import AVAILABLE_SSL_VERSION_ARG_MAPPING, DEFAULT_SSL_CIPHERS parser = HTTPieArgumentParser( diff --git a/httpie/client.py b/httpie/client.py index 8d338576..374e14fd 100644 --- a/httpie/client.py +++ b/httpie/client.py @@ -17,7 +17,7 @@ from .encoding import UTF8 from .models import RequestsMessage from .plugins.registry import plugin_manager from .sessions import get_httpie_session -from .ssl import AVAILABLE_SSL_VERSION_ARG_MAPPING, HTTPieHTTPSAdapter +from .ssl_ import AVAILABLE_SSL_VERSION_ARG_MAPPING, HTTPieHTTPSAdapter from .uploads import ( compress_request, prepare_request_body, get_multipart_data_and_content_type, diff --git a/httpie/compat.py b/httpie/compat.py index 43333571..3b89f8ca 100644 --- a/httpie/compat.py +++ b/httpie/compat.py @@ -1,4 +1,5 @@ import sys +from typing import Any, Optional, Iterable is_windows = 'win32' in str(sys.platform).lower() @@ -52,3 +53,38 @@ except ImportError: return self res = instance.__dict__[self.name] = self.func(instance) return res + + +# importlib_metadata was a provisional module, so the APIs changed quite a few times +# between 3.8-3.10. It was also not included in the standard library until 3.8, so +# we install the backport for <3.8. + +if sys.version_info >= (3, 8): + import importlib.metadata as importlib_metadata +else: + import importlib_metadata + + +def find_entry_points(entry_points: Any, group: str) -> Iterable[importlib_metadata.EntryPoint]: + if hasattr(entry_points, "select"): # Python 3.10+ / importlib_metadata >= 3.9.0 + return entry_points.select(group=group) + else: + return set(entry_points.get(group, ())) + + +def get_dist_name(entry_point: importlib_metadata.EntryPoint) -> Optional[str]: + dist = getattr(entry_point, "dist", None) + if dist is not None: # Python 3.10+ + return dist.name + + match = entry_point.pattern.match(entry_point.value) + if not (match and match.group('module')): + return None + + package = match.group('module').split('.')[0] + try: + metadata = importlib_metadata.metadata(package) + except importlib_metadata.PackageNotFoundError: + return None + else: + return metadata.get('name') diff --git a/httpie/config.py b/httpie/config.py index e2cc5e0e..28574e4a 100644 --- a/httpie/config.py +++ b/httpie/config.py @@ -128,3 +128,7 @@ class Config(BaseConfigDict): @property def default_options(self) -> list: return self['default_options'] + + @property + def plugins_dir(self) -> Path: + return Path(self.get('plugins_dir', self.directory / 'plugins')).resolve() diff --git a/httpie/context.py b/httpie/context.py index be2e0565..7a6e6a86 100644 --- a/httpie/context.py +++ b/httpie/context.py @@ -1,7 +1,8 @@ import sys import os +from contextlib import contextmanager from pathlib import Path -from typing import IO, Optional +from typing import Iterator, IO, Optional try: @@ -120,6 +121,19 @@ class Environment: self._devnull = open(os.devnull, 'w+') return self._devnull + @contextmanager + def as_silent(self) -> Iterator[None]: + original_stdout = self.stdout + original_stderr = self.stderr + + try: + self.stdout = self.devnull + self.stderr = self.devnull + yield + finally: + self.stdout = original_stdout + self.stderr = original_stderr + def log_error(self, msg, level='error'): assert level in ['error', 'warning'] self._orig_stderr.write(f'\n{self.program_name}: {level}: {msg}\n\n') diff --git a/httpie/core.py b/httpie/core.py index 3c6c1021..48d21bc4 100644 --- a/httpie/core.py +++ b/httpie/core.py @@ -2,7 +2,7 @@ import argparse import os import platform import sys -from typing import List, Optional, Tuple, Union +from typing import List, Optional, Tuple, Union, Callable import requests from pygments import __version__ as pygments_version @@ -24,22 +24,16 @@ from .status import ExitStatus, http_status_to_exit_status # noinspection PyDefaultArgument -def main(args: List[Union[str, bytes]] = sys.argv, env=Environment()) -> ExitStatus: - """ - The main function. - - Pre-process args, handle some special types of invocations, - and run the main program with error handling. - - Return exit status code. - - """ +def raw_main( + parser: argparse.ArgumentParser, + main_program: Callable[[argparse.Namespace, Environment], ExitStatus], + args: List[Union[str, bytes]] = sys.argv, + env: Environment = Environment() +) -> ExitStatus: program_name, *args = args env.program_name = os.path.basename(program_name) args = decode_raw_args(args, env.stdin_encoding) - plugin_manager.load_installed_plugins() - - from .cli.definition import parser + plugin_manager.load_installed_plugins(env.config.plugins_dir) if env.config.default_options: args = env.config.default_options + args @@ -72,7 +66,7 @@ def main(args: List[Union[str, bytes]] = sys.argv, env=Environment()) -> ExitSta exit_status = ExitStatus.ERROR else: try: - exit_status = program( + exit_status = main_program( args=parsed_args, env=env, ) @@ -114,6 +108,30 @@ def main(args: List[Union[str, bytes]] = sys.argv, env=Environment()) -> ExitSta return exit_status +def main( + args: List[Union[str, bytes]] = sys.argv, + env: Environment = Environment() +) -> ExitStatus: + """ + The main function. + + Pre-process args, handle some special types of invocations, + and run the main program with error handling. + + Return exit status code. + + """ + + from .cli.definition import parser + + return raw_main( + parser=parser, + main_program=program, + args=args, + env=env + ) + + def get_output_options( args: argparse.Namespace, message: RequestsMessage diff --git a/httpie/manager/__init__.py b/httpie/manager/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/httpie/manager/__main__.py b/httpie/manager/__main__.py new file mode 100644 index 00000000..50a5fbd3 --- /dev/null +++ b/httpie/manager/__main__.py @@ -0,0 +1,61 @@ +import argparse +import sys + +from typing import List, Union + +from httpie.context import Environment +from httpie.status import ExitStatus +from httpie.manager.cli import parser +from httpie.manager.core import MSG_COMMAND_CONFUSION, program as main_program + + +def is_http_command(args: List[Union[str, bytes]], env: Environment) -> bool: + """Check whether http/https parser can parse the arguments.""" + + from httpie.cli.definition import parser as http_parser + from httpie.manager.cli import COMMANDS + + # If the user already selected a top-level sub-command, never + # show the http/https version. E.g httpie plugins pie.dev/post + if len(args) >= 1 and args[0] in COMMANDS: + return False + + with env.as_silent(): + try: + http_parser.parse_args(env=env, args=args) + except (Exception, SystemExit): + return False + else: + return True + + +def main(args: List[Union[str, bytes]] = sys.argv, env: Environment = Environment()) -> ExitStatus: + from httpie.core import raw_main + + try: + return raw_main( + parser=parser, + main_program=main_program, + args=args, + env=env + ) + except argparse.ArgumentError: + program_args = args[1:] + if is_http_command(program_args, env): + env.stderr.write(MSG_COMMAND_CONFUSION.format(args=' '.join(program_args)) + "\n") + + return ExitStatus.ERROR + + +def program(): + try: + exit_status = main() + except KeyboardInterrupt: + from httpie.status import ExitStatus + exit_status = ExitStatus.ERROR_CTRL_C + + return exit_status + + +if __name__ == '__main__': # pragma: nocover + sys.exit(program()) diff --git a/httpie/manager/cli.py b/httpie/manager/cli.py new file mode 100644 index 00000000..52a6764a --- /dev/null +++ b/httpie/manager/cli.py @@ -0,0 +1,93 @@ +from textwrap import dedent +from httpie.cli.argparser import HTTPieManagerArgumentParser + +COMMANDS = { + 'plugins': { + 'help': 'Manage HTTPie plugins.', + 'install': [ + 'Install the given targets from PyPI ' + 'or from a local paths.', + { + 'dest': 'targets', + 'nargs': '+', + 'help': 'targets to install' + } + ], + 'uninstall': [ + 'Uninstall the given HTTPie plugins.', + { + 'dest': 'targets', + 'nargs': '+', + 'help': 'targets to install' + } + ], + 'list': [ + 'List all installed HTTPie plugins.' + ], + }, +} + + +def missing_subcommand(*args) -> str: + base = COMMANDS + for arg in args: + base = base[arg] + + assert isinstance(base, dict) + subcommands = ', '.join(map(repr, base.keys())) + return f'Please specify one of these: {subcommands}' + + +def generate_subparsers(root, parent_parser, definitions): + action_dest = '_'.join(parent_parser.prog.split()[1:] + ['action']) + actions = parent_parser.add_subparsers( + dest=action_dest + ) + for command, properties in definitions.items(): + is_subparser = isinstance(properties, dict) + descr = properties.pop('help', None) if is_subparser else properties.pop(0) + command_parser = actions.add_parser(command, description=descr) + command_parser.root = root + if is_subparser: + generate_subparsers(root, command_parser, properties) + continue + + for argument in properties: + command_parser.add_argument(**argument) + + +parser = HTTPieManagerArgumentParser( + prog='httpie', + description=dedent( + ''' + Managing interface for the HTTPie itself. + + Be aware that you might be looking for http/https commands for sending + HTTP requests. This command is only available for managing the HTTTPie + plugins and the configuration around it. + ''' + ), +) + +parser.add_argument( + '--debug', + action='store_true', + default=False, + help=''' + Prints the exception traceback should one occur, as well as other + information useful for debugging HTTPie itself and for reporting bugs. + + ''' +) + +parser.add_argument( + '--traceback', + action='store_true', + default=False, + help=''' + Prints the exception traceback should one occur. + + ''' +) + +generate_subparsers(parser, parser, COMMANDS) diff --git a/httpie/manager/core.py b/httpie/manager/core.py new file mode 100644 index 00000000..e2134b55 --- /dev/null +++ b/httpie/manager/core.py @@ -0,0 +1,33 @@ +import argparse + +from httpie.context import Environment +from httpie.manager.plugins import PluginInstaller +from httpie.status import ExitStatus +from httpie.manager.cli import missing_subcommand, parser + +MSG_COMMAND_CONFUSION = '''\ +This command is only for managing HTTPie plugins. +To send a request, please use the http/https commands: + + $ http {args} + + $ https {args} +''' + +# noinspection PyStringFormat +MSG_NAKED_INVOCATION = f'''\ +{missing_subcommand()} + +{MSG_COMMAND_CONFUSION} +'''.rstrip("\n").format(args='POST pie.dev/post hello=world') + + +def program(args: argparse.Namespace, env: Environment) -> ExitStatus: + if args.action is None: + parser.error(MSG_NAKED_INVOCATION) + + if args.action == 'plugins': + plugins = PluginInstaller(env, debug=args.debug) + return plugins.run(args.plugins_action, args) + + return ExitStatus.SUCCESS diff --git a/httpie/manager/plugins.py b/httpie/manager/plugins.py new file mode 100644 index 00000000..77fc0d12 --- /dev/null +++ b/httpie/manager/plugins.py @@ -0,0 +1,188 @@ +import argparse +import os +import subprocess +import sys +import textwrap +from collections import defaultdict +from contextlib import suppress +from pathlib import Path +from typing import Optional, List + +import importlib_metadata + +from httpie.manager.cli import parser, missing_subcommand +from httpie.compat import get_dist_name +from httpie.context import Environment +from httpie.status import ExitStatus + + +class PluginInstaller: + + def __init__(self, env: Environment, debug: bool = False) -> None: + self.env = env + self.dir = env.config.plugins_dir + self.debug = debug + + self.setup_plugins_dir() + + def setup_plugins_dir(self) -> None: + try: + self.dir.mkdir( + exist_ok=True, + parents=True + ) + except OSError: + self.env.stderr.write( + f'Couldn\'t create "{self.dir!s}"' + ' directory for plugin installation.' + ' Please re-check the permissions for that directory,' + ' and if needed, allow write-access.' + ) + raise + + def fail( + self, + command: str, + target: Optional[str] = None, + reason: Optional[str] = None + ) -> ExitStatus: + message = f'Can\'t {command}' + if target: + message += f' {target!r}' + if reason: + message += f': {reason}' + + self.env.stderr.write(message + '\n') + return ExitStatus.ERROR + + def pip(self, *args, **kwargs) -> subprocess.CompletedProcess: + options = { + 'check': True, + 'shell': False, + 'stdout': self.env.stdout, + 'stderr': subprocess.PIPE, + } + options.update(kwargs) + + cmd = [sys.executable, '-m', 'pip', *args] + return subprocess.run( + cmd, + **options + ) + + def install(self, targets: List[str]) -> Optional[ExitStatus]: + self.env.stdout.write(f"Installing {', '.join(targets)}...\n") + self.env.stdout.flush() + + try: + self.pip( + 'install', + f'--prefix={self.dir}', + '--no-warn-script-location', + *targets, + ) + except subprocess.CalledProcessError as error: + reason = None + if error.stderr: + stderr = error.stderr.decode() + + if self.debug: + self.env.stderr.write('Command failed: ') + self.env.stderr.write(' '.join(error.cmd) + '\n') + self.env.stderr.write(textwrap.indent(' ', stderr)) + + last_line = stderr.strip().splitlines()[-1] + severity, _, message = last_line.partition(': ') + if severity == 'ERROR': + reason = message + + return self.fail('install', ', '.join(targets), reason) + + def _uninstall(self, target: str) -> Optional[ExitStatus]: + try: + distribution = importlib_metadata.distribution(target) + except importlib_metadata.PackageNotFoundError: + return self.fail('uninstall', target, 'package is not installed') + + base_dir = Path(distribution.locate_file('.')).resolve() + if self.dir not in base_dir.parents: + # If the package is installed somewhere else (e.g on the site packages + # of the real python interpreter), than that means this package is not + # installed through us. + return self.fail('uninstall', target, + 'package is not installed through httpie plugins' + ' interface') + + files = distribution.files + if files is None: + return self.fail('uninstall', target, 'couldn\'t locate the package') + + # TODO: Consider handling failures here (e.g if it fails, + # just rever the operation and leave the site-packages + # in a proper shape). + for file in files: + with suppress(FileNotFoundError): + os.unlink(distribution.locate_file(file)) + + metadata_path = getattr(distribution, '_path', None) + if ( + metadata_path + and metadata_path.exists() + and not any(metadata_path.iterdir()) + ): + metadata_path.rmdir() + + self.env.stdout.write(f'Successfully uninstalled {target}\n') + + def uninstall(self, targets: List[str]) -> ExitStatus: + # Unfortunately uninstall doesn't work with custom pip schemes. See: + # - https://github.com/pypa/pip/issues/5595 + # - https://github.com/pypa/pip/issues/4575 + # so we have to implement our own uninstalling logic. Which works + # on top of the importlib_metadata. + + exit_code = ExitStatus.SUCCESS + for target in targets: + exit_code |= self._uninstall(target) or ExitStatus.SUCCESS + return ExitStatus(exit_code) + + def list(self) -> None: + from httpie.plugins.registry import plugin_manager + + known_plugins = defaultdict(list) + + for entry_point in plugin_manager.iter_entry_points(self.dir): + ep_info = (entry_point.group, entry_point.name) + ep_name = get_dist_name(entry_point) or entry_point.module + known_plugins[ep_name].append(ep_info) + + for plugin, entry_points in known_plugins.items(): + self.env.stdout.write(plugin) + + version = importlib_metadata.version(plugin) + if version is not None: + self.env.stdout.write(f' ({version})') + self.env.stdout.write('\n') + + for group, entry_point in sorted(entry_points): + self.env.stdout.write(f' {entry_point} ({group})\n') + + def run( + self, + action: Optional[str], + args: argparse.Namespace, + ) -> ExitStatus: + from httpie.plugins.manager import enable_plugins + + if action is None: + parser.error(missing_subcommand('plugins')) + + with enable_plugins(self.dir): + if action == 'install': + status = self.install(args.targets) + elif action == 'uninstall': + status = self.uninstall(args.targets) + elif action == 'list': + status = self.list() + + return status or ExitStatus.SUCCESS diff --git a/httpie/plugins/manager.py b/httpie/plugins/manager.py index 420fb36b..1b188e57 100644 --- a/httpie/plugins/manager.py +++ b/httpie/plugins/manager.py @@ -1,24 +1,55 @@ +import sys +import os + from itertools import groupby from operator import attrgetter -from typing import Dict, List, Type +from typing import Dict, List, Type, Iterator, TypeVar, Optional, ContextManager +from pathlib import Path +from contextlib import contextmanager -from pkg_resources import iter_entry_points +from ..compat import importlib_metadata, find_entry_points, get_dist_name -from ..utils import repr_dict -from . import AuthPlugin, ConverterPlugin, FormatterPlugin -from .base import BasePlugin, TransportPlugin +from ..utils import repr_dict, as_site +from . import AuthPlugin, ConverterPlugin, FormatterPlugin, TransportPlugin +from .base import BasePlugin -ENTRY_POINT_NAMES = [ - 'httpie.plugins.auth.v1', - 'httpie.plugins.formatter.v1', - 'httpie.plugins.converter.v1', - 'httpie.plugins.transport.v1', -] +ENTRY_POINT_CLASSES = { + 'httpie.plugins.auth.v1': AuthPlugin, + 'httpie.plugins.converter.v1': ConverterPlugin, + 'httpie.plugins.formatter.v1': FormatterPlugin, + 'httpie.plugins.transport.v1': TransportPlugin +} +ENTRY_POINT_NAMES = list(ENTRY_POINT_CLASSES.keys()) + + +@contextmanager +def _load_directory(plugins_dir: Path) -> Iterator[None]: + plugins_path = os.fspath(plugins_dir) + sys.path.insert(0, plugins_path) + try: + yield + finally: + sys.path.remove(plugins_path) + + +T = TypeVar("T") + + +@contextmanager +def nullcontext(obj: Optional[T] = None) -> Iterator[Optional[T]]: + # A naive replacement of the nullcontext() for 3.6 + yield obj + + +def enable_plugins(plugins_dir: Optional[Path]) -> ContextManager[None]: + if plugins_dir is None: + return nullcontext() + else: + return _load_directory(as_site(plugins_dir)) class PluginManager(list): - def register(self, *plugins: Type[BasePlugin]): for plugin in plugins: self.append(plugin) @@ -29,12 +60,18 @@ class PluginManager(list): def filter(self, by_type=Type[BasePlugin]): return [plugin for plugin in self if issubclass(plugin, by_type)] - def load_installed_plugins(self): - for entry_point_name in ENTRY_POINT_NAMES: - for entry_point in iter_entry_points(entry_point_name): - plugin = entry_point.load() - plugin.package_name = entry_point.dist.key - self.register(entry_point.load()) + def iter_entry_points(self, directory: Optional[Path] = None): + with enable_plugins(directory): + eps = importlib_metadata.entry_points() + + for entry_point_name in ENTRY_POINT_NAMES: + yield from find_entry_points(eps, group=entry_point_name) + + def load_installed_plugins(self, directory: Optional[Path] = None): + for entry_point in self.iter_entry_points(directory): + plugin = entry_point.load() + plugin.package_name = get_dist_name(entry_point) + self.register(entry_point.load()) # Auth def get_auth_plugins(self) -> List[Type[AuthPlugin]]: diff --git a/httpie/ssl.py b/httpie/ssl_.py similarity index 100% rename from httpie/ssl.py rename to httpie/ssl_.py diff --git a/httpie/utils.py b/httpie/utils.py index f40625ad..66f6e21d 100644 --- a/httpie/utils.py +++ b/httpie/utils.py @@ -3,8 +3,11 @@ import mimetypes import re import sys import time +import sysconfig + from collections import OrderedDict from http.cookiejar import parse_ns_headers +from pathlib import Path from pprint import pformat from typing import Any, List, Optional, Tuple @@ -207,3 +210,11 @@ def parse_content_type_header(header): value = param[index_of_equals + 1:].strip(items_to_strip) params_dict[key.lower()] = value return content_type, params_dict + + +def as_site(path: Path) -> Path: + site_packages_path = sysconfig.get_path( + 'purelib', + vars={'base': str(path)} + ) + return Path(site_packages_path) diff --git a/setup.py b/setup.py index 1238bfd3..0269d5ce 100644 --- a/setup.py +++ b/setup.py @@ -35,6 +35,7 @@ install_requires = [ 'requests-toolbelt>=0.9.1', 'multidict>=4.7.0', 'setuptools', + 'importlib-metadata>=1.4.0', ] install_requires_win_only = [ 'colorama>=0.2.4', @@ -80,6 +81,7 @@ setup( 'console_scripts': [ 'http = httpie.__main__:main', 'https = httpie.__main__:main', + 'httpie = httpie.manager.__main__:main', ], }, python_requires='>=3.6', diff --git a/tests/conftest.py b/tests/conftest.py index fe5f2116..fa0b367a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,6 +5,13 @@ import pytest from pytest_httpbin import certs from .utils import HTTPBIN_WITH_CHUNKED_SUPPORT_DOMAIN, HTTPBIN_WITH_CHUNKED_SUPPORT +from .utils.plugins_cli import ( # noqa + dummy_plugin, + dummy_plugins, + httpie_plugins, + httpie_plugins_success, + interface, +) from .utils.http_server import http_server # noqa diff --git a/tests/test_plugins_cli.py b/tests/test_plugins_cli.py new file mode 100644 index 00000000..39ee9c08 --- /dev/null +++ b/tests/test_plugins_cli.py @@ -0,0 +1,132 @@ +import pytest + +from httpie.status import ExitStatus +from tests.utils import httpie +from tests.utils.plugins_cli import parse_listing + + +def test_plugins_installation(httpie_plugins_success, interface, dummy_plugin): + lines = httpie_plugins_success('install', dummy_plugin.path) + assert lines[0].startswith( + f'Installing {dummy_plugin.path}' + ) + assert f'Successfully installed {dummy_plugin.name}-{dummy_plugin.version}' in lines + assert interface.is_installed(dummy_plugin.name) + + +def test_plugins_listing(httpie_plugins_success, interface, dummy_plugin): + httpie_plugins_success('install', dummy_plugin.path) + data = parse_listing(httpie_plugins_success('list')) + + assert data == { + dummy_plugin.name: dummy_plugin.dump() + } + + +def test_plugins_listing_multiple(interface, httpie_plugins_success, dummy_plugins): + paths = [plugin.path for plugin in dummy_plugins] + httpie_plugins_success('install', *paths) + data = parse_listing(httpie_plugins_success('list')) + + assert data == { + plugin.name: plugin.dump() + for plugin in dummy_plugins + } + + +def test_plugins_uninstall(interface, httpie_plugins_success, dummy_plugin): + httpie_plugins_success('install', dummy_plugin.path) + httpie_plugins_success('uninstall', dummy_plugin.name) + assert not interface.is_installed(dummy_plugin.name) + + +def test_plugins_listing_after_uninstall(interface, httpie_plugins_success, dummy_plugin): + httpie_plugins_success('install', dummy_plugin.path) + httpie_plugins_success('uninstall', dummy_plugin.name) + + data = parse_listing(httpie_plugins_success('list')) + assert len(data) == 0 + + +def test_plugins_uninstall_specific(interface, httpie_plugins_success): + new_plugin_1 = interface.make_dummy_plugin() + new_plugin_2 = interface.make_dummy_plugin() + target_plugin = interface.make_dummy_plugin() + + httpie_plugins_success('install', new_plugin_1.path, new_plugin_2.path, target_plugin.path) + httpie_plugins_success('uninstall', target_plugin.name) + + assert interface.is_installed(new_plugin_1.name) + assert interface.is_installed(new_plugin_2.name) + assert not interface.is_installed(target_plugin.name) + + +def test_plugins_installation_failed(httpie_plugins, interface): + plugin = interface.make_dummy_plugin(build=False) + result = httpie_plugins('install', plugin.path) + + assert result.exit_status == ExitStatus.ERROR + assert result.stderr.splitlines()[-1].strip().startswith("Can't install") + + +def test_plugins_uninstall_non_existent(httpie_plugins, interface): + plugin = interface.make_dummy_plugin(build=False) + result = httpie_plugins('uninstall', plugin.name) + + assert result.exit_status == ExitStatus.ERROR + assert ( + result.stderr.splitlines()[-1].strip() + == f"Can't uninstall '{plugin.name}': package is not installed" + ) + + +def test_plugins_double_uninstall(httpie_plugins, httpie_plugins_success, dummy_plugin): + httpie_plugins_success("install", dummy_plugin.path) + httpie_plugins_success("uninstall", dummy_plugin.name) + + result = httpie_plugins("uninstall", dummy_plugin.name) + + assert result.exit_status == ExitStatus.ERROR + assert ( + result.stderr.splitlines()[-1].strip() + == f"Can't uninstall '{dummy_plugin.name}': package is not installed" + ) + + +def test_plugins_cli_error_message_without_args(): + # No arguments + result = httpie(no_debug=True) + assert result.exit_status == ExitStatus.ERROR + assert 'usage: ' in result.stderr + assert 'specify one of these' in result.stderr + assert 'please use the http/https commands:' in result.stderr + + +@pytest.mark.parametrize( + 'example', [ + 'pie.dev/get', + 'DELETE localhost:8000/delete', + 'POST pie.dev/post header:value a=b header_2:value x:=1' + ] +) +def test_plugins_cli_error_messages_with_example(example): + result = httpie(*example.split(), no_debug=True) + assert result.exit_status == ExitStatus.ERROR + assert 'usage: ' in result.stderr + assert f'http {example}' in result.stderr + assert f'https {example}' in result.stderr + + +@pytest.mark.parametrize( + 'example', [ + 'plugins unknown', + 'plugins unknown.com A:B c=d', + 'unknown.com UNPARSABLE????SYNTAX', + ] +) +def test_plugins_cli_error_messages_invalid_example(example): + result = httpie(*example.split(), no_debug=True) + assert result.exit_status == ExitStatus.ERROR + assert 'usage: ' in result.stderr + assert f'http {example}' not in result.stderr + assert f'https {example}' not in result.stderr diff --git a/tests/test_ssl.py b/tests/test_ssl.py index e7192448..f930bf28 100644 --- a/tests/test_ssl.py +++ b/tests/test_ssl.py @@ -6,7 +6,7 @@ import pytest_httpbin.certs import requests.exceptions import urllib3 -from httpie.ssl import AVAILABLE_SSL_VERSION_ARG_MAPPING, DEFAULT_SSL_CIPHERS +from httpie.ssl_ import AVAILABLE_SSL_VERSION_ARG_MAPPING, DEFAULT_SSL_CIPHERS from httpie.status import ExitStatus from .utils import HTTP_OK, TESTS_ROOT, http diff --git a/tests/utils/__init__.py b/tests/utils/__init__.py index 0877b9ca..6f05b258 100644 --- a/tests/utils/__init__.py +++ b/tests/utils/__init__.py @@ -7,12 +7,14 @@ import json import tempfile from io import BytesIO from pathlib import Path -from typing import Optional, Union, List +from typing import Any, Optional, Union, List, Iterable + +import httpie.core as core +import httpie.manager.__main__ as manager from httpie.status import ExitStatus from httpie.config import Config from httpie.context import Environment -from httpie.core import main # pytest-httpbin currently does not support chunked requests: @@ -58,10 +60,10 @@ class MockEnvironment(Environment): stdout_isatty = True is_windows = False - def __init__(self, create_temp_config_dir=True, **kwargs): + def __init__(self, create_temp_config_dir=True, *, stdout_mode='b', **kwargs): if 'stdout' not in kwargs: kwargs['stdout'] = tempfile.TemporaryFile( - mode='w+b', + mode=f'w+{stdout_mode}', prefix='httpie_stdout' ) if 'stderr' not in kwargs: @@ -177,6 +179,46 @@ class ExitStatusError(Exception): pass +def normalize_args(args: Iterable[Any]) -> List[str]: + return [str(arg) for arg in args] + + +def httpie( + *args, + **kwargs +) -> StrCLIResponse: + """ + Run HTTPie manager command with the given + args/kwargs, and capture stderr/out and exit + status. + """ + + env = kwargs.setdefault('env', MockEnvironment()) + cli_args = ['httpie'] + if not kwargs.pop('no_debug', False): + cli_args.append('--debug') + cli_args += normalize_args(args) + exit_status = manager.main( + args=cli_args, + **kwargs + ) + + env.stdout.seek(0) + env.stderr.seek(0) + try: + response = StrCLIResponse(env.stdout.read()) + response.stderr = env.stderr.read() + response.exit_status = exit_status + response.args = cli_args + finally: + env.stdout.truncate(0) + env.stderr.truncate(0) + env.stdout.seek(0) + env.stderr.seek(0) + + return response + + def http( *args, program_name='http', @@ -254,7 +296,7 @@ def http( try: try: - exit_status = main(args=complete_args, **kwargs) + exit_status = core.main(args=complete_args, **kwargs) if '--download' in args: # Let the progress reporter thread finish. time.sleep(.5) diff --git a/tests/utils/plugins_cli.py b/tests/utils/plugins_cli.py new file mode 100644 index 00000000..4e4d8a07 --- /dev/null +++ b/tests/utils/plugins_cli.py @@ -0,0 +1,222 @@ +import secrets +import site +import sys +import textwrap + +import pytest + +from collections import defaultdict +from dataclasses import dataclass, field, asdict +from pathlib import Path +from typing import Any, List, Dict, Tuple +from unittest.mock import patch + +from httpie.context import Environment +from httpie.compat import importlib_metadata +from httpie.status import ExitStatus +from httpie.plugins.manager import ( + enable_plugins, + ENTRY_POINT_CLASSES as CLASSES, +) + + +def make_name() -> str: + return 'httpie-' + secrets.token_hex(4) + + +@dataclass +class EntryPoint: + name: str + group: str + + def dump(self) -> Dict[str, str]: + return asdict(self) + + +@dataclass +class Plugin: + interface: 'Interface' + + name: str = field(default_factory=make_name) + version: str = '1.0.0' + entry_points: List[EntryPoint] = field(default_factory=list) + + def build(self) -> None: + ''' + Create an installable dummy plugin at the given path. + + It will create a setup.py with the specified entry points, + as well as dummy classes in a python module to imitate + real plugins. + ''' + + groups = defaultdict(list) + for entry_point in self.entry_points: + groups[entry_point.group].append(entry_point.name) + + setup_eps = { + group: [ + f'{name} = {self.import_name}:{name.title()}' + for name in names + ] + for group, names in groups.items() + } + + self.path.mkdir(parents=True, exist_ok=True) + with open(self.path / 'setup.py', 'w') as stream: + stream.write(textwrap.dedent(f''' + from setuptools import setup + + setup( + name='{self.name}', + version='{self.version}', + py_modules=['{self.import_name}'], + entry_points={setup_eps!r}, + install_requires=['httpie'] + ) + ''')) + + with open(self.path / (self.import_name + '.py'), 'w') as stream: + stream.write('from httpie.plugins import *\n') + stream.writelines( + f'class {name.title()}({CLASSES[group].__name__}): ...\n' + for group, names in groups.items() + for name in names + ) + + def dump(self) -> Dict[str, Any]: + return { + 'version': self.version, + 'entry_points': [ + entry_point.dump() + for entry_point in self.entry_points + ] + } + + @property + def path(self) -> Path: + return self.interface.path / self.name + + @property + def import_name(self) -> str: + return self.name.replace('-', '_') + + +@dataclass +class Interface: + path: Path + environment: Environment + + def get_plugin(self, target: str) -> importlib_metadata.Distribution: + with enable_plugins(self.environment.config.plugins_dir): + return importlib_metadata.distribution(target) + + def is_installed(self, target: str) -> bool: + try: + self.get_plugin(target) + except ModuleNotFoundError: + return False + else: + return True + + def make_dummy_plugin(self, build=True, **kwargs) -> Plugin: + kwargs.setdefault('entry_points', [EntryPoint('test', 'httpie.plugins.auth.v1')]) + + plugin = Plugin(self, **kwargs) + if build: + plugin.build() + return plugin + + +def parse_listing(lines: List[str]) -> Dict[str, Any]: + plugins = {} + current_plugin = None + + def parse_entry_point(line: str) -> Tuple[str, str]: + entry_point, raw_group = line.strip().split() + return entry_point, raw_group[1:-1] + + def parse_plugin(line: str) -> Tuple[str, str]: + plugin, raw_version = line.strip().split() + return plugin, raw_version[1:-1] + + for line in lines: + if not line.strip(): + continue + + if line[0].isspace(): + # $entry_point ($group) + assert current_plugin is not None + entry_point, group = parse_entry_point(line) + plugins[current_plugin]['entry_points'].append({ + 'name': entry_point, + 'group': group + }) + else: + # $plugin ($version) + current_plugin, version = parse_plugin(line) + plugins[current_plugin] = { + 'version': version, + 'entry_points': [] + } + + return plugins + + +@pytest.fixture(scope='function') +def interface(tmp_path): + from tests.utils import MockEnvironment + + return Interface( + path=tmp_path / 'interface', + environment=MockEnvironment(stdout_mode='t') + ) + + +@pytest.fixture(scope='function') +def dummy_plugin(interface): + return interface.make_dummy_plugin() + + +@pytest.fixture(scope='function') +def dummy_plugins(interface): + # Multiple plugins with different configurations + return [ + interface.make_dummy_plugin(), + interface.make_dummy_plugin( + version='3.2.0' + ), + interface.make_dummy_plugin( + entry_points=[ + EntryPoint('test_1', 'httpie.plugins.converter.v1'), + EntryPoint('test_2', 'httpie.plugins.formatter.v1') + ] + ), + ] + + +@pytest.fixture +def httpie_plugins(interface): + from tests.utils import httpie + from httpie.plugins.registry import plugin_manager + + def runner(*args): + # Prevent installed plugins from showing up. + original_plugins = plugin_manager.copy() + clean_sys_path = set(sys.path).difference(site.getsitepackages()) + with patch('sys.path', list(clean_sys_path)): + response = httpie('plugins', *args, env=interface.environment) + plugin_manager.clear() + plugin_manager.extend(original_plugins) + return response + + return runner + + +@pytest.fixture +def httpie_plugins_success(httpie_plugins): + def runner(*args): + response = httpie_plugins(*args) + assert response.exit_status == ExitStatus.SUCCESS + return response.splitlines() + return runner