cmd: Implement httpie plugins interface (#1200)

docs/README.md

@@ -1880,6 +1880,11 @@ $ cat ~/.config/httpie/config.json
 Technically, it is possible to include any HTTPie options in there.
 However, it is not recommended to modify the default behavior in a way that would break your compatibility with the wider world as that may become confusing.
 
+#### `plugins_dir`
+
+The directory where the plugins will be installed. HTTPie needs to have read/write access on that directory, since
+`httpie plugins install` will download new plugins to there.
+
 ### Un-setting previously specified options
 
 Default options from the config file, or specified any other way, can be unset for a particular invocation via `--no-OPTION` arguments passed via the command line (e.g., `--no-style` or `--no-session`).
@@ -1919,6 +1924,70 @@ And since there’s neither data nor `EOF`, it will get stuck. So unless you’r
 
 Also, it might be good to set a connection `--timeout` limit to prevent your program from hanging if the server never responds.
 
+## Plugins Manager
+
+HTTPie offers extensibility through plugins, and there are over 50+ of them available to try!
+They add things like new authentication methods ([akamai/httpie-edgegrid](https://github.com/akamai/httpie-edgegrid)),
+transport mechanisms ([httpie/httpie-unixsocket](https://github.com/httpie/httpie-unixsocket)),
+message convertors ([banteg/httpie-image](https://github.com/banteg/httpie-image)), or simply
+change how a response is formatted.
+
+> Note: Plugins are usually made by our community members, and thus have no direct relationship with
+> the HTTPie project. We do not control / review them at the moment, so use them at your own discretion.
+
+For managing these plugins; starting with 3.0, we are offering a new plugin manager.
+
+This command is currently in beta.
+
+### `httpie plugins`
+
+`plugins` interface is a very simple plugin manager for installing, listing and uninstalling HTTPie plugins.
+
+> In the past `pip` was used to install/uninstall plugins, but on some environments (e.g brew installed
+packages) it wasn't working properly. The new interface is a very simple overlay on top of `pip` to allow
+plugin installations on every installation method.
+
+> By default the plugins (and their missing dependencies) will be stored under the configuration directory,
+but this can be modified through `plugins_dir` variable on the config.
+
+#### `httpie plugins install`
+
+For installing plugins from [PyPI](https://pypi.org/) or from local paths, `httpie plugins install`
+can be used.
+
+```bash
+$ httpie plugins install httpie-plugin
+Installing httpie-plugin...
+Successfully installed httpie-plugin-1.0.2
+```
+
+> Tip: Generally HTTPie plugins start with `httpie-` prefix. Try searching for it on [PyPI](https://pypi.org/search/?q=httpie-)
+> to find out all plugins from the community.
+
+#### `httpie plugins list`
+
+List all installed plugins.
+
+```bash
+$ httpie plugins list
+httpie_plugin (1.0.2)
+  httpie_plugin (httpie.plugins.auth.v1)
+httpie_plugin_2 (1.0.6)
+  httpie_plugin_2 (httpie.plugins.auth.v1)
+httpie_converter (1.0.0)
+  httpie_iterm_converter (httpie.plugins.converter.v1)
+  httpie_konsole_konverter (httpie.plugins.converter.v1)
+```
+
+#### `httpie plugins uninstall`
+
+Uninstall plugins from the isolated plugins directory. If the plugin is not installed
+through `httpie plugins install`, it won't uninstall it.
+
+```bash
+$ httpie plugins uninstall httpie-plugin
+```
+
 ## Meta
 
 ### Interface design

docs/markdownlint.rb

@@ -17,6 +17,9 @@ exclude_rule 'MD013'
 # MD014 Dollar signs used before commands without showing output
 exclude_rule 'MD014'
 
+# MD028 Blank line inside blockquote
+exclude_rule 'MD028'
+
 # Tell the linter to use ordered lists:
 #   1. Foo
 #   2. Bar

httpie/cli/argparser.py

@@ -50,7 +50,64 @@ class HTTPieHelpFormatter(RawDescriptionHelpFormatter):
 
 # TODO: refactor and design type-annotated data structures
 #       for raw args + parsed args and keep things immutable.
-class HTTPieArgumentParser(argparse.ArgumentParser):
+class BaseHTTPieArgumentParser(argparse.ArgumentParser):
+    def __init__(self, *args, formatter_class=HTTPieHelpFormatter, **kwargs):
+        super().__init__(*args, formatter_class=formatter_class, **kwargs)
+        self.env = None
+        self.args = None
+        self.has_stdin_data = False
+        self.has_input_data = False
+
+    # noinspection PyMethodOverriding
+    def parse_args(
+        self,
+        env: Environment,
+        args=None,
+        namespace=None
+    ) -> argparse.Namespace:
+        self.env = env
+        self.args, no_options = self.parse_known_args(args, namespace)
+        if self.args.debug:
+            self.args.traceback = True
+        self.has_stdin_data = (
+            self.env.stdin
+            and not getattr(self.args, 'ignore_stdin', False)
+            and not self.env.stdin_isatty
+        )
+        self.has_input_data = self.has_stdin_data or getattr(self.args, 'raw', None) is not None
+        return self.args
+
+    # noinspection PyShadowingBuiltins
+    def _print_message(self, message, file=None):
+        # Sneak in our stderr/stdout.
+        if hasattr(self, 'root'):
+            env = self.root.env
+        else:
+            env = self.env
+
+        if env is not None:
+            file = {
+                sys.stdout: env.stdout,
+                sys.stderr: env.stderr,
+                None: env.stderr
+            }.get(file, file)
+
+        if not hasattr(file, 'buffer') and isinstance(message, str):
+            message = message.encode(env.stdout_encoding)
+        super()._print_message(message, file)
+
+
+class HTTPieManagerArgumentParser(BaseHTTPieArgumentParser):
+    def parse_known_args(self, args=None, namespace=None):
+        try:
+            return super().parse_known_args(args, namespace)
+        except SystemExit as exc:
+            if not hasattr(self, 'root') and exc.code == 2:  # Argument Parser Error
+                raise argparse.ArgumentError(None, None)
+            raise
+
+
+class HTTPieArgumentParser(BaseHTTPieArgumentParser):
     """Adds additional logic to `argparse.ArgumentParser`.
 
     Handles all input (CLI args, file args, stdin), applies defaults,
@@ -58,13 +115,9 @@ class HTTPieArgumentParser(argparse.ArgumentParser):
 
     """
 
-    def __init__(self, *args, formatter_class=HTTPieHelpFormatter, **kwargs):
+    def __init__(self, *args, **kwargs):
-        kwargs['add_help'] = False
+        kwargs.setdefault('add_help', False)
-        super().__init__(*args, formatter_class=formatter_class, **kwargs)
+        super().__init__(*args, **kwargs)
-        self.env = None
-        self.args = None
-        self.has_stdin_data = False
-        self.has_input_data = False
 
     # noinspection PyMethodOverriding
     def parse_args(
@@ -141,18 +194,6 @@ class HTTPieArgumentParser(argparse.ArgumentParser):
             else:
                 self.args.url = scheme + self.args.url
 
-    # noinspection PyShadowingBuiltins
-    def _print_message(self, message, file=None):
-        # Sneak in our stderr/stdout.
-        file = {
-            sys.stdout: self.env.stdout,
-            sys.stderr: self.env.stderr,
-            None: self.env.stderr
-        }.get(file, file)
-        if not hasattr(file, 'buffer') and isinstance(message, str):
-            message = message.encode(self.env.stdout_encoding)
-        super()._print_message(message, file)
-
     def _setup_standard_streams(self):
         """
         Modify `env.stdout` and `env.stdout_isatty` based on args, if needed.

httpie/cli/definition.py

@@ -25,7 +25,7 @@ from ..output.formatters.colors import (
 from ..plugins.builtin import BuiltinAuthPlugin
 from ..plugins.registry import plugin_manager
 from ..sessions import DEFAULT_SESSIONS_DIR
-from ..ssl import AVAILABLE_SSL_VERSION_ARG_MAPPING, DEFAULT_SSL_CIPHERS
+from ..ssl_ import AVAILABLE_SSL_VERSION_ARG_MAPPING, DEFAULT_SSL_CIPHERS
 
 
 parser = HTTPieArgumentParser(

httpie/client.py

@@ -17,7 +17,7 @@ from .encoding import UTF8
 from .models import RequestsMessage
 from .plugins.registry import plugin_manager
 from .sessions import get_httpie_session
-from .ssl import AVAILABLE_SSL_VERSION_ARG_MAPPING, HTTPieHTTPSAdapter
+from .ssl_ import AVAILABLE_SSL_VERSION_ARG_MAPPING, HTTPieHTTPSAdapter
 from .uploads import (
     compress_request, prepare_request_body,
     get_multipart_data_and_content_type,

httpie/compat.py

@@ -1,4 +1,5 @@
 import sys
+from typing import Any, Optional, Iterable
 
 
 is_windows = 'win32' in str(sys.platform).lower()
@@ -52,3 +53,38 @@ except ImportError:
                 return self
             res = instance.__dict__[self.name] = self.func(instance)
             return res
+
+
+# importlib_metadata was a provisional module, so the APIs changed quite a few times
+# between 3.8-3.10. It was also not included in the standard library until 3.8, so
+# we install the backport for <3.8.
+
+if sys.version_info >= (3, 8):
+    import importlib.metadata as importlib_metadata
+else:
+    import importlib_metadata
+
+
+def find_entry_points(entry_points: Any, group: str) -> Iterable[importlib_metadata.EntryPoint]:
+    if hasattr(entry_points, "select"):  # Python 3.10+ / importlib_metadata >= 3.9.0
+        return entry_points.select(group=group)
+    else:
+        return set(entry_points.get(group, ()))
+
+
+def get_dist_name(entry_point: importlib_metadata.EntryPoint) -> Optional[str]:
+    dist = getattr(entry_point, "dist", None)
+    if dist is not None:  # Python 3.10+
+        return dist.name
+
+    match = entry_point.pattern.match(entry_point.value)
+    if not (match and match.group('module')):
+        return None
+
+    package = match.group('module').split('.')[0]
+    try:
+        metadata = importlib_metadata.metadata(package)
+    except importlib_metadata.PackageNotFoundError:
+        return None
+    else:
+        return metadata.get('name')

httpie/config.py

@@ -128,3 +128,7 @@ class Config(BaseConfigDict):
     @property
     def default_options(self) -> list:
         return self['default_options']
+
+    @property
+    def plugins_dir(self) -> Path:
+        return Path(self.get('plugins_dir', self.directory / 'plugins')).resolve()

httpie/context.py

@@ -1,7 +1,8 @@
 import sys
 import os
+from contextlib import contextmanager
 from pathlib import Path
-from typing import IO, Optional
+from typing import Iterator, IO, Optional
 
 
 try:
@@ -120,6 +121,19 @@ class Environment:
             self._devnull = open(os.devnull, 'w+')
         return self._devnull
 
+    @contextmanager
+    def as_silent(self) -> Iterator[None]:
+        original_stdout = self.stdout
+        original_stderr = self.stderr
+
+        try:
+            self.stdout = self.devnull
+            self.stderr = self.devnull
+            yield
+        finally:
+            self.stdout = original_stdout
+            self.stderr = original_stderr
+
     def log_error(self, msg, level='error'):
         assert level in ['error', 'warning']
         self._orig_stderr.write(f'\n{self.program_name}: {level}: {msg}\n\n')

httpie/core.py

@@ -2,7 +2,7 @@ import argparse
 import os
 import platform
 import sys
-from typing import List, Optional, Tuple, Union
+from typing import List, Optional, Tuple, Union, Callable
 
 import requests
 from pygments import __version__ as pygments_version
@@ -24,22 +24,16 @@ from .status import ExitStatus, http_status_to_exit_status
 
 
 # noinspection PyDefaultArgument
-def main(args: List[Union[str, bytes]] = sys.argv, env=Environment()) -> ExitStatus:
+def raw_main(
-    """
+    parser: argparse.ArgumentParser,
-    The main function.
+    main_program: Callable[[argparse.Namespace, Environment], ExitStatus],
-
+    args: List[Union[str, bytes]] = sys.argv,
-    Pre-process args, handle some special types of invocations,
+    env: Environment = Environment()
-    and run the main program with error handling.
+) -> ExitStatus:
-
-    Return exit status code.
-
-    """
     program_name, *args = args
     env.program_name = os.path.basename(program_name)
     args = decode_raw_args(args, env.stdin_encoding)
-    plugin_manager.load_installed_plugins()
+    plugin_manager.load_installed_plugins(env.config.plugins_dir)
-
-    from .cli.definition import parser
 
     if env.config.default_options:
         args = env.config.default_options + args
@@ -72,7 +66,7 @@ def main(args: List[Union[str, bytes]] = sys.argv, env=Environment()) -> ExitSta
             exit_status = ExitStatus.ERROR
     else:
         try:
-            exit_status = program(
+            exit_status = main_program(
                 args=parsed_args,
                 env=env,
             )
@@ -114,6 +108,30 @@ def main(args: List[Union[str, bytes]] = sys.argv, env=Environment()) -> ExitSta
     return exit_status
 
 
+def main(
+    args: List[Union[str, bytes]] = sys.argv,
+    env: Environment = Environment()
+) -> ExitStatus:
+    """
+    The main function.
+
+    Pre-process args, handle some special types of invocations,
+    and run the main program with error handling.
+
+    Return exit status code.
+
+    """
+
+    from .cli.definition import parser
+
+    return raw_main(
+        parser=parser,
+        main_program=program,
+        args=args,
+        env=env
+    )
+
+
 def get_output_options(
     args: argparse.Namespace,
     message: RequestsMessage

httpie/manager/__main__.py

@@ -0,0 +1,61 @@
+import argparse
+import sys
+
+from typing import List, Union
+
+from httpie.context import Environment
+from httpie.status import ExitStatus
+from httpie.manager.cli import parser
+from httpie.manager.core import MSG_COMMAND_CONFUSION, program as main_program
+
+
+def is_http_command(args: List[Union[str, bytes]], env: Environment) -> bool:
+    """Check whether http/https parser can parse the arguments."""
+
+    from httpie.cli.definition import parser as http_parser
+    from httpie.manager.cli import COMMANDS
+
+    # If the user already selected a top-level sub-command, never
+    # show the http/https version. E.g httpie plugins pie.dev/post
+    if len(args) >= 1 and args[0] in COMMANDS:
+        return False
+
+    with env.as_silent():
+        try:
+            http_parser.parse_args(env=env, args=args)
+        except (Exception, SystemExit):
+            return False
+        else:
+            return True
+
+
+def main(args: List[Union[str, bytes]] = sys.argv, env: Environment = Environment()) -> ExitStatus:
+    from httpie.core import raw_main
+
+    try:
+        return raw_main(
+            parser=parser,
+            main_program=main_program,
+            args=args,
+            env=env
+        )
+    except argparse.ArgumentError:
+        program_args = args[1:]
+        if is_http_command(program_args, env):
+            env.stderr.write(MSG_COMMAND_CONFUSION.format(args=' '.join(program_args)) + "\n")
+
+        return ExitStatus.ERROR
+
+
+def program():
+    try:
+        exit_status = main()
+    except KeyboardInterrupt:
+        from httpie.status import ExitStatus
+        exit_status = ExitStatus.ERROR_CTRL_C
+
+    return exit_status
+
+
+if __name__ == '__main__':  # pragma: nocover
+    sys.exit(program())

httpie/manager/cli.py

@@ -0,0 +1,93 @@
+from textwrap import dedent
+from httpie.cli.argparser import HTTPieManagerArgumentParser
+
+COMMANDS = {
+    'plugins': {
+        'help': 'Manage HTTPie plugins.',
+        'install': [
+            'Install the given targets from PyPI '
+            'or from a local paths.',
+            {
+                'dest': 'targets',
+                'nargs': '+',
+                'help': 'targets to install'
+            }
+        ],
+        'uninstall': [
+            'Uninstall the given HTTPie plugins.',
+            {
+                'dest': 'targets',
+                'nargs': '+',
+                'help': 'targets to install'
+            }
+        ],
+        'list': [
+            'List all installed HTTPie plugins.'
+        ],
+    },
+}
+
+
+def missing_subcommand(*args) -> str:
+    base = COMMANDS
+    for arg in args:
+        base = base[arg]
+
+    assert isinstance(base, dict)
+    subcommands = ', '.join(map(repr, base.keys()))
+    return f'Please specify one of these: {subcommands}'
+
+
+def generate_subparsers(root, parent_parser, definitions):
+    action_dest = '_'.join(parent_parser.prog.split()[1:] + ['action'])
+    actions = parent_parser.add_subparsers(
+        dest=action_dest
+    )
+    for command, properties in definitions.items():
+        is_subparser = isinstance(properties, dict)
+        descr = properties.pop('help', None) if is_subparser else properties.pop(0)
+        command_parser = actions.add_parser(command, description=descr)
+        command_parser.root = root
+        if is_subparser:
+            generate_subparsers(root, command_parser, properties)
+            continue
+
+        for argument in properties:
+            command_parser.add_argument(**argument)
+
+
+parser = HTTPieManagerArgumentParser(
+    prog='httpie',
+    description=dedent(
+        '''
+        Managing interface for the HTTPie itself. <https://httpie.io/docs#manager>
+
+        Be aware that you might be looking for http/https commands for sending
+        HTTP requests. This command is only available for managing the HTTTPie
+        plugins and the configuration around it.
+        '''
+    ),
+)
+
+parser.add_argument(
+    '--debug',
+    action='store_true',
+    default=False,
+    help='''
+    Prints the exception traceback should one occur, as well as other
+    information useful for debugging HTTPie itself and for reporting bugs.
+
+    '''
+)
+
+parser.add_argument(
+    '--traceback',
+    action='store_true',
+    default=False,
+    help='''
+    Prints the exception traceback should one occur.
+
+    '''
+)
+
+generate_subparsers(parser, parser, COMMANDS)

httpie/manager/core.py

@@ -0,0 +1,33 @@
+import argparse
+
+from httpie.context import Environment
+from httpie.manager.plugins import PluginInstaller
+from httpie.status import ExitStatus
+from httpie.manager.cli import missing_subcommand, parser
+
+MSG_COMMAND_CONFUSION = '''\
+This command is only for managing HTTPie plugins.
+To send a request, please use the http/https commands:
+
+  $ http {args}
+
+  $ https {args}
+'''
+
+# noinspection PyStringFormat
+MSG_NAKED_INVOCATION = f'''\
+{missing_subcommand()}
+
+{MSG_COMMAND_CONFUSION}
+'''.rstrip("\n").format(args='POST pie.dev/post hello=world')
+
+
+def program(args: argparse.Namespace, env: Environment) -> ExitStatus:
+    if args.action is None:
+        parser.error(MSG_NAKED_INVOCATION)
+
+    if args.action == 'plugins':
+        plugins = PluginInstaller(env, debug=args.debug)
+        return plugins.run(args.plugins_action, args)
+
+    return ExitStatus.SUCCESS

httpie/manager/plugins.py

@@ -0,0 +1,188 @@
+import argparse
+import os
+import subprocess
+import sys
+import textwrap
+from collections import defaultdict
+from contextlib import suppress
+from pathlib import Path
+from typing import Optional, List
+
+import importlib_metadata
+
+from httpie.manager.cli import parser, missing_subcommand
+from httpie.compat import get_dist_name
+from httpie.context import Environment
+from httpie.status import ExitStatus
+
+
+class PluginInstaller:
+
+    def __init__(self, env: Environment, debug: bool = False) -> None:
+        self.env = env
+        self.dir = env.config.plugins_dir
+        self.debug = debug
+
+        self.setup_plugins_dir()
+
+    def setup_plugins_dir(self) -> None:
+        try:
+            self.dir.mkdir(
+                exist_ok=True,
+                parents=True
+            )
+        except OSError:
+            self.env.stderr.write(
+                f'Couldn\'t create "{self.dir!s}"'
+                ' directory for plugin installation.'
+                ' Please re-check the permissions for that directory,'
+                ' and if needed, allow write-access.'
+            )
+            raise
+
+    def fail(
+        self,
+        command: str,
+        target: Optional[str] = None,
+        reason: Optional[str] = None
+    ) -> ExitStatus:
+        message = f'Can\'t {command}'
+        if target:
+            message += f' {target!r}'
+        if reason:
+            message += f': {reason}'
+
+        self.env.stderr.write(message + '\n')
+        return ExitStatus.ERROR
+
+    def pip(self, *args, **kwargs) -> subprocess.CompletedProcess:
+        options = {
+            'check': True,
+            'shell': False,
+            'stdout': self.env.stdout,
+            'stderr': subprocess.PIPE,
+        }
+        options.update(kwargs)
+
+        cmd = [sys.executable, '-m', 'pip', *args]
+        return subprocess.run(
+            cmd,
+            **options
+        )
+
+    def install(self, targets: List[str]) -> Optional[ExitStatus]:
+        self.env.stdout.write(f"Installing {', '.join(targets)}...\n")
+        self.env.stdout.flush()
+
+        try:
+            self.pip(
+                'install',
+                f'--prefix={self.dir}',
+                '--no-warn-script-location',
+                *targets,
+            )
+        except subprocess.CalledProcessError as error:
+            reason = None
+            if error.stderr:
+                stderr = error.stderr.decode()
+
+                if self.debug:
+                    self.env.stderr.write('Command failed: ')
+                    self.env.stderr.write(' '.join(error.cmd) + '\n')
+                    self.env.stderr.write(textwrap.indent('  ', stderr))
+
+                last_line = stderr.strip().splitlines()[-1]
+                severity, _, message = last_line.partition(': ')
+                if severity == 'ERROR':
+                    reason = message
+
+            return self.fail('install', ', '.join(targets), reason)
+
+    def _uninstall(self, target: str) -> Optional[ExitStatus]:
+        try:
+            distribution = importlib_metadata.distribution(target)
+        except importlib_metadata.PackageNotFoundError:
+            return self.fail('uninstall', target, 'package is not installed')
+
+        base_dir = Path(distribution.locate_file('.')).resolve()
+        if self.dir not in base_dir.parents:
+            # If the package is installed somewhere else (e.g on the site packages
+            # of the real python interpreter), than that means this package is not
+            # installed through us.
+            return self.fail('uninstall', target,
+                             'package is not installed through httpie plugins'
+                             ' interface')
+
+        files = distribution.files
+        if files is None:
+            return self.fail('uninstall', target, 'couldn\'t locate the package')
+
+        # TODO: Consider handling failures here (e.g if it fails,
+        # just rever the operation and leave the site-packages
+        # in a proper shape).
+        for file in files:
+            with suppress(FileNotFoundError):
+                os.unlink(distribution.locate_file(file))
+
+        metadata_path = getattr(distribution, '_path', None)
+        if (
+            metadata_path
+            and metadata_path.exists()
+            and not any(metadata_path.iterdir())
+        ):
+            metadata_path.rmdir()
+
+        self.env.stdout.write(f'Successfully uninstalled {target}\n')
+
+    def uninstall(self, targets: List[str]) -> ExitStatus:
+        # Unfortunately uninstall doesn't work with custom pip schemes. See:
+        # - https://github.com/pypa/pip/issues/5595
+        # - https://github.com/pypa/pip/issues/4575
+        # so we have to implement our own uninstalling logic. Which works
+        # on top of the importlib_metadata.
+
+        exit_code = ExitStatus.SUCCESS
+        for target in targets:
+            exit_code |= self._uninstall(target) or ExitStatus.SUCCESS
+        return ExitStatus(exit_code)
+
+    def list(self) -> None:
+        from httpie.plugins.registry import plugin_manager
+
+        known_plugins = defaultdict(list)
+
+        for entry_point in plugin_manager.iter_entry_points(self.dir):
+            ep_info = (entry_point.group, entry_point.name)
+            ep_name = get_dist_name(entry_point) or entry_point.module
+            known_plugins[ep_name].append(ep_info)
+
+        for plugin, entry_points in known_plugins.items():
+            self.env.stdout.write(plugin)
+
+            version = importlib_metadata.version(plugin)
+            if version is not None:
+                self.env.stdout.write(f' ({version})')
+            self.env.stdout.write('\n')
+
+            for group, entry_point in sorted(entry_points):
+                self.env.stdout.write(f'  {entry_point} ({group})\n')
+
+    def run(
+        self,
+        action: Optional[str],
+        args: argparse.Namespace,
+    ) -> ExitStatus:
+        from httpie.plugins.manager import enable_plugins
+
+        if action is None:
+            parser.error(missing_subcommand('plugins'))
+
+        with enable_plugins(self.dir):
+            if action == 'install':
+                status = self.install(args.targets)
+            elif action == 'uninstall':
+                status = self.uninstall(args.targets)
+            elif action == 'list':
+                status = self.list()
+
+        return status or ExitStatus.SUCCESS

httpie/plugins/manager.py

@@ -1,24 +1,55 @@
+import sys
+import os
+
 from itertools import groupby
 from operator import attrgetter
-from typing import Dict, List, Type
+from typing import Dict, List, Type, Iterator, TypeVar, Optional, ContextManager
+from pathlib import Path
+from contextlib import contextmanager
 
-from pkg_resources import iter_entry_points
+from ..compat import importlib_metadata, find_entry_points, get_dist_name
 
-from ..utils import repr_dict
+from ..utils import repr_dict, as_site
-from . import AuthPlugin, ConverterPlugin, FormatterPlugin
+from . import AuthPlugin, ConverterPlugin, FormatterPlugin, TransportPlugin
-from .base import BasePlugin, TransportPlugin
+from .base import BasePlugin
 
 
-ENTRY_POINT_NAMES = [
+ENTRY_POINT_CLASSES = {
-    'httpie.plugins.auth.v1',
+    'httpie.plugins.auth.v1': AuthPlugin,
-    'httpie.plugins.formatter.v1',
+    'httpie.plugins.converter.v1': ConverterPlugin,
-    'httpie.plugins.converter.v1',
+    'httpie.plugins.formatter.v1': FormatterPlugin,
-    'httpie.plugins.transport.v1',
+    'httpie.plugins.transport.v1': TransportPlugin
-]
+}
+ENTRY_POINT_NAMES = list(ENTRY_POINT_CLASSES.keys())
+
+
+@contextmanager
+def _load_directory(plugins_dir: Path) -> Iterator[None]:
+    plugins_path = os.fspath(plugins_dir)
+    sys.path.insert(0, plugins_path)
+    try:
+        yield
+    finally:
+        sys.path.remove(plugins_path)
+
+
+T = TypeVar("T")
+
+
+@contextmanager
+def nullcontext(obj: Optional[T] = None) -> Iterator[Optional[T]]:
+    # A naive replacement of the nullcontext() for 3.6
+    yield obj
+
+
+def enable_plugins(plugins_dir: Optional[Path]) -> ContextManager[None]:
+    if plugins_dir is None:
+        return nullcontext()
+    else:
+        return _load_directory(as_site(plugins_dir))
 
 
 class PluginManager(list):
-
     def register(self, *plugins: Type[BasePlugin]):
         for plugin in plugins:
             self.append(plugin)
@@ -29,11 +60,17 @@ class PluginManager(list):
     def filter(self, by_type=Type[BasePlugin]):
         return [plugin for plugin in self if issubclass(plugin, by_type)]
 
-    def load_installed_plugins(self):
+    def iter_entry_points(self, directory: Optional[Path] = None):
+        with enable_plugins(directory):
+            eps = importlib_metadata.entry_points()
+
             for entry_point_name in ENTRY_POINT_NAMES:
-            for entry_point in iter_entry_points(entry_point_name):
+                yield from find_entry_points(eps, group=entry_point_name)
+
+    def load_installed_plugins(self, directory: Optional[Path] = None):
+        for entry_point in self.iter_entry_points(directory):
             plugin = entry_point.load()
-                plugin.package_name = entry_point.dist.key
+            plugin.package_name = get_dist_name(entry_point)
             self.register(entry_point.load())
 
     # Auth

httpie/utils.py

@@ -3,8 +3,11 @@ import mimetypes
 import re
 import sys
 import time
+import sysconfig
+
 from collections import OrderedDict
 from http.cookiejar import parse_ns_headers
+from pathlib import Path
 from pprint import pformat
 from typing import Any, List, Optional, Tuple
 
@@ -207,3 +210,11 @@ def parse_content_type_header(header):
                 value = param[index_of_equals + 1:].strip(items_to_strip)
             params_dict[key.lower()] = value
     return content_type, params_dict
+
+
+def as_site(path: Path) -> Path:
+    site_packages_path = sysconfig.get_path(
+        'purelib',
+        vars={'base': str(path)}
+    )
+    return Path(site_packages_path)

setup.py

@@ -35,6 +35,7 @@ install_requires = [
     'requests-toolbelt>=0.9.1',
     'multidict>=4.7.0',
     'setuptools',
+    'importlib-metadata>=1.4.0',
 ]
 install_requires_win_only = [
     'colorama>=0.2.4',
@@ -80,6 +81,7 @@ setup(
         'console_scripts': [
             'http = httpie.__main__:main',
             'https = httpie.__main__:main',
+            'httpie = httpie.manager.__main__:main',
         ],
     },
     python_requires='>=3.6',

tests/conftest.py

@@ -5,6 +5,13 @@ import pytest
 from pytest_httpbin import certs
 
 from .utils import HTTPBIN_WITH_CHUNKED_SUPPORT_DOMAIN, HTTPBIN_WITH_CHUNKED_SUPPORT
+from .utils.plugins_cli import ( # noqa
+    dummy_plugin,
+    dummy_plugins,
+    httpie_plugins,
+    httpie_plugins_success,
+    interface,
+)
 from .utils.http_server import http_server # noqa
 
 

tests/test_plugins_cli.py

@@ -0,0 +1,132 @@
+import pytest
+
+from httpie.status import ExitStatus
+from tests.utils import httpie
+from tests.utils.plugins_cli import parse_listing
+
+
+def test_plugins_installation(httpie_plugins_success, interface, dummy_plugin):
+    lines = httpie_plugins_success('install', dummy_plugin.path)
+    assert lines[0].startswith(
+        f'Installing {dummy_plugin.path}'
+    )
+    assert f'Successfully installed {dummy_plugin.name}-{dummy_plugin.version}' in lines
+    assert interface.is_installed(dummy_plugin.name)
+
+
+def test_plugins_listing(httpie_plugins_success, interface, dummy_plugin):
+    httpie_plugins_success('install', dummy_plugin.path)
+    data = parse_listing(httpie_plugins_success('list'))
+
+    assert data == {
+        dummy_plugin.name: dummy_plugin.dump()
+    }
+
+
+def test_plugins_listing_multiple(interface, httpie_plugins_success, dummy_plugins):
+    paths = [plugin.path for plugin in dummy_plugins]
+    httpie_plugins_success('install', *paths)
+    data = parse_listing(httpie_plugins_success('list'))
+
+    assert data == {
+        plugin.name: plugin.dump()
+        for plugin in dummy_plugins
+    }
+
+
+def test_plugins_uninstall(interface, httpie_plugins_success, dummy_plugin):
+    httpie_plugins_success('install', dummy_plugin.path)
+    httpie_plugins_success('uninstall', dummy_plugin.name)
+    assert not interface.is_installed(dummy_plugin.name)
+
+
+def test_plugins_listing_after_uninstall(interface, httpie_plugins_success, dummy_plugin):
+    httpie_plugins_success('install', dummy_plugin.path)
+    httpie_plugins_success('uninstall', dummy_plugin.name)
+
+    data = parse_listing(httpie_plugins_success('list'))
+    assert len(data) == 0
+
+
+def test_plugins_uninstall_specific(interface, httpie_plugins_success):
+    new_plugin_1 = interface.make_dummy_plugin()
+    new_plugin_2 = interface.make_dummy_plugin()
+    target_plugin = interface.make_dummy_plugin()
+
+    httpie_plugins_success('install', new_plugin_1.path, new_plugin_2.path, target_plugin.path)
+    httpie_plugins_success('uninstall', target_plugin.name)
+
+    assert interface.is_installed(new_plugin_1.name)
+    assert interface.is_installed(new_plugin_2.name)
+    assert not interface.is_installed(target_plugin.name)
+
+
+def test_plugins_installation_failed(httpie_plugins, interface):
+    plugin = interface.make_dummy_plugin(build=False)
+    result = httpie_plugins('install', plugin.path)
+
+    assert result.exit_status == ExitStatus.ERROR
+    assert result.stderr.splitlines()[-1].strip().startswith("Can't install")
+
+
+def test_plugins_uninstall_non_existent(httpie_plugins, interface):
+    plugin = interface.make_dummy_plugin(build=False)
+    result = httpie_plugins('uninstall', plugin.name)
+
+    assert result.exit_status == ExitStatus.ERROR
+    assert (
+        result.stderr.splitlines()[-1].strip()
+        == f"Can't uninstall '{plugin.name}': package is not installed"
+    )
+
+
+def test_plugins_double_uninstall(httpie_plugins, httpie_plugins_success, dummy_plugin):
+    httpie_plugins_success("install", dummy_plugin.path)
+    httpie_plugins_success("uninstall", dummy_plugin.name)
+
+    result = httpie_plugins("uninstall", dummy_plugin.name)
+
+    assert result.exit_status == ExitStatus.ERROR
+    assert (
+        result.stderr.splitlines()[-1].strip()
+        == f"Can't uninstall '{dummy_plugin.name}': package is not installed"
+    )
+
+
+def test_plugins_cli_error_message_without_args():
+    # No arguments
+    result = httpie(no_debug=True)
+    assert result.exit_status == ExitStatus.ERROR
+    assert 'usage: ' in result.stderr
+    assert 'specify one of these' in result.stderr
+    assert 'please use the http/https commands:' in result.stderr
+
+
+@pytest.mark.parametrize(
+    'example', [
+        'pie.dev/get',
+        'DELETE localhost:8000/delete',
+        'POST pie.dev/post header:value a=b header_2:value x:=1'
+    ]
+)
+def test_plugins_cli_error_messages_with_example(example):
+    result = httpie(*example.split(), no_debug=True)
+    assert result.exit_status == ExitStatus.ERROR
+    assert 'usage: ' in result.stderr
+    assert f'http {example}' in result.stderr
+    assert f'https {example}' in result.stderr
+
+
+@pytest.mark.parametrize(
+    'example', [
+        'plugins unknown',
+        'plugins unknown.com A:B c=d',
+        'unknown.com UNPARSABLE????SYNTAX',
+    ]
+)
+def test_plugins_cli_error_messages_invalid_example(example):
+    result = httpie(*example.split(), no_debug=True)
+    assert result.exit_status == ExitStatus.ERROR
+    assert 'usage: ' in result.stderr
+    assert f'http {example}' not in result.stderr
+    assert f'https {example}' not in result.stderr

tests/test_ssl.py

@@ -6,7 +6,7 @@ import pytest_httpbin.certs
 import requests.exceptions
 import urllib3
 
-from httpie.ssl import AVAILABLE_SSL_VERSION_ARG_MAPPING, DEFAULT_SSL_CIPHERS
+from httpie.ssl_ import AVAILABLE_SSL_VERSION_ARG_MAPPING, DEFAULT_SSL_CIPHERS
 from httpie.status import ExitStatus
 
 from .utils import HTTP_OK, TESTS_ROOT, http

tests/utils/__init__.py

@@ -7,12 +7,14 @@ import json
 import tempfile
 from io import BytesIO
 from pathlib import Path
-from typing import Optional, Union, List
+from typing import Any, Optional, Union, List, Iterable
+
+import httpie.core as core
+import httpie.manager.__main__ as manager
 
 from httpie.status import ExitStatus
 from httpie.config import Config
 from httpie.context import Environment
-from httpie.core import main
 
 
 # pytest-httpbin currently does not support chunked requests:
@@ -58,10 +60,10 @@ class MockEnvironment(Environment):
     stdout_isatty = True
     is_windows = False
 
-    def __init__(self, create_temp_config_dir=True, **kwargs):
+    def __init__(self, create_temp_config_dir=True, *, stdout_mode='b', **kwargs):
         if 'stdout' not in kwargs:
             kwargs['stdout'] = tempfile.TemporaryFile(
-                mode='w+b',
+                mode=f'w+{stdout_mode}',
                 prefix='httpie_stdout'
             )
         if 'stderr' not in kwargs:
@@ -177,6 +179,46 @@ class ExitStatusError(Exception):
     pass
 
 
+def normalize_args(args: Iterable[Any]) -> List[str]:
+    return [str(arg) for arg in args]
+
+
+def httpie(
+    *args,
+    **kwargs
+) -> StrCLIResponse:
+    """
+    Run HTTPie manager command with the given
+    args/kwargs, and capture stderr/out and exit
+    status.
+    """
+
+    env = kwargs.setdefault('env', MockEnvironment())
+    cli_args = ['httpie']
+    if not kwargs.pop('no_debug', False):
+        cli_args.append('--debug')
+    cli_args += normalize_args(args)
+    exit_status = manager.main(
+        args=cli_args,
+        **kwargs
+    )
+
+    env.stdout.seek(0)
+    env.stderr.seek(0)
+    try:
+        response = StrCLIResponse(env.stdout.read())
+        response.stderr = env.stderr.read()
+        response.exit_status = exit_status
+        response.args = cli_args
+    finally:
+        env.stdout.truncate(0)
+        env.stderr.truncate(0)
+        env.stdout.seek(0)
+        env.stderr.seek(0)
+
+    return response
+
+
 def http(
     *args,
     program_name='http',
@@ -254,7 +296,7 @@ def http(
 
     try:
         try:
-            exit_status = main(args=complete_args, **kwargs)
+            exit_status = core.main(args=complete_args, **kwargs)
             if '--download' in args:
                 # Let the progress reporter thread finish.
                 time.sleep(.5)

tests/utils/plugins_cli.py

@@ -0,0 +1,222 @@
+import secrets
+import site
+import sys
+import textwrap
+
+import pytest
+
+from collections import defaultdict
+from dataclasses import dataclass, field, asdict
+from pathlib import Path
+from typing import Any, List, Dict, Tuple
+from unittest.mock import patch
+
+from httpie.context import Environment
+from httpie.compat import importlib_metadata
+from httpie.status import ExitStatus
+from httpie.plugins.manager import (
+    enable_plugins,
+    ENTRY_POINT_CLASSES as CLASSES,
+)
+
+
+def make_name() -> str:
+    return 'httpie-' + secrets.token_hex(4)
+
+
+@dataclass
+class EntryPoint:
+    name: str
+    group: str
+
+    def dump(self) -> Dict[str, str]:
+        return asdict(self)
+
+
+@dataclass
+class Plugin:
+    interface: 'Interface'
+
+    name: str = field(default_factory=make_name)
+    version: str = '1.0.0'
+    entry_points: List[EntryPoint] = field(default_factory=list)
+
+    def build(self) -> None:
+        '''
+        Create an installable dummy plugin at the given path.
+
+        It will create a setup.py with the specified entry points,
+        as well as dummy classes in a python module to imitate
+        real plugins.
+        '''
+
+        groups = defaultdict(list)
+        for entry_point in self.entry_points:
+            groups[entry_point.group].append(entry_point.name)
+
+        setup_eps = {
+            group: [
+                f'{name} = {self.import_name}:{name.title()}'
+                for name in names
+            ]
+            for group, names in groups.items()
+        }
+
+        self.path.mkdir(parents=True, exist_ok=True)
+        with open(self.path / 'setup.py', 'w') as stream:
+            stream.write(textwrap.dedent(f'''
+            from setuptools import setup
+
+            setup(
+                name='{self.name}',
+                version='{self.version}',
+                py_modules=['{self.import_name}'],
+                entry_points={setup_eps!r},
+                install_requires=['httpie']
+            )
+            '''))
+
+        with open(self.path / (self.import_name + '.py'), 'w') as stream:
+            stream.write('from httpie.plugins import *\n')
+            stream.writelines(
+                f'class {name.title()}({CLASSES[group].__name__}): ...\n'
+                for group, names in groups.items()
+                for name in names
+            )
+
+    def dump(self) -> Dict[str, Any]:
+        return {
+            'version': self.version,
+            'entry_points': [
+                entry_point.dump()
+                for entry_point in self.entry_points
+            ]
+        }
+
+    @property
+    def path(self) -> Path:
+        return self.interface.path / self.name
+
+    @property
+    def import_name(self) -> str:
+        return self.name.replace('-', '_')
+
+
+@dataclass
+class Interface:
+    path: Path
+    environment: Environment
+
+    def get_plugin(self, target: str) -> importlib_metadata.Distribution:
+        with enable_plugins(self.environment.config.plugins_dir):
+            return importlib_metadata.distribution(target)
+
+    def is_installed(self, target: str) -> bool:
+        try:
+            self.get_plugin(target)
+        except ModuleNotFoundError:
+            return False
+        else:
+            return True
+
+    def make_dummy_plugin(self, build=True, **kwargs) -> Plugin:
+        kwargs.setdefault('entry_points', [EntryPoint('test', 'httpie.plugins.auth.v1')])
+
+        plugin = Plugin(self, **kwargs)
+        if build:
+            plugin.build()
+        return plugin
+
+
+def parse_listing(lines: List[str]) -> Dict[str, Any]:
+    plugins = {}
+    current_plugin = None
+
+    def parse_entry_point(line: str) -> Tuple[str, str]:
+        entry_point, raw_group = line.strip().split()
+        return entry_point, raw_group[1:-1]
+
+    def parse_plugin(line: str) -> Tuple[str, str]:
+        plugin, raw_version = line.strip().split()
+        return plugin, raw_version[1:-1]
+
+    for line in lines:
+        if not line.strip():
+            continue
+
+        if line[0].isspace():
+            # <indent> $entry_point ($group)
+            assert current_plugin is not None
+            entry_point, group = parse_entry_point(line)
+            plugins[current_plugin]['entry_points'].append({
+                'name': entry_point,
+                'group': group
+            })
+        else:
+            # $plugin ($version)
+            current_plugin, version = parse_plugin(line)
+            plugins[current_plugin] = {
+                'version': version,
+                'entry_points': []
+            }
+
+    return plugins
+
+
+@pytest.fixture(scope='function')
+def interface(tmp_path):
+    from tests.utils import MockEnvironment
+
+    return Interface(
+        path=tmp_path / 'interface',
+        environment=MockEnvironment(stdout_mode='t')
+    )
+
+
+@pytest.fixture(scope='function')
+def dummy_plugin(interface):
+    return interface.make_dummy_plugin()
+
+
+@pytest.fixture(scope='function')
+def dummy_plugins(interface):
+    # Multiple plugins with different configurations
+    return [
+        interface.make_dummy_plugin(),
+        interface.make_dummy_plugin(
+            version='3.2.0'
+        ),
+        interface.make_dummy_plugin(
+            entry_points=[
+                EntryPoint('test_1', 'httpie.plugins.converter.v1'),
+                EntryPoint('test_2', 'httpie.plugins.formatter.v1')
+            ]
+        ),
+    ]
+
+
+@pytest.fixture
+def httpie_plugins(interface):
+    from tests.utils import httpie
+    from httpie.plugins.registry import plugin_manager
+
+    def runner(*args):
+        # Prevent installed plugins from showing up.
+        original_plugins = plugin_manager.copy()
+        clean_sys_path = set(sys.path).difference(site.getsitepackages())
+        with patch('sys.path', list(clean_sys_path)):
+            response = httpie('plugins', *args, env=interface.environment)
+        plugin_manager.clear()
+        plugin_manager.extend(original_plugins)
+        return response
+
+    return runner
+
+
+@pytest.fixture
+def httpie_plugins_success(httpie_plugins):
+    def runner(*args):
+        response = httpie_plugins(*args)
+        assert response.exit_status == ExitStatus.SUCCESS
+        return response.splitlines()
+    return runner