Converted built-in formatters to formatter plugins.

Still work in progress and the API should be considered private for now.
This commit is contained in:
Jakub Roztocil 2014-04-28 23:33:30 +02:00
parent 858555abb5
commit e4c68063b9
25 changed files with 311 additions and 249 deletions

View File

@ -3,7 +3,7 @@ HTTPie - a CLI, cURL-like tool for humans.
"""
__author__ = 'Jakub Roztocil'
__version__ = '0.9.0dev'
__version__ = '0.9.0-dev'
__licence__ = 'BSD'

View File

@ -8,17 +8,17 @@ from textwrap import dedent, wrap
from argparse import (RawDescriptionHelpFormatter, FileType,
OPTIONAL, ZERO_OR_MORE, SUPPRESS)
from . import __doc__
from . import __version__
from .plugins.builtin import BuiltinAuthPlugin
from .plugins import plugin_manager
from .sessions import DEFAULT_SESSIONS_DIR
from .output.processors.colors import AVAILABLE_STYLES, DEFAULT_STYLE
from .input import (Parser, AuthCredentialsArgType, KeyValueArgType,
from httpie import __doc__, __version__
from httpie.plugins.builtin import BuiltinAuthPlugin
from httpie.plugins import plugin_manager
from httpie.sessions import DEFAULT_SESSIONS_DIR
from httpie.output.formatters.colors import AVAILABLE_STYLES, DEFAULT_STYLE
from httpie.input import (Parser, AuthCredentialsArgType, KeyValueArgType,
SEP_PROXY, SEP_CREDENTIALS, SEP_GROUP_ALL_ITEMS,
OUT_REQ_HEAD, OUT_REQ_BODY, OUT_RESP_HEAD,
OUT_RESP_BODY, OUTPUT_OPTIONS, OUTPUT_OPTIONS_DEFAULT,
PRETTY_MAP, PRETTY_STDOUT_TTY_ONLY, SessionNameValidator,
OUT_RESP_BODY, OUTPUT_OPTIONS,
OUTPUT_OPTIONS_DEFAULT, PRETTY_MAP,
PRETTY_STDOUT_TTY_ONLY, SessionNameValidator,
readable_file_arg)
@ -60,7 +60,7 @@ parser = Parser(
#######################################################################
positional = parser.add_argument_group(
title='Positional arguments',
title='Positional Arguments',
description=dedent("""
These arguments come after any flags and in the order they are listed here.
Only URL is required.
@ -147,7 +147,7 @@ positional.add_argument(
#######################################################################
content_type = parser.add_argument_group(
title='Predefined content types',
title='Predefined Content Types',
description=None
)
@ -179,7 +179,7 @@ content_type.add_argument(
# Output processing
#######################################################################
output_processing = parser.add_argument_group(title='Output processing')
output_processing = parser.add_argument_group(title='Output Processing')
output_processing.add_argument(
'--pretty',
@ -208,14 +208,12 @@ output_processing.add_argument(
environment variable is set to "xterm-256color" or similar
(e.g., via `export TERM=xterm-256color' in your ~/.bashrc).
"""
.format(
""".format(
default=DEFAULT_STYLE,
available='\n'.join(
'{0: >20}'.format(line.strip())
for line in
wrap(' '.join(sorted(AVAILABLE_STYLES)), 60)
),
'{0}{1}'.format(8*' ', line.strip())
for line in wrap(', '.join(sorted(AVAILABLE_STYLES)), 60)
).rstrip(),
)
)
@ -223,7 +221,7 @@ output_processing.add_argument(
#######################################################################
# Output options
#######################################################################
output_options = parser.add_argument_group(title='Output options')
output_options = parser.add_argument_group(title='Output Options')
output_options.add_argument(
'--print', '-p',

View File

@ -4,10 +4,10 @@ from pprint import pformat
import requests
from . import sessions
from . import __version__
from .compat import str
from .plugins import plugin_manager
from httpie import sessions
from httpie import __version__
from httpie.compat import str
from httpie.plugins import plugin_manager
FORM = 'application/x-www-form-urlencoded; charset=utf-8'

View File

@ -1,15 +1,10 @@
"""
Python 2/3 compatibility.
Python 2.6, 2.7, and 3.x compatibility.
"""
# Borrow these from requests:
#noinspection PyUnresolvedReferences
from requests.compat import (
is_windows,
bytes,
str,
is_py3,
is_py26,
)
from requests.compat import is_windows, bytes, str, is_py3, is_py26
try:
#noinspection PyUnresolvedReferences,PyCompatibility
@ -25,7 +20,6 @@ except ImportError:
#noinspection PyCompatibility
from urllib2 import urlopen
try:
from collections import OrderedDict
except ImportError:

View File

@ -2,8 +2,8 @@ import os
import json
import errno
from . import __version__
from .compat import is_windows
from httpie import __version__
from httpie.compat import is_windows
DEFAULT_CONFIG_DIR = os.environ.get(

View File

@ -72,10 +72,9 @@ def main(args=sys.argv[1:], env=Environment()):
Return exit status code.
"""
args = decode_args(args, env.stdin_encoding)
from httpie.cli import parser
plugin_manager.load_installed_plugins()
from .cli import parser
if env.config.default_options:
args = env.config.default_options + args

View File

@ -10,14 +10,13 @@ import getpass
from io import BytesIO
#noinspection PyCompatibility
from argparse import ArgumentParser, ArgumentTypeError, ArgumentError
from .compat import OrderedDict
# TODO: Use MultiDict for headers once added to `requests`.
# https://github.com/jakubroztocil/httpie/issues/130
from requests.structures import CaseInsensitiveDict
from .compat import urlsplit, str
from .sessions import VALID_SESSION_NAME_PATTERN
from httpie.compat import OrderedDict, urlsplit, str
from httpie.sessions import VALID_SESSION_NAME_PATTERN
HTTP_POST = 'POST'

View File

@ -1,4 +1,4 @@
from .compat import urlsplit, str
from httpie.compat import urlsplit, str
class HTTPMessage(object):

View File

View File

@ -8,7 +8,7 @@ from pygments.formatters.terminal256 import Terminal256Formatter
from pygments.util import ClassNotFound
from httpie.compat import is_windows
from .base import BaseProcessor
from httpie.plugins import FormatterPlugin
# Colors on Windows via colorama don't look that
@ -18,6 +18,52 @@ AVAILABLE_STYLES.add('solarized')
DEFAULT_STYLE = 'solarized' if not is_windows else 'fruity'
class ColorFormatter(FormatterPlugin):
"""
Colorize using Pygments
This processor that applies syntax highlighting to the headers,
and also to the body if its content type is recognized.
"""
group_name = 'colors'
def __init__(self, env, color_scheme=DEFAULT_STYLE, **kwargs):
super(ColorFormatter, self).__init__(**kwargs)
if not env.colors:
self.enabled = False
return
# Cache to speed things up when we process streamed body by line.
self.lexer_cache = {}
try:
style_class = pygments.styles.get_style_by_name(color_scheme)
except ClassNotFound:
style_class = Solarized256Style
if env.is_windows or env.colors == 256:
fmt_class = Terminal256Formatter
else:
fmt_class = TerminalFormatter
self.formatter = fmt_class(style=style_class)
def format_headers(self, headers):
return pygments.highlight(headers, HTTPLexer(), self.formatter).strip()
def format_body(self, body, mime):
lexer = self.get_lexer(mime)
if lexer:
body = pygments.highlight(body, lexer, self.formatter)
return body.strip()
def get_lexer(self, mime):
if mime in self.lexer_cache:
return self.lexer_cache[mime]
self.lexer_cache[mime] = get_lexer(mime)
return self.lexer_cache[mime]
def get_lexer(mime):
mime_types, lexer_names = [mime], []
type_, subtype = mime.split('/')
@ -46,52 +92,6 @@ def get_lexer(mime):
return lexer
class PygmentsProcessor(BaseProcessor):
"""
Colorize using Pygments
This processor that applies syntax highlighting to the headers,
and also to the body if its content type is recognized.
"""
def __init__(self, *args, **kwargs):
super(PygmentsProcessor, self).__init__(*args, **kwargs)
if not self.env.colors:
self.enabled = False
return
# Cache to speed things up when we process streamed body by line.
self.lexers_by_type = {}
try:
style = pygments.styles.get_style_by_name(
self.kwargs.get('pygments_style', DEFAULT_STYLE))
except ClassNotFound:
style = Solarized256Style
if self.env.is_windows or self.env.colors == 256:
fmt_class = Terminal256Formatter
else:
fmt_class = TerminalFormatter
self.formatter = fmt_class(style=style)
def process_headers(self, headers):
return pygments.highlight(headers, HTTPLexer(), self.formatter).strip()
def process_body(self, body, mime):
lexer = self.get_lexer(mime)
if lexer:
body = pygments.highlight(body, lexer, self.formatter)
return body.strip()
def get_lexer(self, mime):
if mime in self.lexers_by_type:
return self.lexers_by_type[mime]
self.lexers_by_type[mime] = get_lexer(mime)
return self.lexers_by_type[mime]
class HTTPLexer(pygments.lexer.RegexLexer):
"""Simplified HTTP lexer for Pygments.

View File

@ -1,9 +1,9 @@
from .base import BaseProcessor
from httpie.plugins import FormatterPlugin
class HeadersProcessor(BaseProcessor):
class HeadersFormatter(FormatterPlugin):
def process_headers(self, headers):
def format_headers(self, headers):
"""
Sorts headers by name while retaining relative
order of multiple headers with the same name.

View File

@ -1,12 +1,15 @@
from __future__ import absolute_import
import json
from .base import BaseProcessor, DEFAULT_INDENT
from httpie.plugins import FormatterPlugin
class JSONProcessor(BaseProcessor):
DEFAULT_INDENT = 4
def process_body(self, body, mime):
class JSONFormatter(FormatterPlugin):
def format_body(self, body, mime):
if 'json' in mime:
try:
obj = json.loads(body)

View File

@ -2,13 +2,16 @@ from __future__ import absolute_import
import re
from xml.etree import ElementTree
from .base import BaseProcessor, DEFAULT_INDENT
from httpie.plugins import FormatterPlugin
DECLARATION_RE = re.compile('<\?xml[^\n]+?\?>', flags=re.I)
DOCTYPE_RE = re.compile('<!DOCTYPE[^\n]+?>', flags=re.I)
DEFAULT_INDENT = 4
def indent(elem, indent_text=' ' * DEFAULT_INDENT):
"""
In-place prettyprint formatter
@ -33,10 +36,10 @@ def indent(elem, indent_text=' ' * DEFAULT_INDENT):
return _indent(elem)
class XMLProcessor(BaseProcessor):
class XMLFormatter(FormatterPlugin):
# TODO: tests
def process_body(self, body, mime):
def format_body(self, body, mime):
if 'xml' in mime:
# FIXME: orig NS names get forgotten during the conversion, etc.
try:

View File

@ -0,0 +1,50 @@
import re
from httpie.plugins import plugin_manager
from httpie.context import Environment
MIME_RE = re.compile(r'^[^/]+/[^/]+$')
def is_valid_mime(mime):
return mime and MIME_RE.match(mime)
class Conversion(object):
def get_converter(self, mime):
if is_valid_mime(mime):
for converter_class in plugin_manager.get_converters():
if converter_class.supports(mime):
return converter_class(mime)
class Formatting(object):
"""A delegate class that invokes the actual processors."""
def __init__(self, groups, env=Environment(), **kwargs):
"""
:param groups: names of processor groups to be applied
:param env: Environment
:param kwargs: additional keyword arguments for processors
"""
available_plugins = plugin_manager.get_formatters_grouped()
self.enabled_plugins = []
for group in groups:
for cls in available_plugins[group]:
p = cls(env=env, **kwargs)
if p.enabled:
self.enabled_plugins.append(p)
def format_headers(self, headers):
for p in self.enabled_plugins:
headers = p.format_headers(headers)
return headers
def format_body(self, content, mime):
if is_valid_mime(mime):
for p in self.enabled_plugins:
content = p.format_body(content, mime)
return content

View File

@ -1,44 +0,0 @@
from httpie.context import Environment
from .headers import HeadersProcessor
from .json import JSONProcessor
from .xml import XMLProcessor
from .colors import PygmentsProcessor
class ProcessorManager(object):
"""A delegate class that invokes the actual processors."""
available = {
'format': [
HeadersProcessor,
JSONProcessor,
XMLProcessor
],
'colors': [
PygmentsProcessor
]
}
def __init__(self, groups, env=Environment(), **kwargs):
"""
:param groups: names of processor groups to be applied
:param env: Environment
:param kwargs: additional keyword arguments for processors
"""
self.enabled = []
for group in groups:
for cls in self.available[group]:
p = cls(env, **kwargs)
if p.enabled:
self.enabled.append(p)
def process_headers(self, headers):
for p in self.enabled:
headers = p.process_headers(headers)
return headers
def process_body(self, body, mime):
for p in self.enabled:
body = p.process_body(body, mime)
return body

View File

@ -1,37 +0,0 @@
from httpie.context import Environment
# The default number of spaces to indent when pretty printing
DEFAULT_INDENT = 4
class BaseProcessor(object):
"""Base output processor class."""
def __init__(self, env=Environment(), **kwargs):
"""
:param env: an class:`Environment` instance
:param kwargs: additional keyword argument that some
processor might require.
"""
self.enabled = True
self.env = env
self.kwargs = kwargs
def process_headers(self, headers):
"""Return processed `headers`
:param headers: The headers as text.
"""
return headers
def process_body(self, content, mime):
"""Return processed `content`.
:param content: The body content as text
:param mime: E.g., 'application/atom+xml'.
"""
return content

View File

@ -1,11 +1,12 @@
from itertools import chain
from functools import partial
from httpie.compat import str
from httpie.context import Environment
from httpie.models import HTTPRequest, HTTPResponse
from httpie.input import (OUT_REQ_BODY, OUT_REQ_HEAD,
OUT_RESP_HEAD, OUT_RESP_BODY)
from httpie.output.processors import ProcessorManager
from httpie.output.processing import Formatting, Conversion
BINARY_SUPPRESSED_NOTICE = (
@ -59,7 +60,6 @@ def build_output_stream(args, env, request, response):
exchange each of which yields `bytes` chunks.
"""
req_h = OUT_REQ_HEAD in args.output_options
req_b = OUT_REQ_BODY in args.output_options
resp_h = OUT_RESP_HEAD in args.output_options
@ -111,11 +111,9 @@ def get_stream_type(env, args):
Stream = partial(
PrettyStream if args.stream else BufferedPrettyStream,
env=env,
processor=ProcessorManager(
env=env,
groups=args.prettify,
pygments_style=args.style
),
conversion=Conversion(),
formatting=Formatting(env=env, groups=args.prettify,
color_scheme=args.style),
)
else:
Stream = partial(EncodedStream, env=env)
@ -140,23 +138,23 @@ class BaseStream(object):
self.with_body = with_body
self.on_body_chunk_downloaded = on_body_chunk_downloaded
def _get_headers(self):
def get_headers(self):
"""Return the headers' bytes."""
return self.msg.headers.encode('utf8')
def _iter_body(self):
def iter_body(self):
"""Return an iterator over the message body."""
raise NotImplementedError()
def __iter__(self):
"""Return an iterator over `self.msg`."""
if self.with_headers:
yield self._get_headers()
yield self.get_headers()
yield b'\r\n\r\n'
if self.with_body:
try:
for chunk in self._iter_body():
for chunk in self.iter_body():
yield chunk
if self.on_body_chunk_downloaded:
self.on_body_chunk_downloaded(chunk)
@ -176,7 +174,7 @@ class RawStream(BaseStream):
super(RawStream, self).__init__(**kwargs)
self.chunk_size = chunk_size
def _iter_body(self):
def iter_body(self):
return self.msg.iter_body(self.chunk_size)
@ -204,7 +202,7 @@ class EncodedStream(BaseStream):
# Default to utf8 when unsure.
self.output_encoding = output_encoding or 'utf8'
def _iter_body(self):
def iter_body(self):
for line, lf in self.msg.iter_lines(self.CHUNK_SIZE):
@ -226,25 +224,44 @@ class PrettyStream(EncodedStream):
CHUNK_SIZE = 1
def __init__(self, processor, **kwargs):
def __init__(self, conversion, formatting, **kwargs):
super(PrettyStream, self).__init__(**kwargs)
self.processor = processor
self.formatting = formatting
self.conversion = conversion
self.mime = self.msg.content_type.split(';')[0]
def _get_headers(self):
return self.processor.process_headers(
def get_headers(self):
return self.formatting.format_headers(
self.msg.headers).encode(self.output_encoding)
def _iter_body(self):
for line, lf in self.msg.iter_lines(self.CHUNK_SIZE):
def iter_body(self):
first_chunk = True
iter_lines = self.msg.iter_lines(self.CHUNK_SIZE)
for line, lf in iter_lines:
if b'\0' in line:
if first_chunk:
converter = self.conversion.get_converter(self.mime)
if converter:
body = bytearray()
# noinspection PyAssignmentToLoopOrWithParameter
for line, lf in chain([(line, lf)], iter_lines):
body.extend(line)
body.extend(lf)
self.mime, body = converter.convert(body)
assert isinstance(body, str)
yield self.process_body(body)
return
raise BinarySuppressedError()
yield self._process_body(line) + lf
yield self.process_body(line) + lf
first_chunk = False
def _process_body(self, chunk):
return self.processor.process_body(
body=chunk.decode(self.msg.encoding, 'replace'),
mime=self.msg.content_type.split(';')[0]
).encode(self.output_encoding, 'replace')
def process_body(self, chunk):
if not isinstance(chunk, str):
# Text when a converter has been used,
# otherwise it will always be bytes.
chunk = chunk.decode(self.msg.encoding, 'replace')
chunk = self.formatting.format_body(content=chunk, mime=self.mime)
return chunk.encode(self.output_encoding, 'replace')
class BufferedPrettyStream(PrettyStream):
@ -257,14 +274,20 @@ class BufferedPrettyStream(PrettyStream):
CHUNK_SIZE = 1024 * 10
def _iter_body(self):
def iter_body(self):
# Read the whole body before prettifying it,
# but bail out immediately if the body is binary.
converter = None
body = bytearray()
for chunk in self.msg.iter_body(self.CHUNK_SIZE):
if b'\0' in chunk:
if not converter and b'\0' in chunk:
converter = self.conversion.get_converter(self.mime)
if not converter:
raise BinarySuppressedError()
body.extend(chunk)
yield self._process_body(body)
if converter:
self.mime, body = converter.convert(body)
yield self.process_body(body)

View File

@ -1,9 +1,16 @@
from .base import AuthPlugin
from .manager import PluginManager
from .builtin import BasicAuthPlugin, DigestAuthPlugin
from httpie.plugins.base import AuthPlugin, FormatterPlugin, ConverterPlugin
from httpie.plugins.manager import PluginManager
from httpie.plugins.builtin import BasicAuthPlugin, DigestAuthPlugin
from httpie.output.formatters.headers import HeadersFormatter
from httpie.output.formatters.json import JSONFormatter
from httpie.output.formatters.xml import XMLFormatter
from httpie.output.formatters.colors import ColorFormatter
plugin_manager = PluginManager()
plugin_manager.register(BasicAuthPlugin)
plugin_manager.register(DigestAuthPlugin)
plugin_manager.register(BasicAuthPlugin,
DigestAuthPlugin)
plugin_manager.register(HeadersFormatter,
JSONFormatter,
XMLFormatter,
ColorFormatter)

View File

@ -1,14 +1,4 @@
class AuthPlugin(object):
"""
Base auth plugin class.
See <https://github.com/jakubroztocil/httpie-ntlm> for an example auth plugin.
"""
# The value that should be passed to --auth-type
# to use this auth plugin. Eg. "my-auth"
auth_type = None
class BasePlugin(object):
# The name of the plugin, eg. "My auth".
name = None
@ -20,9 +10,64 @@ class AuthPlugin(object):
# This be set automatically once the plugin has been loaded.
package_name = None
class AuthPlugin(BasePlugin):
"""
Base auth plugin class.
See <https://github.com/jkbr/httpie-ntlm> for an example auth plugin.
"""
# The value that should be passed to --auth-type
# to use this auth plugin. Eg. "my-auth"
auth_type = None
def get_auth(self, username, password):
"""
Return a ``requests.auth.AuthBase`` subclass instance.
"""
raise NotImplementedError()
class ConverterPlugin(object):
def __init__(self, mime):
self.mime = mime
def convert(self, content_bytes):
raise NotImplementedError
@classmethod
def supports(cls, mime):
raise NotImplementedError
class FormatterPlugin(object):
def __init__(self, **kwargs):
"""
:param env: an class:`Environment` instance
:param kwargs: additional keyword argument that some
processor might require.
"""
self.enabled = True
self.kwargs = kwargs
def format_headers(self, headers):
"""Return processed `headers`
:param headers: The headers as text.
"""
return headers
def format_body(self, content, mime):
"""Return processed `content`.
:param mime: E.g., 'application/atom+xml'.
:param content: The body content as text
"""
return content

View File

@ -2,7 +2,7 @@ from base64 import b64encode
import requests.auth
from .base import AuthPlugin
from httpie.plugins.base import AuthPlugin
class BuiltinAuthPlugin(AuthPlugin):

View File

@ -1,8 +1,12 @@
from itertools import groupby
from pkg_resources import iter_entry_points
from httpie.plugins import AuthPlugin, FormatterPlugin, ConverterPlugin
ENTRY_POINT_NAMES = [
'httpie.plugins.auth.v1'
'httpie.plugins.auth.v1',
'httpie.plugins.formatter.v1',
'httpie.plugins.converter.v1',
]
@ -14,22 +18,41 @@ class PluginManager(object):
def __iter__(self):
return iter(self._plugins)
def register(self, plugin):
def register(self, *plugins):
for plugin in plugins:
self._plugins.append(plugin)
def get_auth_plugins(self):
return list(self._plugins)
def get_auth_plugin_mapping(self):
return dict((plugin.auth_type, plugin) for plugin in self)
def get_auth_plugin(self, auth_type):
return self.get_auth_plugin_mapping()[auth_type]
def load_installed_plugins(self):
for entry_point_name in ENTRY_POINT_NAMES:
for entry_point in iter_entry_points(entry_point_name):
plugin = entry_point.load()
plugin.package_name = entry_point.dist.key
self.register(entry_point.load())
# Auth
def get_auth_plugins(self):
return [plugin for plugin in self if issubclass(plugin, AuthPlugin)]
def get_auth_plugin_mapping(self):
return dict((plugin.auth_type, plugin)
for plugin in self.get_auth_plugins())
def get_auth_plugin(self, auth_type):
return self.get_auth_plugin_mapping()[auth_type]
# Output processing
def get_formatters(self):
return [plugin for plugin in self
if issubclass(plugin, FormatterPlugin)]
def get_formatters_grouped(self):
groups = {}
for group_name, group in groupby(
self.get_formatters(),
key=lambda p: getattr(p, 'group_name', 'format')):
groups[group_name] = list(group)
return groups
def get_converters(self):
return [plugin for plugin in self
if issubclass(plugin, ConverterPlugin)]

View File

@ -7,8 +7,8 @@ import os
import requests
from requests.cookies import RequestsCookieJar, create_cookie
from .compat import urlsplit
from .config import BaseConfigDict, DEFAULT_CONFIG_DIR
from httpie.compat import urlsplit
from httpie.config import BaseConfigDict, DEFAULT_CONFIG_DIR
from httpie.plugins import plugin_manager

View File

@ -1,7 +1,7 @@
import pytest
from httpie import ExitStatus
from httpie.output.processors.colors import get_lexer
from httpie.output.formatters.colors import get_lexer
from utils import TestEnvironment, http, httpbin, HTTP_OK, COLOR, CRLF

View File

@ -44,14 +44,14 @@ class TestUnicode:
def test_unicode_raw_json_item(self):
r = http('--json', 'POST', httpbin('/post'),
u'test:={ "%s" : [ "%s" ] }' % (UNICODE, UNICODE))
assert HTTP_OK in r
assert r.json['json'] == {'test': {UNICODE: [UNICODE]}}
def test_unicode_raw_json_item_verbose(self):
r = http('--verbose',
'--json', 'POST', httpbin('/post'),
r = http('--json', 'POST', httpbin('/post'),
u'test:={ "%s" : [ "%s" ] }' % (UNICODE, UNICODE))
assert HTTP_OK in r
assert UNICODE in r
assert r.json['json'] == {'test': {UNICODE: [UNICODE]}}
def test_unicode_url_query_arg_item(self):
r = http(httpbin('/get'), u'test==%s' % UNICODE)

View File

@ -158,7 +158,6 @@ def http(*args, **kwargs):
stdout.seek(0)
stderr.seek(0)
output = stdout.read()
try:
output = output.decode('utf8')
except UnicodeDecodeError: