From 74e4d0b67852ab0b628f687ac62e2d509fa74b61 Mon Sep 17 00:00:00 2001 From: Jakub Roztocil Date: Tue, 1 Mar 2016 14:57:15 +0800 Subject: [PATCH] Added JSON detection when ``--json, -j`` is set To correctly format JSON responses even when an incorrect ``Content-Type`` is returned. Closes #92 Closes #349 Closes #368 --- .gitignore | 19 +++++++------ CHANGELOG.rst | 2 ++ README.rst | 8 ++++-- httpie/output/formatters/colors.py | 44 ++++++++++++++++++++++-------- httpie/output/formatters/json.py | 15 +++++----- httpie/output/streams.py | 8 ++++-- tests/test_output.py | 37 ++++++++++++++----------- 7 files changed, 85 insertions(+), 48 deletions(-) diff --git a/.gitignore b/.gitignore index 3bdb64f3..0131d9ac 100644 --- a/.gitignore +++ b/.gitignore @@ -1,12 +1,13 @@ -dist -httpie.egg-info -build +.DS_Store +.idea/ +__pycache__/ +dist/ +httpie.egg-info/ +build/ +*.egg-info +.cache/ +.tox +.coverage *.pyc *.egg -.tox -README.html -.coverage htmlcov -.idea -.DS_Store -.cache/ diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e5bd6dbc..64ec5660 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -13,6 +13,8 @@ This project adheres to `Semantic Versioning `_. * Added ``--max-redirects`` (default 30) * Added ``-A`` as short name for ``--auth-type`` * Added ``-F`` as short name for ``--follow`` +* Added JSON detection when ``--json, -j`` is used in order to correctly format + JSON responses even when an incorrect ``Content-Type`` is returned. * Changed the default color style back to ``solarized`` as it supports both the light and dark terminal background mode * Fixed ``--session`` when used with ``--download`` diff --git a/README.rst b/README.rst index e46b959e..c0055576 100644 --- a/README.rst +++ b/README.rst @@ -389,7 +389,9 @@ both of which can be overwritten: You can use ``--json, -j`` to explicitly set ``Accept`` to ``application/json`` regardless of whether you are sending data (it's a shortcut for setting the header via the usual header notation – -``http url Accept:application/json``). +``http url Accept:application/json``). Also, with ``--json, -j``, +HTTPie tries to detect if the body is JSON even if the ``Content-Type`` +doesn't specify it in order to correctly format it. Simple example: @@ -1330,8 +1332,8 @@ Support * Use `GitHub issues `_ for bug reports and feature requests. -* Ask questions and discuss features on - `Gitter chat `_. +* Ask questions and discuss features in + ` our Gitter chat room `_. * Ask questions on `StackOverflow `_ (please make sure to use the `httpie `_ tag). diff --git a/httpie/output/formatters/colors.py b/httpie/output/formatters/colors.py index 2ccd5d9a..10ce5db2 100644 --- a/httpie/output/formatters/colors.py +++ b/httpie/output/formatters/colors.py @@ -1,3 +1,5 @@ +import json + import pygments.lexer import pygments.token import pygments.styles @@ -5,6 +7,7 @@ import pygments.lexers import pygments.style from pygments.formatters.terminal import TerminalFormatter from pygments.formatters.terminal256 import Terminal256Formatter +from pygments.lexers.special import TextLexer from pygments.util import ClassNotFound from httpie.plugins import FormatterPlugin @@ -27,12 +30,16 @@ class ColorFormatter(FormatterPlugin): """ group_name = 'colors' - def __init__(self, env, color_scheme=DEFAULT_STYLE, **kwargs): + def __init__(self, env, explicit_json=False, + color_scheme=DEFAULT_STYLE, **kwargs): super(ColorFormatter, self).__init__(**kwargs) if not env.colors: self.enabled = False return + # --json, -j + self.explicit_json = explicit_json + # Cache to speed things up when we process streamed body by line. self.lexer_cache = {} @@ -51,19 +58,18 @@ class ColorFormatter(FormatterPlugin): return pygments.highlight(headers, HTTPLexer(), self.formatter).strip() def format_body(self, body, mime): - lexer = self.get_lexer(mime) + lexer = self.get_lexer(mime, body) if lexer: body = pygments.highlight(body, lexer, self.formatter) return body.strip() - def get_lexer(self, mime): - if mime in self.lexer_cache: - return self.lexer_cache[mime] - self.lexer_cache[mime] = get_lexer(mime) - return self.lexer_cache[mime] + def get_lexer(self, mime, body): + return get_lexer(mime, body, self.explicit_json) -def get_lexer(mime): +def get_lexer(mime, explicit_json=False, body=''): + + # Build candidate mime type and lexer names. mime_types, lexer_names = [mime], [] type_, subtype = mime.split('/', 1) if '+' not in subtype: @@ -75,10 +81,14 @@ def get_lexer(mime): '%s/%s' % (type_, subtype_name), '%s/%s' % (type_, subtype_suffix) ]) - # as a last resort, if no lexer feels responsible, and - # the subtype contains 'json', take the JSON lexer - if 'json' in subtype: + + # As a last resort, if no lexer feels responsible, and + # the subtype contains 'json' or explicit --json is set, + # take the JSON lexer + if 'json' in subtype or explicit_json: lexer_names.append('json') + + # Try to resolve the right lexer. lexer = None for mime_type in mime_types: try: @@ -92,6 +102,18 @@ def get_lexer(mime): lexer = pygments.lexers.get_lexer_by_name(name) except ClassNotFound: pass + + if lexer and explicit_json and body and isinstance(lexer, TextLexer): + # When a text lexer is resolved even with --json (i.e. explicit + # text/plain Content-Type), try to parse the response as JSON + # and if it parses, sneak in the JSON lexer instead. + try: + json.loads(body) # FIXME: it also gets parsed in json.py + except ValueError: + pass # Invalid JSON, ignore. + else: + lexer = pygments.lexers.get_lexer_by_name('json') + return lexer diff --git a/httpie/output/formatters/json.py b/httpie/output/formatters/json.py index 292cc142..64b15f0e 100644 --- a/httpie/output/formatters/json.py +++ b/httpie/output/formatters/json.py @@ -10,17 +10,18 @@ DEFAULT_INDENT = 4 class JSONFormatter(FormatterPlugin): def format_body(self, body, mime): - if 'json' in mime: + if 'json' in mime or self.kwargs['explicit_json']: try: obj = json.loads(body) except ValueError: - # Invalid JSON, ignore. - pass + pass # Invalid JSON, ignore. else: # Indent, sort keys by name, and avoid # unicode escapes to improve readability. - body = json.dumps(obj, - sort_keys=True, - ensure_ascii=False, - indent=DEFAULT_INDENT) + body = json.dumps( + obj=obj, + sort_keys=True, + ensure_ascii=False, + indent=DEFAULT_INDENT + ) return body diff --git a/httpie/output/streams.py b/httpie/output/streams.py index 03046a57..b2269814 100644 --- a/httpie/output/streams.py +++ b/httpie/output/streams.py @@ -112,8 +112,12 @@ def get_stream_type(env, args): PrettyStream if args.stream else BufferedPrettyStream, env=env, conversion=Conversion(), - formatting=Formatting(env=env, groups=args.prettify, - color_scheme=args.style), + formatting=Formatting( + env=env, + groups=args.prettify, + color_scheme=args.style, + explicit_json=args.json, + ), ) else: Stream = partial(EncodedStream, env=env) diff --git a/tests/test_output.py b/tests/test_output.py index 6a4ebf41..483eeb39 100644 --- a/tests/test_output.py +++ b/tests/test_output.py @@ -28,23 +28,28 @@ class TestVerboseFlag: class TestColors: - @pytest.mark.parametrize('mime', [ - 'application/json', - 'application/json+foo', - 'application/foo+json', - 'application/json-foo', - 'application/x-json', - 'foo/json', - 'foo/json+bar', - 'foo/bar+json', - 'foo/json-foo', - 'foo/x-json', - 'application/vnd.comverge.grid+hal+json', - ]) - def test_get_lexer(self, mime): - lexer = get_lexer(mime) + @pytest.mark.parametrize( + argnames=['mime', 'explicit_json', 'body', 'expected_lexer_name'], + argvalues=[ + ('application/json', False, None, 'JSON'), + ('application/json+foo', False, None, 'JSON'), + ('application/foo+json', False, None, 'JSON'), + ('application/json-foo', False, None, 'JSON'), + ('application/x-json', False, None, 'JSON'), + ('foo/json', False, None, 'JSON'), + ('foo/json+bar', False, None, 'JSON'), + ('foo/bar+json', False, None, 'JSON'), + ('foo/json-foo', False, None, 'JSON'), + ('foo/x-json', False, None, 'JSON'), + ('application/vnd.comverge.grid+hal+json', False, None, 'JSON'), + ('text/plain', True, '{}', 'JSON'), + ('text/plain', True, 'foo', 'Text only'), + ] + ) + def test_get_lexer(self, mime, explicit_json, body, expected_lexer_name): + lexer = get_lexer(mime, body=body, explicit_json=explicit_json) assert lexer is not None - assert lexer.name == 'JSON' + assert lexer.name == expected_lexer_name def test_get_lexer_not_found(self): assert get_lexer('xxx/yyy') is None