Added JSON detection when `--json, -j` is set

To correctly format JSON responses even when an incorrect ``Content-Type`` is returned.

Closes #92
Closes #349
Closes #368

.gitignore

@@ -1,12 +1,13 @@
-dist
+.DS_Store
-httpie.egg-info
+.idea/
-build
+__pycache__/
+dist/
+httpie.egg-info/
+build/
+*.egg-info
+.cache/
+.tox
+.coverage
 *.pyc
 *.egg
-.tox
-README.html
-.coverage
 htmlcov
-.idea
-.DS_Store
-.cache/

CHANGELOG.rst

@@ -13,6 +13,8 @@ This project adheres to `Semantic Versioning <http://semver.org/>`_.
 * Added ``--max-redirects`` (default 30)
 * Added ``-A`` as short name for ``--auth-type``
 * Added ``-F`` as short name for ``--follow``
+* Added JSON detection when ``--json, -j`` is used in order to correctly format
+  JSON responses even when an incorrect ``Content-Type`` is returned.
 * Changed the default color style back to ``solarized`` as it supports
   both the light and dark terminal background mode
 * Fixed ``--session`` when used with ``--download``

README.rst

@@ -389,7 +389,9 @@ both of which can be overwritten:
 You can use ``--json, -j`` to explicitly set ``Accept``
 to ``application/json`` regardless of whether you are sending data
 (it's a shortcut for setting the header via the usual header notation –
-``http url Accept:application/json``).
+``http url Accept:application/json``). Also, with ``--json, -j``,
+HTTPie tries to detect if the body is JSON even if the ``Content-Type``
+doesn't specify it in order to correctly format it.
 
 Simple example:
 
@@ -1330,8 +1332,8 @@ Support
 
 * Use `GitHub issues <https://github.com/jkbr/httpie/issues>`_
   for bug reports and feature requests.
-* Ask questions and discuss features on
+* Ask questions and discuss features in
-  `Gitter chat <https://gitter.im/jkbrzt/httpie>`_.
+  ` our Gitter chat room <https://gitter.im/jkbrzt/httpie>`_.
 * Ask questions on `StackOverflow <https://stackoverflow.com>`_
   (please make sure to use the
   `httpie <http://stackoverflow.com/questions/tagged/httpie>`_ tag).

httpie/output/formatters/colors.py

@@ -1,3 +1,5 @@
+import json
+
 import pygments.lexer
 import pygments.token
 import pygments.styles
@@ -5,6 +7,7 @@ import pygments.lexers
 import pygments.style
 from pygments.formatters.terminal import TerminalFormatter
 from pygments.formatters.terminal256 import Terminal256Formatter
+from pygments.lexers.special import TextLexer
 from pygments.util import ClassNotFound
 
 from httpie.plugins import FormatterPlugin
@@ -27,12 +30,16 @@ class ColorFormatter(FormatterPlugin):
     """
     group_name = 'colors'
 
-    def __init__(self, env, color_scheme=DEFAULT_STYLE, **kwargs):
+    def __init__(self, env, explicit_json=False,
+                 color_scheme=DEFAULT_STYLE, **kwargs):
         super(ColorFormatter, self).__init__(**kwargs)
         if not env.colors:
             self.enabled = False
             return
 
+        # --json, -j
+        self.explicit_json = explicit_json
+
         # Cache to speed things up when we process streamed body by line.
         self.lexer_cache = {}
 
@@ -51,19 +58,18 @@ class ColorFormatter(FormatterPlugin):
         return pygments.highlight(headers, HTTPLexer(), self.formatter).strip()
 
     def format_body(self, body, mime):
-        lexer = self.get_lexer(mime)
+        lexer = self.get_lexer(mime, body)
         if lexer:
             body = pygments.highlight(body, lexer, self.formatter)
         return body.strip()
 
-    def get_lexer(self, mime):
+    def get_lexer(self, mime, body):
-        if mime in self.lexer_cache:
+        return get_lexer(mime, body, self.explicit_json)
-            return self.lexer_cache[mime]
-        self.lexer_cache[mime] = get_lexer(mime)
-        return self.lexer_cache[mime]
 
 
-def get_lexer(mime):
+def get_lexer(mime, explicit_json=False, body=''):
+
+    # Build candidate mime type and lexer names.
     mime_types, lexer_names = [mime], []
     type_, subtype = mime.split('/', 1)
     if '+' not in subtype:
@@ -75,10 +81,14 @@ def get_lexer(mime):
             '%s/%s' % (type_, subtype_name),
             '%s/%s' % (type_, subtype_suffix)
         ])
-    # as a last resort, if no lexer feels responsible, and
+
-    # the subtype contains 'json', take the JSON lexer
+    # As a last resort, if no lexer feels responsible, and
-    if 'json' in subtype:
+    # the subtype contains 'json' or explicit --json is set,
+    # take the JSON lexer
+    if 'json' in subtype or explicit_json:
         lexer_names.append('json')
+
+    # Try to resolve the right lexer.
     lexer = None
     for mime_type in mime_types:
         try:
@@ -92,6 +102,18 @@ def get_lexer(mime):
                 lexer = pygments.lexers.get_lexer_by_name(name)
             except ClassNotFound:
                 pass
+
+    if lexer and explicit_json and body and isinstance(lexer, TextLexer):
+        # When a text lexer is resolved even with --json (i.e. explicit
+        # text/plain Content-Type), try to parse the response as JSON
+        # and if it parses, sneak in the JSON lexer instead.
+        try:
+            json.loads(body)  # FIXME: it also gets parsed in json.py
+        except ValueError:
+            pass  # Invalid JSON, ignore.
+        else:
+            lexer = pygments.lexers.get_lexer_by_name('json')
+
     return lexer
 
 

httpie/output/formatters/json.py

@@ -10,17 +10,18 @@ DEFAULT_INDENT = 4
 class JSONFormatter(FormatterPlugin):
 
     def format_body(self, body, mime):
-        if 'json' in mime:
+        if 'json' in mime or self.kwargs['explicit_json']:
             try:
                 obj = json.loads(body)
             except ValueError:
-                # Invalid JSON, ignore.
+                pass  # Invalid JSON, ignore.
-                pass
             else:
                 # Indent, sort keys by name, and avoid
                 # unicode escapes to improve readability.
-                body = json.dumps(obj,
+                body = json.dumps(
-                                  sort_keys=True,
+                    obj=obj,
-                                  ensure_ascii=False,
+                    sort_keys=True,
-                                  indent=DEFAULT_INDENT)
+                    ensure_ascii=False,
+                    indent=DEFAULT_INDENT
+                )
         return body

httpie/output/streams.py

@@ -112,8 +112,12 @@ def get_stream_type(env, args):
             PrettyStream if args.stream else BufferedPrettyStream,
             env=env,
             conversion=Conversion(),
-            formatting=Formatting(env=env, groups=args.prettify,
+            formatting=Formatting(
-                                  color_scheme=args.style),
+                env=env,
+                groups=args.prettify,
+                color_scheme=args.style,
+                explicit_json=args.json,
+            ),
         )
     else:
         Stream = partial(EncodedStream, env=env)

tests/test_output.py

@@ -28,23 +28,28 @@ class TestVerboseFlag:
 
 class TestColors:
 
-    @pytest.mark.parametrize('mime', [
+    @pytest.mark.parametrize(
-        'application/json',
+        argnames=['mime', 'explicit_json', 'body', 'expected_lexer_name'],
-        'application/json+foo',
+        argvalues=[
-        'application/foo+json',
+            ('application/json',     False, None, 'JSON'),
-        'application/json-foo',
+            ('application/json+foo', False, None, 'JSON'),
-        'application/x-json',
+            ('application/foo+json', False, None, 'JSON'),
-        'foo/json',
+            ('application/json-foo', False, None, 'JSON'),
-        'foo/json+bar',
+            ('application/x-json',   False, None, 'JSON'),
-        'foo/bar+json',
+            ('foo/json',             False, None, 'JSON'),
-        'foo/json-foo',
+            ('foo/json+bar',         False, None, 'JSON'),
-        'foo/x-json',
+            ('foo/bar+json',         False, None, 'JSON'),
-        'application/vnd.comverge.grid+hal+json',
+            ('foo/json-foo',         False, None, 'JSON'),
-    ])
+            ('foo/x-json',           False, None, 'JSON'),
-    def test_get_lexer(self, mime):
+            ('application/vnd.comverge.grid+hal+json', False, None, 'JSON'),
-        lexer = get_lexer(mime)
+            ('text/plain',           True, '{}', 'JSON'),
+            ('text/plain',           True, 'foo', 'Text only'),
+        ]
+    )
+    def test_get_lexer(self, mime, explicit_json, body, expected_lexer_name):
+        lexer = get_lexer(mime, body=body, explicit_json=explicit_json)
         assert lexer is not None
-        assert lexer.name == 'JSON'
+        assert lexer.name == expected_lexer_name
 
     def test_get_lexer_not_found(self):
         assert get_lexer('xxx/yyy') is None