Allow to overwrite the response Content-Type from options (#1134)

* Allow to override the response `Content-Type` from options

* Apply suggestions from code review

Co-authored-by: Jakub Roztocil <jakub@roztocil.co>

* Rename the option from `--response.content-type` to `--response-as`

* Update CHANGELOG.md

Co-authored-by: Jakub Roztocil <jakub@roztocil.co>

CHANGELOG.md

@@ -6,6 +6,8 @@ This project adheres to [Semantic Versioning](https://semver.org/).
 ## [2.6.0.dev0](https://github.com/httpie/httpie/compare/2.5.0...master) (unreleased)
 
 - Added support for formatting & coloring of JSON bodies preceded by non-JSON data (e.g., an XXSI prefix). ([#1130](https://github.com/httpie/httpie/issues/1130))
+- Added `--format-options=response.as:CONTENT_TYPE` to allow overriding the response `Content-Type`. ([#1134](https://github.com/httpie/httpie/issues/1134))
+- Added `--response-as` shortcut for setting the response `Content-Type`-related `--format-options`. ([#1134](https://github.com/httpie/httpie/issues/1134))
 - Installed plugins are now listed in `--debug` output. ([#1165](https://github.com/httpie/httpie/issues/1165))
 - Fixed duplicate keys preservation of JSON data. ([#1163](https://github.com/httpie/httpie/issues/1163))
 

docs/README.md

@@ -1214,14 +1214,15 @@ You can further control the applied formatting via the more granular [format opt
 The `--format-options=opt1:value,opt2:value` option allows you to control how the output should be formatted
 when formatting is applied. The following options are available:
 
-|           Option | Default value | Shortcuts                |
+|           Option | Default value | Shortcuts                                 |
-| ---------------: | :-----------: | ------------------------ |
+| ---------------: | :-----------: | ----------------------------------------- |
-|   `headers.sort` |    `true`     | `--sorted`, `--unsorted` |
+|   `headers.sort` |    `true`     | `--sorted`, `--unsorted`                  |
-|    `json.format` |    `true`     | N/A                      |
+|    `json.format` |    `true`     | N/A                                       |
-|    `json.indent` |      `4`      | N/A                      |
+|    `json.indent` |      `4`      | N/A                                       |
-| `json.sort_keys` |    `true`     | `--sorted`, `--unsorted` |
+| `json.sort_keys` |    `true`     | `--sorted`, `--unsorted`                  |
-|     `xml.format` |    `true`     | N/A                      |
+|    `response.as` |     `''`      | [`--response-as`](#response-content-type) |
-|     `xml.indent` |      `2`      | N/A                      |
+|     `xml.format` |    `true`     | N/A                                       |
+|     `xml.indent` |      `2`      | N/A                                       |
 
 For example, this is how you would disable the default header and JSON key
 sorting, and specify a custom JSON indent size:
@@ -1236,6 +1237,18 @@ sorting-related format options (currently it means JSON keys and headers):
 
 This is something you will typically store as one of the default options in your [config](#config) file.
 
+#### Response `Content-Type`
+
+The `--response-as=value` option is a shortcut for `--format-options response.as:value`,
+and it allows you to override the response `Content-Type` sent by the server.
+That makes it possible for HTTPie to pretty-print the response even when the server specifies the type incorrectly.
+
+For example, the following request will force the response to be treated as XML:
+
+```bash
+$ http --response-as=application/xml pie.dev/get
+```
+
 ### Binary data
 
 Binary data is suppressed for terminal output, which makes it safe to perform requests to URLs that send back binary data.

httpie/cli/argparser.py

@@ -457,7 +457,10 @@ class HTTPieArgumentParser(argparse.ArgumentParser):
             self.error('--continue requires --output to be specified')
 
     def _process_format_options(self):
+        format_options = self.args.format_options or []
+        if self.args.response_as is not None:
+            format_options.append('response.as:' + self.args.response_as)
         parsed_options = PARSED_DEFAULT_FORMAT_OPTIONS
-        for options_group in self.args.format_options or []:
+        for options_group in format_options:
             parsed_options = parse_format_options(options_group, defaults=parsed_options)
         self.args.format_options = parsed_options

httpie/cli/constants.py

@@ -85,11 +85,13 @@ PRETTY_MAP = {
 PRETTY_STDOUT_TTY_ONLY = object()
 
 
+EMPTY_FORMAT_OPTION = "''"
 DEFAULT_FORMAT_OPTIONS = [
     'headers.sort:true',
     'json.format:true',
     'json.indent:4',
     'json.sort_keys:true',
+    'response.as:' + EMPTY_FORMAT_OPTION,
     'xml.format:true',
     'xml.indent:2',
 ]

httpie/cli/definition.py

@@ -309,6 +309,20 @@ output_processing.add_argument(
     '''
 )
 
+output_processing.add_argument(
+    '--response-as',
+    metavar='CONTENT_TYPE',
+    help='''
+    Override the response Content-Type for formatting purposes, e.g.:
+
+        --response-as=application/xml
+
+    It is a shortcut for:
+
+        --format-options=response.as:CONTENT_TYPE
+    '''
+)
+
 
 output_processing.add_argument(
     '--format-options',

httpie/output/processing.py

@@ -33,6 +33,7 @@ class Formatting:
         :param kwargs: additional keyword arguments for processors
 
         """
+        self.options = kwargs['format_options']
         available_plugins = plugin_manager.get_formatters_grouped()
         self.enabled_plugins = []
         for group in groups:

httpie/output/streams.py

@@ -2,11 +2,12 @@ from abc import ABCMeta, abstractmethod
 from itertools import chain
 from typing import Callable, Iterable, Union
 
+from ..cli.constants import EMPTY_FORMAT_OPTION
 from ..context import Environment
 from ..constants import UTF8
-from ..models import HTTPMessage
+from ..models import HTTPMessage, HTTPResponse
 from .processing import Conversion, Formatting
-
+from .utils import parse_header_content_type
 
 BINARY_SUPPRESSED_NOTICE = (
     b'\n'
@@ -136,7 +137,15 @@ class PrettyStream(EncodedStream):
         super().__init__(**kwargs)
         self.formatting = formatting
         self.conversion = conversion
-        self.mime = self.msg.content_type.split(';')[0]
+        self.mime = self.get_mime()
+
+    def get_mime(self) -> str:
+        mime = parse_header_content_type(self.msg.content_type)[0]
+        if isinstance(self.msg, HTTPResponse):
+            forced_content_type = self.formatting.options['response']['as']
+            if forced_content_type != EMPTY_FORMAT_OPTION:
+                mime = parse_header_content_type(forced_content_type)[0] or mime
+        return mime
 
     def get_headers(self) -> bytes:
         return self.formatting.format_headers(

httpie/output/utils.py

@@ -35,3 +35,57 @@ def parse_prefixed_json(data: str) -> Tuple[str, str]:
     data_prefix = matches[0] if matches else ''
     body = data[len(data_prefix):]
     return data_prefix, body
+
+
+def parse_header_content_type(line):
+    """Parse a Content-Type like header.
+    Return the main Content-Type and a dictionary of options.
+        >>> parse_header_content_type('application/xml; charset=utf-8')
+        ('application/xml', {'charset': 'utf-8'})
+        >>> parse_header_content_type('application/xml; charset = utf-8')
+        ('application/xml', {'charset': 'utf-8'})
+        >>> parse_header_content_type('application/html+xml;ChArSeT="UTF-8"')
+        ('application/html+xml', {'charset': 'UTF-8'})
+        >>> parse_header_content_type('application/xml')
+        ('application/xml', {})
+        >>> parse_header_content_type(';charset=utf-8')
+        ('', {'charset': 'utf-8'})
+        >>> parse_header_content_type('charset=utf-8')
+        ('', {'charset': 'utf-8'})
+        >>> parse_header_content_type('multipart/mixed; boundary="gc0pJq0M:08jU534c0p"')
+        ('multipart/mixed', {'boundary': 'gc0pJq0M:08jU534c0p'})
+        >>> parse_header_content_type('Message/Partial; number=3; total=3; id="oc=jpbe0M2Yt4s@foo.com"')
+        ('Message/Partial', {'number': '3', 'total': '3', 'id': 'oc=jpbe0M2Yt4s@foo.com'})
+    """
+    # Source: https://github.com/python/cpython/blob/bb3e0c2/Lib/cgi.py#L230
+
+    def _parseparam(s: str):
+        # Source: https://github.com/python/cpython/blob/bb3e0c2/Lib/cgi.py#L218
+        while s[:1] == ';':
+            s = s[1:]
+            end = s.find(';')
+            while end > 0 and (s.count('"', 0, end) - s.count('\\"', 0, end)) % 2:
+                end = s.find(';', end + 1)
+            if end < 0:
+                end = len(s)
+            f = s[:end]
+            yield f.strip()
+            s = s[end:]
+
+    # Special case: 'key=value' only (without starting with ';').
+    if ';' not in line and '=' in line:
+        line = ';' + line
+
+    parts = _parseparam(';' + line)
+    key = parts.__next__()
+    pdict = {}
+    for p in parts:
+        i = p.find('=')
+        if i >= 0:
+            name = p[:i].strip().lower()
+            value = p[i + 1:].strip()
+            if len(value) >= 2 and value[0] == value[-1] == '"':
+                value = value[1:-1]
+                value = value.replace('\\\\', '\\').replace('\\"', '"')
+            pdict[name] = value
+    return key, pdict

tests/test_output.py

@@ -377,6 +377,9 @@ class TestFormatOptions:
                         'indent': 10,
                         'format': True
                     },
+                    'response': {
+                        'as': "''",
+                    },
                     'xml': {
                         'format': True,
                         'indent': 2,
@@ -396,6 +399,9 @@ class TestFormatOptions:
                         'indent': 4,
                         'format': True
                     },
+                    'response': {
+                        'as': "''",
+                    },
                     'xml': {
                         'format': True,
                         'indent': 2,
@@ -417,6 +423,9 @@ class TestFormatOptions:
                         'indent': 4,
                         'format': True
                     },
+                    'response': {
+                        'as': "''",
+                    },
                     'xml': {
                         'format': True,
                         'indent': 2,
@@ -435,6 +444,7 @@ class TestFormatOptions:
             (
                 [
                     '--format-options=json.indent:2',
+                    '--format-options=response.as:application/xml; charset=utf-8',
                     '--format-options=xml.format:false',
                     '--format-options=xml.indent:4',
                     '--unsorted',
@@ -449,6 +459,9 @@ class TestFormatOptions:
                         'indent': 2,
                         'format': True
                     },
+                    'response': {
+                        'as': 'application/xml; charset=utf-8',
+                    },
                     'xml': {
                         'format': False,
                         'indent': 4,
@@ -470,6 +483,9 @@ class TestFormatOptions:
                         'indent': 2,
                         'format': True
                     },
+                    'response': {
+                        'as': "''",
+                    },
                     'xml': {
                         'format': True,
                         'indent': 2,
@@ -492,6 +508,9 @@ class TestFormatOptions:
                         'indent': 2,
                         'format': True
                     },
+                    'response': {
+                        'as': "''",
+                    },
                     'xml': {
                         'format': True,
                         'indent': 2,

tests/test_xml.py

@@ -9,20 +9,21 @@ from httpie.output.formatters.xml import parse_xml, pretty_xml
 from .fixtures import XML_FILES_PATH, XML_FILES_VALID, XML_FILES_INVALID
 from .utils import http, URL_EXAMPLE
 
-SAMPLE_XML_DATA = '<?xml version="1.0" encoding="utf-8"?><root><e>text</e></root>'
+XML_DATA_RAW = '<?xml version="1.0" encoding="utf-8"?><root><e>text</e></root>'
+XML_DATA_FORMATTED = pretty_xml(parse_xml(XML_DATA_RAW))
 
 
 @pytest.mark.parametrize(
     'options, expected_xml',
     [
-        ('xml.format:false', SAMPLE_XML_DATA),
+        ('xml.format:false', XML_DATA_RAW),
-        ('xml.indent:2', pretty_xml(parse_xml(SAMPLE_XML_DATA))),
+        ('xml.indent:2', XML_DATA_FORMATTED),
-        ('xml.indent:4', pretty_xml(parse_xml(SAMPLE_XML_DATA), indent=4)),
+        ('xml.indent:4', pretty_xml(parse_xml(XML_DATA_RAW), indent=4)),
     ]
 )
 @responses.activate
 def test_xml_format_options(options, expected_xml):
-    responses.add(responses.GET, URL_EXAMPLE, body=SAMPLE_XML_DATA,
+    responses.add(responses.GET, URL_EXAMPLE, body=XML_DATA_RAW,
                   content_type='application/xml')
 
     r = http('--format-options', options, URL_EXAMPLE)
@@ -83,3 +84,55 @@ def test_invalid_xml(file):
     # No formatting done, data is simply printed as-is
     r = http(URL_EXAMPLE)
     assert xml_data in r
+
+
+@responses.activate
+def test_content_type_from_format_options_argument():
+    """Test XML response with a incorrect Content-Type header.
+    Using the --format-options to force the good one.
+    """
+    responses.add(responses.GET, URL_EXAMPLE, body=XML_DATA_RAW,
+                  content_type='plain/text')
+    args = ('--format-options', 'response.as:application/xml',
+            URL_EXAMPLE)
+
+    # Ensure the option is taken into account only for responses.
+    # Request
+    r = http('--offline', '--raw', XML_DATA_RAW, *args)
+    assert XML_DATA_RAW in r
+
+    # Response
+    r = http(*args)
+    assert XML_DATA_FORMATTED in r
+
+
+@responses.activate
+def test_content_type_from_shortcut_argument():
+    """Test XML response with a incorrect Content-Type header.
+    Using the --format-options shortcut to force the good one.
+    """
+    responses.add(responses.GET, URL_EXAMPLE, body=XML_DATA_RAW,
+                  content_type='text/plain')
+    args = ('--response-as', 'application/xml', URL_EXAMPLE)
+
+    # Ensure the option is taken into account only for responses.
+    # Request
+    r = http('--offline', '--raw', XML_DATA_RAW, *args)
+    assert XML_DATA_RAW in r
+
+    # Response
+    r = http(*args)
+    assert XML_DATA_FORMATTED in r
+
+
+@responses.activate
+def test_content_type_from_incomplete_format_options_argument():
+    """Test XML response with a incorrect Content-Type header.
+    Using the --format-options to use a partial Content-Type without mime type.
+    """
+    responses.add(responses.GET, URL_EXAMPLE, body=XML_DATA_RAW,
+                  content_type='text/plain')
+
+    # The provided Content-Type is simply ignored, and so no formatting is done.
+    r = http('--response-as', 'charset=utf-8', URL_EXAMPLE)
+    assert XML_DATA_RAW in r