Fix encoding error with non-prettified encoded responses

Removed `--format-option response.as` an promote `--response-as`: using
the format option would be misleading as it is now also used by non-prettified
responses.

CHANGELOG.md

@@ -8,7 +8,7 @@ This project adheres to [Semantic Versioning](https://semver.org/).
 - Added support for formatting & coloring of JSON bodies preceded by non-JSON data (e.g., an XXSI prefix). ([#1130](https://github.com/httpie/httpie/issues/1130))
 - Added `--format-options=response.as:CONTENT_TYPE` to allow overriding the response `Content-Type`. ([#1134](https://github.com/httpie/httpie/issues/1134))
 - Added `--response-as` shortcut for setting the response `Content-Type`-related `--format-options`. ([#1134](https://github.com/httpie/httpie/issues/1134))
-- Improved handling of prettified responses without correct `Content-Type` encoding. ([#1110](https://github.com/httpie/httpie/issues/1110))
+- Improved handling of responses without correct `Content-Type` encoding. ([#1110](https://github.com/httpie/httpie/issues/1110), [#1168](https://github.com/httpie/httpie/issues/1168))
 - Installed plugins are now listed in `--debug` output. ([#1165](https://github.com/httpie/httpie/issues/1165))
 - Fixed duplicate keys preservation of JSON data. ([#1163](https://github.com/httpie/httpie/issues/1163))
 

docs/README.md

@@ -1214,15 +1214,14 @@ You can further control the applied formatting via the more granular [format opt
 The `--format-options=opt1:value,opt2:value` option allows you to control how the output should be formatted
 when formatting is applied. The following options are available:
 
-|           Option | Default value | Shortcuts                                 |
+|           Option | Default value | Shortcuts                |
-| ---------------: | :-----------: | ----------------------------------------- |
+| ---------------: | :-----------: | ------------------------ |
-|   `headers.sort` |    `true`     | `--sorted`, `--unsorted`                  |
+|   `headers.sort` |    `true`     | `--sorted`, `--unsorted` |
-|    `json.format` |    `true`     | N/A                                       |
+|    `json.format` |    `true`     | N/A                      |
-|    `json.indent` |      `4`      | N/A                                       |
+|    `json.indent` |      `4`      | N/A                      |
-| `json.sort_keys` |    `true`     | `--sorted`, `--unsorted`                  |
+| `json.sort_keys` |    `true`     | `--sorted`, `--unsorted` |
-|    `response.as` |     `''`      | [`--response-as`](#response-content-type) |
+|     `xml.format` |    `true`     | N/A                      |
-|     `xml.format` |    `true`     | N/A                                       |
+|     `xml.indent` |      `2`      | N/A                      |
-|     `xml.indent` |      `2`      | N/A                                       |
 
 For example, this is how you would disable the default header and JSON key
 sorting, and specify a custom JSON indent size:
@@ -1237,11 +1236,10 @@ sorting-related format options (currently it means JSON keys and headers):
 
 This is something you will typically store as one of the default options in your [config](#config) file.
 
-#### Response `Content-Type`
+### Response `Content-Type`
 
-The `--response-as=value` option is a shortcut for `--format-options response.as:value`,
+The `--response-as=value` option allows you to override the response `Content-Type` sent by the server.
-and it allows you to override the response `Content-Type` sent by the server.
+That makes it possible for HTTPie to print the response even when the server specifies the type incorrectly.
-That makes it possible for HTTPie to pretty-print the response even when the server specifies the type incorrectly.
 
 For example, the following request will force the response to be treated as XML:
 

httpie/cli/argparser.py

@@ -458,8 +458,6 @@ class HTTPieArgumentParser(argparse.ArgumentParser):
 
     def _process_format_options(self):
         format_options = self.args.format_options or []
-        if self.args.response_as is not None:
-            format_options.append('response.as:' + self.args.response_as)
         parsed_options = PARSED_DEFAULT_FORMAT_OPTIONS
         for options_group in format_options:
             parsed_options = parse_format_options(options_group, defaults=parsed_options)

httpie/cli/constants.py

@@ -85,13 +85,11 @@ PRETTY_MAP = {
 PRETTY_STDOUT_TTY_ONLY = object()
 
 
-EMPTY_FORMAT_OPTION = "''"
 DEFAULT_FORMAT_OPTIONS = [
     'headers.sort:true',
     'json.format:true',
     'json.indent:4',
     'json.sort_keys:true',
-    'response.as:' + EMPTY_FORMAT_OPTION,
     'xml.format:true',
     'xml.indent:2',
 ]

httpie/cli/definition.py

@@ -313,15 +313,12 @@ output_processing.add_argument(
     '--response-as',
     metavar='CONTENT_TYPE',
     help='''
-    Override the response Content-Type for formatting purposes, e.g.:
+    Override the response Content-Type for display purposes, e.g.:
 
         --response-as=application/xml
         --response-as=charset=utf-8
         --response-as='application/xml; charset=utf-8'
 
-    It is a shortcut for:
-
-        --format-options=response.as:CONTENT_TYPE
     '''
 )
 

httpie/output/processing.py

@@ -33,7 +33,6 @@ class Formatting:
         :param kwargs: additional keyword arguments for processors
 
         """
-        self.options = kwargs['format_options']
         available_plugins = plugin_manager.get_formatters_grouped()
         self.enabled_plugins = []
         for group in groups:

httpie/output/streams.py

@@ -1,12 +1,11 @@
 from abc import ABCMeta, abstractmethod
 from itertools import chain
-from typing import Any, Callable, Dict, Iterable, Tuple, Union
+from typing import Callable, Dict, Iterable, Tuple, Union
 
 from .. import codec
-from ..cli.constants import EMPTY_FORMAT_OPTION
 from ..context import Environment
 from ..constants import UTF8
-from ..models import HTTPMessage, HTTPResponse
+from ..models import HTTPMessage, HTTPRequest
 from .processing import Conversion, Formatting
 from .utils import parse_header_content_type
 
@@ -100,8 +99,11 @@ class EncodedStream(BaseStream):
     """
     CHUNK_SIZE = 1
 
-    def __init__(self, env=Environment(), **kwargs):
+    def __init__(self, env=Environment(), response_as: str = None, **kwargs):
         super().__init__(**kwargs)
+        self.response_as = response_as
+        self.mime, self.encoding = self._get_mime_and_encoding()
+
         if env.stdout_isatty:
             # Use the encoding supported by the terminal.
             output_encoding = env.stdout_encoding
@@ -111,11 +113,31 @@ class EncodedStream(BaseStream):
         # Default to UTF-8 when unsure.
         self.output_encoding = output_encoding or UTF8
 
+    def _get_mime_and_encoding(self) -> Tuple[str, Dict[str, str]]:
+        """Parse `Content-Type` header or `--response-as` value to guess
+        correct mime type and encoding.
+
+        """
+        # Defaults from the `Content-Type` header.
+        mime, options = parse_header_content_type(self.msg.content_type)
+
+        if isinstance(self.msg, HTTPRequest):
+            encoding = self.msg.encoding
+        elif self.response_as is None:
+            encoding = options.get('charset')
+        else:
+            # Override from the `--response-as` option.
+            forced_mime, forced_options = parse_header_content_type(self.response_as)
+            mime = forced_mime or mime
+            encoding = forced_options.get('charset') or options.get('charset')
+
+        return mime, encoding or ''
+
     def iter_body(self) -> Iterable[bytes]:
         for line, lf in self.msg.iter_lines(self.CHUNK_SIZE):
             if b'\0' in line:
                 raise BinarySuppressedError()
-            line = codec.decode(line, self.msg.encoding)
+            line = codec.decode(line, self.encoding)
             yield codec.encode(line, self.output_encoding) + lf
 
 
@@ -138,23 +160,6 @@ class PrettyStream(EncodedStream):
         super().__init__(**kwargs)
         self.formatting = formatting
         self.conversion = conversion
-        self.mime, mime_options = self._get_mime_and_options()
-        self.encoding = mime_options.get('charset') or ''
-
-    def _get_mime_and_options(self) -> Tuple[str, Dict[str, Any]]:
-        # Defaults from the `Content-Type` header.
-        mime, options = parse_header_content_type(self.msg.content_type)
-
-        if not isinstance(self.msg, HTTPResponse):
-            return mime, options
-
-        # Override from the `--response-as` option.
-        forced_content_type = self.formatting.options['response']['as']
-        if forced_content_type == EMPTY_FORMAT_OPTION:
-            return mime, options
-
-        forced_mime, forced_options = parse_header_content_type(forced_content_type)
-        return (forced_mime or mime, forced_options or options)
 
     def get_headers(self) -> bytes:
         return self.formatting.format_headers(

httpie/output/writer.py

@@ -134,23 +134,23 @@ def get_stream_type_and_kwargs(
                 else RawStream.CHUNK_SIZE
             )
         }
-    elif args.prettify:
-        stream_class = PrettyStream if args.stream else BufferedPrettyStream
-        stream_kwargs = {
-            'env': env,
-            'conversion': Conversion(),
-            'formatting': Formatting(
-                env=env,
-                groups=args.prettify,
-                color_scheme=args.style,
-                explicit_json=args.json,
-                format_options=args.format_options,
-            )
-        }
     else:
         stream_class = EncodedStream
         stream_kwargs = {
-            'env': env
+            'env': env,
+            'response_as': args.response_as,
         }
+        if args.prettify:
+            stream_class = PrettyStream if args.stream else BufferedPrettyStream
+            stream_kwargs.update({
+                'conversion': Conversion(),
+                'formatting': Formatting(
+                    env=env,
+                    groups=args.prettify,
+                    color_scheme=args.style,
+                    explicit_json=args.json,
+                    format_options=args.format_options,
+                )
+            })
 
     return stream_class, stream_kwargs

tests/test_output.py

@@ -377,9 +377,6 @@ class TestFormatOptions:
                         'indent': 10,
                         'format': True
                     },
-                    'response': {
-                        'as': "''",
-                    },
                     'xml': {
                         'format': True,
                         'indent': 2,
@@ -399,9 +396,6 @@ class TestFormatOptions:
                         'indent': 4,
                         'format': True
                     },
-                    'response': {
-                        'as': "''",
-                    },
                     'xml': {
                         'format': True,
                         'indent': 2,
@@ -423,9 +417,6 @@ class TestFormatOptions:
                         'indent': 4,
                         'format': True
                     },
-                    'response': {
-                        'as': "''",
-                    },
                     'xml': {
                         'format': True,
                         'indent': 2,
@@ -444,7 +435,6 @@ class TestFormatOptions:
             (
                 [
                     '--format-options=json.indent:2',
-                    '--format-options=response.as:application/xml; charset=utf-8',
                     '--format-options=xml.format:false',
                     '--format-options=xml.indent:4',
                     '--unsorted',
@@ -459,9 +449,6 @@ class TestFormatOptions:
                         'indent': 2,
                         'format': True
                     },
-                    'response': {
-                        'as': 'application/xml; charset=utf-8',
-                    },
                     'xml': {
                         'format': False,
                         'indent': 4,
@@ -483,9 +470,6 @@ class TestFormatOptions:
                         'indent': 2,
                         'format': True
                     },
-                    'response': {
-                        'as': "''",
-                    },
                     'xml': {
                         'format': True,
                         'indent': 2,
@@ -508,9 +492,6 @@ class TestFormatOptions:
                         'indent': 2,
                         'format': True
                     },
-                    'response': {
-                        'as': "''",
-                    },
                     'xml': {
                         'format': True,
                         'indent': 2,

tests/test_unicode.py

@@ -142,31 +142,29 @@ def test_GET_encoding_detection_from_content(encoding):
     assert 'Financiën' in r
 
 
+@pytest.mark.parametrize('pretty', PRETTY_MAP.keys())
 @responses.activate
-def test_GET_encoding_provided_by_format_options():
+def test_GET_encoding_provided_by_option(pretty):
     responses.add(responses.GET,
                   URL_EXAMPLE,
-                  body='▒▒▒'.encode('johab'),
+                  body='卷首'.encode('big5'),
-                  content_type='text/plain')
+                  content_type='text/plain; charset=utf-8')
-    r = http('--format-options', 'response.as:text/plain; charset=johab',
+    args = ('--pretty=' + pretty, 'GET', URL_EXAMPLE)
-             'GET', URL_EXAMPLE)
-    assert '▒▒▒' in r
 
+    # Encoding provided by Content-Type is incorrect, thus it should print something unreadable.
+    r = http(*args)
+    assert '卷首' not in r
 
-@responses.activate
+    # Specifying the correct encoding, both in short & long versions, should fix it.
-def test_GET_encoding_provided_by_shortcut_option():
+    r = http('--response-as', 'charset=big5', *args)
-    responses.add(responses.GET,
+    assert '卷首' in r
-                  URL_EXAMPLE,
+    r = http('--response-as', 'text/plain; charset=big5', *args)
-                  body='▒▒▒'.encode('johab'),
+    assert '卷首' in r
-                  content_type='text/plain')
-    r = http('--response-as', 'text/plain; charset=johab',
-             'GET', URL_EXAMPLE)
-    assert '▒▒▒' in r
 
 
 @pytest.mark.parametrize('encoding', ENCODINGS)
 @responses.activate
-def test_GET_encoding_provided_by_empty_shortcut_option_should_use_content_detection(encoding):
+def test_GET_encoding_provided_by_empty_option_should_use_content_detection(encoding):
     body = f'<?xml version="1.0" encoding="{encoding.upper()}"?>\n<c>Financiën</c>'
     responses.add(responses.GET,
                   URL_EXAMPLE,

tests/test_xml.py

@@ -87,29 +87,9 @@ def test_invalid_xml(file):
 
 
 @responses.activate
-def test_content_type_from_format_options_argument():
+def test_content_type_from_option():
     """Test XML response with a incorrect Content-Type header.
-    Using the --format-options to force the good one.
+    Using the --response-as option to force the good one.
-    """
-    responses.add(responses.GET, URL_EXAMPLE, body=XML_DATA_RAW,
-                  content_type='plain/text')
-    args = ('--format-options', 'response.as:application/xml',
-            URL_EXAMPLE)
-
-    # Ensure the option is taken into account only for responses.
-    # Request
-    r = http('--offline', '--raw', XML_DATA_RAW, *args)
-    assert XML_DATA_RAW in r
-
-    # Response
-    r = http(*args)
-    assert XML_DATA_FORMATTED in r
-
-
-@responses.activate
-def test_content_type_from_shortcut_argument():
-    """Test XML response with a incorrect Content-Type header.
-    Using the --format-options shortcut to force the good one.
     """
     responses.add(responses.GET, URL_EXAMPLE, body=XML_DATA_RAW,
                   content_type='text/plain')
@@ -126,9 +106,9 @@ def test_content_type_from_shortcut_argument():
 
 
 @responses.activate
-def test_content_type_from_incomplete_format_options_argument():
+def test_content_type_from_option_incomplete():
     """Test XML response with a incorrect Content-Type header.
-    Using the --format-options to use a partial Content-Type without mime type.
+    Using the --response-as option to set a partial Content-Type without mime type.
     """
     responses.add(responses.GET, URL_EXAMPLE, body=XML_DATA_RAW,
                   content_type='text/plain')