From 0b5f4d6b1c1e71fd2fe3d80ab554bec7dc070562 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Schoentgen?= Date: Thu, 30 Sep 2021 10:58:08 +0200 Subject: [PATCH] Fix encoding error with non-prettified encoded responses Removed `--format-option response.as` an promote `--response-as`: using the format option would be misleading as it is now also used by non-prettified responses. --- CHANGELOG.md | 2 +- docs/README.md | 24 +++++++++--------- httpie/cli/argparser.py | 2 -- httpie/cli/constants.py | 2 -- httpie/cli/definition.py | 5 +--- httpie/output/processing.py | 1 - httpie/output/streams.py | 49 ++++++++++++++++++++----------------- httpie/output/writer.py | 28 ++++++++++----------- tests/test_output.py | 19 -------------- tests/test_unicode.py | 30 +++++++++++------------ tests/test_xml.py | 28 +++------------------ 11 files changed, 72 insertions(+), 118 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index df49ee28..ea7f96dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). - Added support for formatting & coloring of JSON bodies preceded by non-JSON data (e.g., an XXSI prefix). ([#1130](https://github.com/httpie/httpie/issues/1130)) - Added `--format-options=response.as:CONTENT_TYPE` to allow overriding the response `Content-Type`. ([#1134](https://github.com/httpie/httpie/issues/1134)) - Added `--response-as` shortcut for setting the response `Content-Type`-related `--format-options`. ([#1134](https://github.com/httpie/httpie/issues/1134)) -- Improved handling of prettified responses without correct `Content-Type` encoding. ([#1110](https://github.com/httpie/httpie/issues/1110)) +- Improved handling of responses without correct `Content-Type` encoding. ([#1110](https://github.com/httpie/httpie/issues/1110), [#1168](https://github.com/httpie/httpie/issues/1168)) - Installed plugins are now listed in `--debug` output. ([#1165](https://github.com/httpie/httpie/issues/1165)) - Fixed duplicate keys preservation of JSON data. ([#1163](https://github.com/httpie/httpie/issues/1163)) diff --git a/docs/README.md b/docs/README.md index ed315370..0d9ff39f 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1214,15 +1214,14 @@ You can further control the applied formatting via the more granular [format opt The `--format-options=opt1:value,opt2:value` option allows you to control how the output should be formatted when formatting is applied. The following options are available: -| Option | Default value | Shortcuts | -| ---------------: | :-----------: | ----------------------------------------- | -| `headers.sort` | `true` | `--sorted`, `--unsorted` | -| `json.format` | `true` | N/A | -| `json.indent` | `4` | N/A | -| `json.sort_keys` | `true` | `--sorted`, `--unsorted` | -| `response.as` | `''` | [`--response-as`](#response-content-type) | -| `xml.format` | `true` | N/A | -| `xml.indent` | `2` | N/A | +| Option | Default value | Shortcuts | +| ---------------: | :-----------: | ------------------------ | +| `headers.sort` | `true` | `--sorted`, `--unsorted` | +| `json.format` | `true` | N/A | +| `json.indent` | `4` | N/A | +| `json.sort_keys` | `true` | `--sorted`, `--unsorted` | +| `xml.format` | `true` | N/A | +| `xml.indent` | `2` | N/A | For example, this is how you would disable the default header and JSON key sorting, and specify a custom JSON indent size: @@ -1237,11 +1236,10 @@ sorting-related format options (currently it means JSON keys and headers): This is something you will typically store as one of the default options in your [config](#config) file. -#### Response `Content-Type` +### Response `Content-Type` -The `--response-as=value` option is a shortcut for `--format-options response.as:value`, -and it allows you to override the response `Content-Type` sent by the server. -That makes it possible for HTTPie to pretty-print the response even when the server specifies the type incorrectly. +The `--response-as=value` option allows you to override the response `Content-Type` sent by the server. +That makes it possible for HTTPie to print the response even when the server specifies the type incorrectly. For example, the following request will force the response to be treated as XML: diff --git a/httpie/cli/argparser.py b/httpie/cli/argparser.py index 5739e9d3..0b689410 100644 --- a/httpie/cli/argparser.py +++ b/httpie/cli/argparser.py @@ -458,8 +458,6 @@ class HTTPieArgumentParser(argparse.ArgumentParser): def _process_format_options(self): format_options = self.args.format_options or [] - if self.args.response_as is not None: - format_options.append('response.as:' + self.args.response_as) parsed_options = PARSED_DEFAULT_FORMAT_OPTIONS for options_group in format_options: parsed_options = parse_format_options(options_group, defaults=parsed_options) diff --git a/httpie/cli/constants.py b/httpie/cli/constants.py index 969a2d60..a1b78d33 100644 --- a/httpie/cli/constants.py +++ b/httpie/cli/constants.py @@ -85,13 +85,11 @@ PRETTY_MAP = { PRETTY_STDOUT_TTY_ONLY = object() -EMPTY_FORMAT_OPTION = "''" DEFAULT_FORMAT_OPTIONS = [ 'headers.sort:true', 'json.format:true', 'json.indent:4', 'json.sort_keys:true', - 'response.as:' + EMPTY_FORMAT_OPTION, 'xml.format:true', 'xml.indent:2', ] diff --git a/httpie/cli/definition.py b/httpie/cli/definition.py index 77be93a9..eccfc44b 100644 --- a/httpie/cli/definition.py +++ b/httpie/cli/definition.py @@ -313,15 +313,12 @@ output_processing.add_argument( '--response-as', metavar='CONTENT_TYPE', help=''' - Override the response Content-Type for formatting purposes, e.g.: + Override the response Content-Type for display purposes, e.g.: --response-as=application/xml --response-as=charset=utf-8 --response-as='application/xml; charset=utf-8' - It is a shortcut for: - - --format-options=response.as:CONTENT_TYPE ''' ) diff --git a/httpie/output/processing.py b/httpie/output/processing.py index 25d9c45a..ddee9ca9 100644 --- a/httpie/output/processing.py +++ b/httpie/output/processing.py @@ -33,7 +33,6 @@ class Formatting: :param kwargs: additional keyword arguments for processors """ - self.options = kwargs['format_options'] available_plugins = plugin_manager.get_formatters_grouped() self.enabled_plugins = [] for group in groups: diff --git a/httpie/output/streams.py b/httpie/output/streams.py index 1c6afaa8..e53998c3 100644 --- a/httpie/output/streams.py +++ b/httpie/output/streams.py @@ -1,12 +1,11 @@ from abc import ABCMeta, abstractmethod from itertools import chain -from typing import Any, Callable, Dict, Iterable, Tuple, Union +from typing import Callable, Dict, Iterable, Tuple, Union from .. import codec -from ..cli.constants import EMPTY_FORMAT_OPTION from ..context import Environment from ..constants import UTF8 -from ..models import HTTPMessage, HTTPResponse +from ..models import HTTPMessage, HTTPRequest from .processing import Conversion, Formatting from .utils import parse_header_content_type @@ -100,8 +99,11 @@ class EncodedStream(BaseStream): """ CHUNK_SIZE = 1 - def __init__(self, env=Environment(), **kwargs): + def __init__(self, env=Environment(), response_as: str = None, **kwargs): super().__init__(**kwargs) + self.response_as = response_as + self.mime, self.encoding = self._get_mime_and_encoding() + if env.stdout_isatty: # Use the encoding supported by the terminal. output_encoding = env.stdout_encoding @@ -111,11 +113,31 @@ class EncodedStream(BaseStream): # Default to UTF-8 when unsure. self.output_encoding = output_encoding or UTF8 + def _get_mime_and_encoding(self) -> Tuple[str, Dict[str, str]]: + """Parse `Content-Type` header or `--response-as` value to guess + correct mime type and encoding. + + """ + # Defaults from the `Content-Type` header. + mime, options = parse_header_content_type(self.msg.content_type) + + if isinstance(self.msg, HTTPRequest): + encoding = self.msg.encoding + elif self.response_as is None: + encoding = options.get('charset') + else: + # Override from the `--response-as` option. + forced_mime, forced_options = parse_header_content_type(self.response_as) + mime = forced_mime or mime + encoding = forced_options.get('charset') or options.get('charset') + + return mime, encoding or '' + def iter_body(self) -> Iterable[bytes]: for line, lf in self.msg.iter_lines(self.CHUNK_SIZE): if b'\0' in line: raise BinarySuppressedError() - line = codec.decode(line, self.msg.encoding) + line = codec.decode(line, self.encoding) yield codec.encode(line, self.output_encoding) + lf @@ -138,23 +160,6 @@ class PrettyStream(EncodedStream): super().__init__(**kwargs) self.formatting = formatting self.conversion = conversion - self.mime, mime_options = self._get_mime_and_options() - self.encoding = mime_options.get('charset') or '' - - def _get_mime_and_options(self) -> Tuple[str, Dict[str, Any]]: - # Defaults from the `Content-Type` header. - mime, options = parse_header_content_type(self.msg.content_type) - - if not isinstance(self.msg, HTTPResponse): - return mime, options - - # Override from the `--response-as` option. - forced_content_type = self.formatting.options['response']['as'] - if forced_content_type == EMPTY_FORMAT_OPTION: - return mime, options - - forced_mime, forced_options = parse_header_content_type(forced_content_type) - return (forced_mime or mime, forced_options or options) def get_headers(self) -> bytes: return self.formatting.format_headers( diff --git a/httpie/output/writer.py b/httpie/output/writer.py index d239e503..05b389eb 100644 --- a/httpie/output/writer.py +++ b/httpie/output/writer.py @@ -134,23 +134,23 @@ def get_stream_type_and_kwargs( else RawStream.CHUNK_SIZE ) } - elif args.prettify: - stream_class = PrettyStream if args.stream else BufferedPrettyStream - stream_kwargs = { - 'env': env, - 'conversion': Conversion(), - 'formatting': Formatting( - env=env, - groups=args.prettify, - color_scheme=args.style, - explicit_json=args.json, - format_options=args.format_options, - ) - } else: stream_class = EncodedStream stream_kwargs = { - 'env': env + 'env': env, + 'response_as': args.response_as, } + if args.prettify: + stream_class = PrettyStream if args.stream else BufferedPrettyStream + stream_kwargs.update({ + 'conversion': Conversion(), + 'formatting': Formatting( + env=env, + groups=args.prettify, + color_scheme=args.style, + explicit_json=args.json, + format_options=args.format_options, + ) + }) return stream_class, stream_kwargs diff --git a/tests/test_output.py b/tests/test_output.py index dfa20228..0d0ee2f0 100644 --- a/tests/test_output.py +++ b/tests/test_output.py @@ -377,9 +377,6 @@ class TestFormatOptions: 'indent': 10, 'format': True }, - 'response': { - 'as': "''", - }, 'xml': { 'format': True, 'indent': 2, @@ -399,9 +396,6 @@ class TestFormatOptions: 'indent': 4, 'format': True }, - 'response': { - 'as': "''", - }, 'xml': { 'format': True, 'indent': 2, @@ -423,9 +417,6 @@ class TestFormatOptions: 'indent': 4, 'format': True }, - 'response': { - 'as': "''", - }, 'xml': { 'format': True, 'indent': 2, @@ -444,7 +435,6 @@ class TestFormatOptions: ( [ '--format-options=json.indent:2', - '--format-options=response.as:application/xml; charset=utf-8', '--format-options=xml.format:false', '--format-options=xml.indent:4', '--unsorted', @@ -459,9 +449,6 @@ class TestFormatOptions: 'indent': 2, 'format': True }, - 'response': { - 'as': 'application/xml; charset=utf-8', - }, 'xml': { 'format': False, 'indent': 4, @@ -483,9 +470,6 @@ class TestFormatOptions: 'indent': 2, 'format': True }, - 'response': { - 'as': "''", - }, 'xml': { 'format': True, 'indent': 2, @@ -508,9 +492,6 @@ class TestFormatOptions: 'indent': 2, 'format': True }, - 'response': { - 'as': "''", - }, 'xml': { 'format': True, 'indent': 2, diff --git a/tests/test_unicode.py b/tests/test_unicode.py index 2a12180e..b1267592 100644 --- a/tests/test_unicode.py +++ b/tests/test_unicode.py @@ -142,31 +142,29 @@ def test_GET_encoding_detection_from_content(encoding): assert 'Financiën' in r +@pytest.mark.parametrize('pretty', PRETTY_MAP.keys()) @responses.activate -def test_GET_encoding_provided_by_format_options(): +def test_GET_encoding_provided_by_option(pretty): responses.add(responses.GET, URL_EXAMPLE, - body='▒▒▒'.encode('johab'), - content_type='text/plain') - r = http('--format-options', 'response.as:text/plain; charset=johab', - 'GET', URL_EXAMPLE) - assert '▒▒▒' in r + body='卷首'.encode('big5'), + content_type='text/plain; charset=utf-8') + args = ('--pretty=' + pretty, 'GET', URL_EXAMPLE) + # Encoding provided by Content-Type is incorrect, thus it should print something unreadable. + r = http(*args) + assert '卷首' not in r -@responses.activate -def test_GET_encoding_provided_by_shortcut_option(): - responses.add(responses.GET, - URL_EXAMPLE, - body='▒▒▒'.encode('johab'), - content_type='text/plain') - r = http('--response-as', 'text/plain; charset=johab', - 'GET', URL_EXAMPLE) - assert '▒▒▒' in r + # Specifying the correct encoding, both in short & long versions, should fix it. + r = http('--response-as', 'charset=big5', *args) + assert '卷首' in r + r = http('--response-as', 'text/plain; charset=big5', *args) + assert '卷首' in r @pytest.mark.parametrize('encoding', ENCODINGS) @responses.activate -def test_GET_encoding_provided_by_empty_shortcut_option_should_use_content_detection(encoding): +def test_GET_encoding_provided_by_empty_option_should_use_content_detection(encoding): body = f'\nFinanciën' responses.add(responses.GET, URL_EXAMPLE, diff --git a/tests/test_xml.py b/tests/test_xml.py index 43a6d8fb..5727c94a 100644 --- a/tests/test_xml.py +++ b/tests/test_xml.py @@ -87,29 +87,9 @@ def test_invalid_xml(file): @responses.activate -def test_content_type_from_format_options_argument(): +def test_content_type_from_option(): """Test XML response with a incorrect Content-Type header. - Using the --format-options to force the good one. - """ - responses.add(responses.GET, URL_EXAMPLE, body=XML_DATA_RAW, - content_type='plain/text') - args = ('--format-options', 'response.as:application/xml', - URL_EXAMPLE) - - # Ensure the option is taken into account only for responses. - # Request - r = http('--offline', '--raw', XML_DATA_RAW, *args) - assert XML_DATA_RAW in r - - # Response - r = http(*args) - assert XML_DATA_FORMATTED in r - - -@responses.activate -def test_content_type_from_shortcut_argument(): - """Test XML response with a incorrect Content-Type header. - Using the --format-options shortcut to force the good one. + Using the --response-as option to force the good one. """ responses.add(responses.GET, URL_EXAMPLE, body=XML_DATA_RAW, content_type='text/plain') @@ -126,9 +106,9 @@ def test_content_type_from_shortcut_argument(): @responses.activate -def test_content_type_from_incomplete_format_options_argument(): +def test_content_type_from_option_incomplete(): """Test XML response with a incorrect Content-Type header. - Using the --format-options to use a partial Content-Type without mime type. + Using the --response-as option to set a partial Content-Type without mime type. """ responses.add(responses.GET, URL_EXAMPLE, body=XML_DATA_RAW, content_type='text/plain')