extern/httpie-cli
1
0
Fork 1
mirror of https://github.com/httpie/cli.git synced 2025-06-24 19:41:23 +02:00
Code Issues Packages Projects Releases Wiki Activity

Fix encoding error with non-prettified encoded responses

Browse Source 
Removed `--format-option response.as` an promote `--response-as`: using
the format option would be misleading as it is now also used by non-prettified
responses.
This commit is contained in:
Mickaël Schoentgen 2021-09-30 10:58:08 +02:00
parent 71adcd97d0
commit 0b5f4d6b1c
11 changed files with 72 additions and 118 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CHANGELOG.md
View File

@ -8,7 +8,7 @@ This project adheres to [Semantic Versioning](https://semver.org/).
- Added support for formatting & coloring of JSON bodies preceded by non-JSON data (e.g., an XXSI prefix). ([#1130](https://github.com/httpie/httpie/issues/1130))
- Added `--format-options=response.as:CONTENT_TYPE` to allow overriding the response `Content-Type`. ([#1134](https://github.com/httpie/httpie/issues/1134))
- Added `--response-as` shortcut for setting the response `Content-Type`-related `--format-options`. ([#1134](https://github.com/httpie/httpie/issues/1134))
- Improved handling of prettified responses without correct `Content-Type` encoding. ([#1110](https://github.com/httpie/httpie/issues/1110))
- Improved handling of responses without correct `Content-Type` encoding. ([#1110](https://github.com/httpie/httpie/issues/1110), [#1168](https://github.com/httpie/httpie/issues/1168))
- Installed plugins are now listed in `--debug` output. ([#1165](https://github.com/httpie/httpie/issues/1165))
- Fixed duplicate keys preservation of JSON data. ([#1163](https://github.com/httpie/httpie/issues/1163))

24
docs/README.md
View File

@ -1214,15 +1214,14 @@ You can further control the applied formatting via the more granular [format opt
The `--format-options=opt1:value,opt2:value` option allows you to control how the output should be formatted
when formatting is applied. The following options are available:
| Option | Default value | Shortcuts |
| ---------------: | :-----------: | ----------------------------------------- |
| `headers.sort` | `true` | `--sorted`, `--unsorted` |
| `json.format` | `true` | N/A |
| `json.indent` | `4` | N/A |
| `json.sort_keys` | `true` | `--sorted`, `--unsorted` |
| `response.as` | `''` | [`--response-as`](#response-content-type) |
| `xml.format` | `true` | N/A |
| `xml.indent` | `2` | N/A |
| Option | Default value | Shortcuts |
| ---------------: | :-----------: | ------------------------ |
| `headers.sort` | `true` | `--sorted`, `--unsorted` |
| `json.format` | `true` | N/A |
| `json.indent` | `4` | N/A |
| `json.sort_keys` | `true` | `--sorted`, `--unsorted` |
| `xml.format` | `true` | N/A |
| `xml.indent` | `2` | N/A |
For example, this is how you would disable the default header and JSON key
sorting, and specify a custom JSON indent size:
@ -1237,11 +1236,10 @@ sorting-related format options (currently it means JSON keys and headers):
This is something you will typically store as one of the default options in your [config](#config) file.
#### Response `Content-Type`
### Response `Content-Type`
The `--response-as=value` option is a shortcut for `--format-options response.as:value`,
and it allows you to override the response `Content-Type` sent by the server.
That makes it possible for HTTPie to pretty-print the response even when the server specifies the type incorrectly.
The `--response-as=value` option allows you to override the response `Content-Type` sent by the server.
That makes it possible for HTTPie to print the response even when the server specifies the type incorrectly.
For example, the following request will force the response to be treated as XML:

2
httpie/cli/argparser.py
View File

@ -458,8 +458,6 @@ class HTTPieArgumentParser(argparse.ArgumentParser):
def _process_format_options(self):
format_options = self.args.format_options or []
if self.args.response_as is not None:
format_options.append('response.as:' + self.args.response_as)
parsed_options = PARSED_DEFAULT_FORMAT_OPTIONS
for options_group in format_options:
parsed_options = parse_format_options(options_group, defaults=parsed_options)

2
httpie/cli/constants.py
View File

@ -85,13 +85,11 @@ PRETTY_MAP = {
PRETTY_STDOUT_TTY_ONLY = object()
EMPTY_FORMAT_OPTION = "''"
DEFAULT_FORMAT_OPTIONS = [
'headers.sort:true',
'json.format:true',
'json.indent:4',
'json.sort_keys:true',
'response.as:' + EMPTY_FORMAT_OPTION,
'xml.format:true',
'xml.indent:2',
]

5
httpie/cli/definition.py
View File

@ -313,15 +313,12 @@ output_processing.add_argument(
'--response-as',
metavar='CONTENT_TYPE',
help='''
Override the response Content-Type for formatting purposes, e.g.:
Override the response Content-Type for display purposes, e.g.:
--response-as=application/xml
--response-as=charset=utf-8
--response-as='application/xml; charset=utf-8'
It is a shortcut for:
--format-options=response.as:CONTENT_TYPE
'''
)

1
httpie/output/processing.py
View File

@ -33,7 +33,6 @@ class Formatting:
:param kwargs: additional keyword arguments for processors
"""
self.options = kwargs['format_options']
available_plugins = plugin_manager.get_formatters_grouped()
self.enabled_plugins = []
for group in groups:

49
httpie/output/streams.py
View File

@ -1,12 +1,11 @@
from abc import ABCMeta, abstractmethod
from itertools import chain
from typing import Any, Callable, Dict, Iterable, Tuple, Union
from typing import Callable, Dict, Iterable, Tuple, Union
from .. import codec
from ..cli.constants import EMPTY_FORMAT_OPTION
from ..context import Environment
from ..constants import UTF8
from ..models import HTTPMessage, HTTPResponse
from ..models import HTTPMessage, HTTPRequest
from .processing import Conversion, Formatting
from .utils import parse_header_content_type
@ -100,8 +99,11 @@ class EncodedStream(BaseStream):
"""
CHUNK_SIZE = 1
def __init__(self, env=Environment(), **kwargs):
def __init__(self, env=Environment(), response_as: str = None, **kwargs):
super().__init__(**kwargs)
self.response_as = response_as
self.mime, self.encoding = self._get_mime_and_encoding()
if env.stdout_isatty:
# Use the encoding supported by the terminal.
output_encoding = env.stdout_encoding
@ -111,11 +113,31 @@ class EncodedStream(BaseStream):
# Default to UTF-8 when unsure.
self.output_encoding = output_encoding or UTF8
def _get_mime_and_encoding(self) -> Tuple[str, Dict[str, str]]:
"""Parse `Content-Type` header or `--response-as` value to guess
correct mime type and encoding.
"""
# Defaults from the `Content-Type` header.
mime, options = parse_header_content_type(self.msg.content_type)
if isinstance(self.msg, HTTPRequest):
encoding = self.msg.encoding
elif self.response_as is None:
encoding = options.get('charset')
else:
# Override from the `--response-as` option.
forced_mime, forced_options = parse_header_content_type(self.response_as)
mime = forced_mime or mime
encoding = forced_options.get('charset') or options.get('charset')
return mime, encoding or ''
def iter_body(self) -> Iterable[bytes]:
for line, lf in self.msg.iter_lines(self.CHUNK_SIZE):
if b'\0' in line:
raise BinarySuppressedError()
line = codec.decode(line, self.msg.encoding)
line = codec.decode(line, self.encoding)
yield codec.encode(line, self.output_encoding) + lf
@ -138,23 +160,6 @@ class PrettyStream(EncodedStream):
super().__init__(**kwargs)
self.formatting = formatting
self.conversion = conversion
self.mime, mime_options = self._get_mime_and_options()
self.encoding = mime_options.get('charset') or ''
def _get_mime_and_options(self) -> Tuple[str, Dict[str, Any]]:
# Defaults from the `Content-Type` header.
mime, options = parse_header_content_type(self.msg.content_type)
if not isinstance(self.msg, HTTPResponse):
return mime, options
# Override from the `--response-as` option.
forced_content_type = self.formatting.options['response']['as']
if forced_content_type == EMPTY_FORMAT_OPTION:
return mime, options
forced_mime, forced_options = parse_header_content_type(forced_content_type)
return (forced_mime or mime, forced_options or options)
def get_headers(self) -> bytes:
return self.formatting.format_headers(

28
httpie/output/writer.py
View File

@ -134,23 +134,23 @@ def get_stream_type_and_kwargs(
else RawStream.CHUNK_SIZE
)
}
elif args.prettify:
stream_class = PrettyStream if args.stream else BufferedPrettyStream
stream_kwargs = {
'env': env,
'conversion': Conversion(),
'formatting': Formatting(
env=env,
groups=args.prettify,
color_scheme=args.style,
explicit_json=args.json,
format_options=args.format_options,
)
}
else:
stream_class = EncodedStream
stream_kwargs = {
'env': env
'env': env,
'response_as': args.response_as,
}
if args.prettify:
stream_class = PrettyStream if args.stream else BufferedPrettyStream
stream_kwargs.update({
'conversion': Conversion(),
'formatting': Formatting(
env=env,
groups=args.prettify,
color_scheme=args.style,
explicit_json=args.json,
format_options=args.format_options,
)
})
return stream_class, stream_kwargs

19
tests/test_output.py
View File

@ -377,9 +377,6 @@ class TestFormatOptions:
'indent': 10,
'format': True
},
'response': {
'as': "''",
},
'xml': {
'format': True,
'indent': 2,
@ -399,9 +396,6 @@ class TestFormatOptions:
'indent': 4,
'format': True
},
'response': {
'as': "''",
},
'xml': {
'format': True,
'indent': 2,
@ -423,9 +417,6 @@ class TestFormatOptions:
'indent': 4,
'format': True
},
'response': {
'as': "''",
},
'xml': {
'format': True,
'indent': 2,
@ -444,7 +435,6 @@ class TestFormatOptions:
(
[
'--format-options=json.indent:2',
'--format-options=response.as:application/xml; charset=utf-8',
'--format-options=xml.format:false',
'--format-options=xml.indent:4',
'--unsorted',
@ -459,9 +449,6 @@ class TestFormatOptions:
'indent': 2,
'format': True
},
'response': {
'as': 'application/xml; charset=utf-8',
},
'xml': {
'format': False,
'indent': 4,
@ -483,9 +470,6 @@ class TestFormatOptions:
'indent': 2,
'format': True
},
'response': {
'as': "''",
},
'xml': {
'format': True,
'indent': 2,
@ -508,9 +492,6 @@ class TestFormatOptions:
'indent': 2,
'format': True
},
'response': {
'as': "''",
},
'xml': {
'format': True,
'indent': 2,

30
tests/test_unicode.py
View File

@ -142,31 +142,29 @@ def test_GET_encoding_detection_from_content(encoding):
assert 'Financiën' in r
@pytest.mark.parametrize('pretty', PRETTY_MAP.keys())
@responses.activate
def test_GET_encoding_provided_by_format_options():
def test_GET_encoding_provided_by_option(pretty):
responses.add(responses.GET,
URL_EXAMPLE,
body='▒▒▒'.encode('johab'),
content_type='text/plain')
r = http('--format-options', 'response.as:text/plain; charset=johab',
'GET', URL_EXAMPLE)
assert '▒▒▒' in r
body='卷首'.encode('big5'),
content_type='text/plain; charset=utf-8')
args = ('--pretty=' + pretty, 'GET', URL_EXAMPLE)
# Encoding provided by Content-Type is incorrect, thus it should print something unreadable.
r = http(*args)
assert '卷首' not in r
@responses.activate
def test_GET_encoding_provided_by_shortcut_option():
responses.add(responses.GET,
URL_EXAMPLE,
body='▒▒▒'.encode('johab'),
content_type='text/plain')
r = http('--response-as', 'text/plain; charset=johab',
'GET', URL_EXAMPLE)
assert '▒▒▒' in r
# Specifying the correct encoding, both in short & long versions, should fix it.
r = http('--response-as', 'charset=big5', *args)
assert '卷首' in r
r = http('--response-as', 'text/plain; charset=big5', *args)
assert '卷首' in r
@pytest.mark.parametrize('encoding', ENCODINGS)
@responses.activate
def test_GET_encoding_provided_by_empty_shortcut_option_should_use_content_detection(encoding):
def test_GET_encoding_provided_by_empty_option_should_use_content_detection(encoding):
body = f'<?xml version="1.0" encoding="{encoding.upper()}"?>\n<c>Financiën</c>'
responses.add(responses.GET,
URL_EXAMPLE,

28
tests/test_xml.py
View File

@ -87,29 +87,9 @@ def test_invalid_xml(file):
@responses.activate
def test_content_type_from_format_options_argument():
def test_content_type_from_option():
"""Test XML response with a incorrect Content-Type header.
Using the --format-options to force the good one.
"""
responses.add(responses.GET, URL_EXAMPLE, body=XML_DATA_RAW,
content_type='plain/text')
args = ('--format-options', 'response.as:application/xml',
URL_EXAMPLE)
# Ensure the option is taken into account only for responses.
# Request
r = http('--offline', '--raw', XML_DATA_RAW, *args)
assert XML_DATA_RAW in r
# Response
r = http(*args)
assert XML_DATA_FORMATTED in r
@responses.activate
def test_content_type_from_shortcut_argument():
"""Test XML response with a incorrect Content-Type header.
Using the --format-options shortcut to force the good one.
Using the --response-as option to force the good one.
"""
responses.add(responses.GET, URL_EXAMPLE, body=XML_DATA_RAW,
content_type='text/plain')
@ -126,9 +106,9 @@ def test_content_type_from_shortcut_argument():
@responses.activate
def test_content_type_from_incomplete_format_options_argument():
def test_content_type_from_option_incomplete():
"""Test XML response with a incorrect Content-Type header.
Using the --format-options to use a partial Content-Type without mime type.
Using the --response-as option to set a partial Content-Type without mime type.
"""
responses.add(responses.GET, URL_EXAMPLE, body=XML_DATA_RAW,
content_type='text/plain')