Fix encoding error with non-prettified encoded responses (#1168)

* Fix encoding error with non-prettified encoded responses

Removed `--format-option response.as` an promote `--response-as`: using
the format option would be misleading as it is now also used by non-prettified
responses.

* Encoding refactoring

* split --response-as into --response-mime and --response-charset
* add support for Content-Type charset for requests printed to terminal
* add support charset detection for requests printed to terminal without a Content-Type charset
* etc.

* `test_unicode.py` → `test_encoding.py`

* Drop sequence length check

* Clean-up tests

* [skip ci] Tweaks

* Use the compatible release clause for `charset_normalizer` requirement

Cf. https://www.python.org/dev/peps/pep-0440/#version-specifiers

* Clean-up

* Partially revert d52a4833e4

* Changelog

* Tweak tests

* [skip ci] Better test name

* Cleanup tests and add request body charset detection

* More test suite cleanups

* Cleanup

* Fix code style in test

* Improve detect_encoding() docstring

* Uniformize pytest.mark.parametrize() calls

* [skip ci] Comment out TODOs (will be tackled in a specific PR)

Co-authored-by: Jakub Roztocil <jakub@roztocil.co>

tests/test_json.py

@@ -9,10 +9,28 @@ from httpie.output.formatters.colors import ColorFormatter
 from httpie.utils import JsonDictPreservingDuplicateKeys
 
 from .fixtures import JSON_WITH_DUPE_KEYS_FILE_PATH
-from .utils import MockEnvironment, http, URL_EXAMPLE
+from .utils import MockEnvironment, http, DUMMY_URL
 
-TEST_JSON_XXSI_PREFIXES = (r")]}',\n", ")]}',", 'while(1);', 'for(;;)', ')', ']', '}')
-TEST_JSON_VALUES = ({}, {'a': 0, 'b': 0}, [], ['a', 'b'], 'foo', True, False, None)  # FIX: missing int & float
+TEST_JSON_XXSI_PREFIXES = [
+    r")]}',\n",
+    ")]}',",
+    'while(1);',
+    'for(;;)',
+    ')',
+    ']',
+    '}',
+]
+TEST_JSON_VALUES = [
+    # FIXME: missing int & float
+    {},
+    {'a': 0, 'b': 0},
+    [],
+    ['a', 'b'],
+    'foo',
+    True,
+    False,
+    None,
+]
 TEST_PREFIX_TOKEN_COLOR = '\x1b[38;5;15m' if is_windows else '\x1b[04m\x1b[91m'
 
 JSON_WITH_DUPES_RAW = '{"key": 15, "key": 15, "key": 3, "key": 7}'
@@ -37,15 +55,19 @@ JSON_WITH_DUPES_FORMATTED_UNSORTED = '''{
 def test_json_formatter_with_body_preceded_by_non_json_data(data_prefix, json_data, pretty):
     """Test JSON bodies preceded by non-JSON data."""
     body = data_prefix + json.dumps(json_data)
-    content_type = 'application/json'
-    responses.add(responses.GET, URL_EXAMPLE, body=body,
-                  content_type=content_type)
+    content_type = 'application/json;charset=utf8'
+    responses.add(
+        responses.GET,
+        DUMMY_URL,
+        body=body,
+        content_type=content_type,
+    )
 
-    colored_output = pretty in ('all', 'colors')
+    colored_output = pretty in {'all', 'colors'}
     env = MockEnvironment(colors=256) if colored_output else None
-    r = http('--pretty=' + pretty, URL_EXAMPLE, env=env)
+    r = http('--pretty', pretty, DUMMY_URL, env=env)
 
-    indent = None if pretty in ('none', 'colors') else 4
+    indent = None if pretty in {'none', 'colors'} else 4
     expected_body = data_prefix + json.dumps(json_data, indent=indent)
     if colored_output:
         fmt = ColorFormatter(env, format_options={'json': {'format': True, 'indent': 4}})
@@ -59,9 +81,13 @@ def test_json_formatter_with_body_preceded_by_non_json_data(data_prefix, json_da
 @responses.activate
 def test_duplicate_keys_support_from_response():
     """JSON with duplicate keys should be handled correctly."""
-    responses.add(responses.GET, URL_EXAMPLE, body=JSON_WITH_DUPES_RAW,
-                  content_type='application/json')
-    args = ('--pretty', 'format', URL_EXAMPLE)
+    responses.add(
+        responses.GET,
+        DUMMY_URL,
+        body=JSON_WITH_DUPES_RAW,
+        content_type='application/json',
+    )
+    args = ('--pretty', 'format', DUMMY_URL)
 
     # Check implicit --sorted
     if JsonDictPreservingDuplicateKeys.SUPPORTS_SORTING:
@@ -75,8 +101,12 @@ def test_duplicate_keys_support_from_response():
 
 def test_duplicate_keys_support_from_input_file():
     """JSON file with duplicate keys should be handled correctly."""
-    args = ('--verbose', '--offline', URL_EXAMPLE,
-            f'@{JSON_WITH_DUPE_KEYS_FILE_PATH}')
+    args = (
+        '--verbose',
+        '--offline',
+        DUMMY_URL,
+        f'@{JSON_WITH_DUPE_KEYS_FILE_PATH}',
+    )
 
     # Check implicit --sorted
     if JsonDictPreservingDuplicateKeys.SUPPORTS_SORTING: