Uniformize UTF-8 naming (#1115)

* Uniformize UTF-8 naming

Replace `utf8` -> `utf-8` everywhere.
It should have no impact, `utf8` is an alias of `utf-8` [1].

[1] ee03bad25e/Lib/encodings/aliases.py (L534)

* Always specify the encoding

Let's be explicit over implicit. And prevent future warnings from PEP-597 [1].

[1] https://www.python.org/dev/peps/pep-0597/#using-the-default-encoding-is-a-common-mistake

* Update `UTF8` constant (`utf-8` -> `utf_8`)

* Remove default argument from `str.encode()` and `bytes.decode()`

* Clean-up
This commit is contained in:
Mickaël Schoentgen 2021-08-05 20:58:43 +02:00 committed by GitHub
parent 11399dde76
commit c6cbc7dfa5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
21 changed files with 54 additions and 42 deletions

View File

@ -301,7 +301,7 @@ class HTTPieArgumentParser(argparse.ArgumentParser):
""" """
self._ensure_one_data_source(self.has_stdin_data, self.args.data, self._ensure_one_data_source(self.has_stdin_data, self.args.data,
self.args.files) self.args.files)
self.args.data = data.encode('utf-8') self.args.data = data.encode()
def _ensure_one_data_source(self, *other_sources): def _ensure_one_data_source(self, *other_sources):
"""There can only be one source of input request data. """There can only be one source of input request data.

View File

@ -144,7 +144,7 @@ def load_text_file(item: KeyValueArg) -> str:
except UnicodeDecodeError: except UnicodeDecodeError:
raise ParseError( raise ParseError(
f'{item.orig!r}: cannot embed the content of {item.value!r},' f'{item.orig!r}: cannot embed the content of {item.value!r},'
' not a UTF8 or ASCII-encoded text file' ' not a UTF-8 or ASCII-encoded text file'
) )

View File

@ -24,7 +24,7 @@ from .utils import get_expired_cookies, repr_dict
urllib3.disable_warnings() urllib3.disable_warnings()
FORM_CONTENT_TYPE = 'application/x-www-form-urlencoded; charset=utf-8' FORM_CONTENT_TYPE = 'application/x-www-form-urlencoded; charset=UTF-8'
JSON_CONTENT_TYPE = 'application/json' JSON_CONTENT_TYPE = 'application/json'
JSON_ACCEPT = f'{JSON_CONTENT_TYPE}, */*;q=0.5' JSON_ACCEPT = f'{JSON_CONTENT_TYPE}, */*;q=0.5'
DEFAULT_UA = f'HTTPie/{__version__}' DEFAULT_UA = f'HTTPie/{__version__}'
@ -188,7 +188,7 @@ def finalize_headers(headers: RequestHeadersDict) -> RequestHeadersDict:
value = value.strip() value = value.strip()
if isinstance(value, str): if isinstance(value, str):
# See <https://github.com/httpie/httpie/issues/212> # See <https://github.com/httpie/httpie/issues/212>
value = value.encode('utf8') value = value.encode()
final_headers[name] = value final_headers[name] = value
return final_headers return final_headers

View File

@ -5,6 +5,7 @@ from typing import Union
from . import __version__ from . import __version__
from .compat import is_windows from .compat import is_windows
from .constants import UTF8
ENV_XDG_CONFIG_HOME = 'XDG_CONFIG_HOME' ENV_XDG_CONFIG_HOME = 'XDG_CONFIG_HOME'
@ -79,7 +80,7 @@ class BaseConfigDict(dict):
def load(self): def load(self):
config_type = type(self).__name__.lower() config_type = type(self).__name__.lower()
try: try:
with self.path.open() as f: with self.path.open(encoding=UTF8) as f:
try: try:
data = json.load(f) data = json.load(f)
except ValueError as e: except ValueError as e:
@ -111,7 +112,7 @@ class BaseConfigDict(dict):
ensure_ascii=True, ensure_ascii=True,
) )
try: try:
self.path.write_text(json_string + '\n') self.path.write_text(json_string + '\n', encoding=UTF8)
except OSError: except OSError:
if not fail_silently: if not fail_silently:
raise raise

2
httpie/constants.py Normal file
View File

@ -0,0 +1,2 @@
# UTF-8 encoding name
UTF8 = 'utf-8'

View File

@ -11,6 +11,7 @@ except ImportError:
from .compat import is_windows from .compat import is_windows
from .config import DEFAULT_CONFIG_DIR, Config, ConfigFileError from .config import DEFAULT_CONFIG_DIR, Config, ConfigFileError
from .constants import UTF8
from .utils import repr_dict from .utils import repr_dict
@ -70,10 +71,10 @@ class Environment:
self._orig_stderr = self.stderr self._orig_stderr = self.stderr
self._devnull = devnull self._devnull = devnull
# Keyword arguments > stream.encoding > default utf8 # Keyword arguments > stream.encoding > default UTF-8
if self.stdin and self.stdin_encoding is None: if self.stdin and self.stdin_encoding is None:
self.stdin_encoding = getattr( self.stdin_encoding = getattr(
self.stdin, 'encoding', None) or 'utf8' self.stdin, 'encoding', None) or UTF8
if self.stdout_encoding is None: if self.stdout_encoding is None:
actual_stdout = self.stdout actual_stdout = self.stdout
if is_windows: if is_windows:
@ -83,7 +84,7 @@ class Environment:
# noinspection PyUnresolvedReferences # noinspection PyUnresolvedReferences
actual_stdout = self.stdout.wrapped actual_stdout = self.stdout.wrapped
self.stdout_encoding = getattr( self.stdout_encoding = getattr(
actual_stdout, 'encoding', None) or 'utf8' actual_stdout, 'encoding', None) or UTF8
def __str__(self): def __str__(self):
defaults = dict(type(self).__dict__) defaults = dict(type(self).__dict__)

View File

@ -2,6 +2,7 @@ from abc import ABCMeta, abstractmethod
from typing import Iterable, Optional from typing import Iterable, Optional
from urllib.parse import urlsplit from urllib.parse import urlsplit
from .constants import UTF8
from .utils import split_cookies from .utils import split_cookies
@ -39,7 +40,7 @@ class HTTPMessage(metaclass=ABCMeta):
"""Return the message content type.""" """Return the message content type."""
ct = self._orig.headers.get('Content-Type', '') ct = self._orig.headers.get('Content-Type', '')
if not isinstance(ct, str): if not isinstance(ct, str):
ct = ct.decode('utf8') ct = ct.decode()
return ct return ct
@ -83,7 +84,7 @@ class HTTPResponse(HTTPMessage):
@property @property
def encoding(self): def encoding(self):
return self._orig.encoding or 'utf8' return self._orig.encoding or UTF8
@property @property
def body(self): def body(self):
@ -116,7 +117,7 @@ class HTTPRequest(HTTPMessage):
headers['Host'] = url.netloc.split('@')[-1] headers['Host'] = url.netloc.split('@')[-1]
headers = [ headers = [
f'{name}: {value if isinstance(value, str) else value.decode("utf-8")}' f'{name}: {value if isinstance(value, str) else value.decode()}'
for name, value in headers.items() for name, value in headers.items()
] ]
@ -126,12 +127,12 @@ class HTTPRequest(HTTPMessage):
@property @property
def encoding(self): def encoding(self):
return 'utf8' return UTF8
@property @property
def body(self): def body(self):
body = self._orig.body body = self._orig.body
if isinstance(body, str): if isinstance(body, str):
# Happens with JSON/form request data parsed from the command line. # Happens with JSON/form request data parsed from the command line.
body = body.encode('utf8') body = body.encode()
return body or b'' return body or b''

View File

@ -3,6 +3,7 @@ from itertools import chain
from typing import Callable, Iterable, Union from typing import Callable, Iterable, Union
from ..context import Environment from ..context import Environment
from ..constants import UTF8
from ..models import HTTPMessage from ..models import HTTPMessage
from .processing import Conversion, Formatting from .processing import Conversion, Formatting
@ -49,7 +50,7 @@ class BaseStream(metaclass=ABCMeta):
def get_headers(self) -> bytes: def get_headers(self) -> bytes:
"""Return the headers' bytes.""" """Return the headers' bytes."""
return self.msg.headers.encode('utf8') return self.msg.headers.encode()
@abstractmethod @abstractmethod
def iter_body(self) -> Iterable[bytes]: def iter_body(self) -> Iterable[bytes]:
@ -105,8 +106,8 @@ class EncodedStream(BaseStream):
else: else:
# Preserve the message encoding. # Preserve the message encoding.
output_encoding = self.msg.encoding output_encoding = self.msg.encoding
# Default to utf8 when unsure. # Default to UTF-8 when unsure.
self.output_encoding = output_encoding or 'utf8' self.output_encoding = output_encoding or UTF8
def iter_body(self) -> Iterable[bytes]: def iter_body(self) -> Iterable[bytes]:
for line, lf in self.msg.iter_lines(self.CHUNK_SIZE): for line, lf in self.msg.iter_lines(self.CHUNK_SIZE):

View File

@ -30,7 +30,7 @@ class HTTPBasicAuth(requests.auth.HTTPBasicAuth):
@staticmethod @staticmethod
def make_header(username: str, password: str) -> str: def make_header(username: str, password: str) -> str:
credentials = f'{username}:{password}' credentials = f'{username}:{password}'
token = b64encode(credentials.encode('utf-8')).strip().decode('latin1') token = b64encode(credentials.encode()).strip().decode('latin1')
return f'Basic {token}' return f'Basic {token}'

View File

@ -78,7 +78,7 @@ class Session(BaseConfigDict):
continue # Ignore explicitly unset headers continue # Ignore explicitly unset headers
if type(value) is not str: if type(value) is not str:
value = value.decode('utf8') value = value.decode()
if name.lower() == 'user-agent' and value.startswith('HTTPie/'): if name.lower() == 'user-agent' and value.startswith('HTTPie/'):
continue continue

View File

@ -1,6 +1,8 @@
"""Test data""" """Test data"""
from pathlib import Path from pathlib import Path
from httpie.constants import UTF8
def patharg(path): def patharg(path):
""" """
@ -23,9 +25,9 @@ JSON_FILE_PATH_ARG = patharg(JSON_FILE_PATH)
# Strip because we don't want new lines in the data so that we can # Strip because we don't want new lines in the data so that we can
# easily count occurrences also when embedded in JSON (where the new # easily count occurrences also when embedded in JSON (where the new
# line would be escaped). # line would be escaped).
FILE_CONTENT = FILE_PATH.read_text('utf8').strip() FILE_CONTENT = FILE_PATH.read_text(encoding=UTF8).strip()
JSON_FILE_CONTENT = JSON_FILE_PATH.read_text('utf8') JSON_FILE_CONTENT = JSON_FILE_PATH.read_text(encoding=UTF8)
BIN_FILE_CONTENT = BIN_FILE_PATH.read_bytes() BIN_FILE_CONTENT = BIN_FILE_PATH.read_bytes()
UNICODE = FILE_CONTENT UNICODE = FILE_CONTENT

View File

@ -118,7 +118,7 @@ class TestItemParsing:
# Parsed file fields # Parsed file fields
assert 'file' in items.files assert 'file' in items.files
assert (items.files['file'][1].read().strip(). assert (items.files['file'][1].read().strip().
decode('utf8') == FILE_CONTENT) decode() == FILE_CONTENT)
def test_multiple_file_fields_with_same_field_name(self): def test_multiple_file_fields_with_same_field_name(self):
items = RequestItems.from_args([ items = RequestItems.from_args([

View File

@ -4,6 +4,7 @@ import pytest
from _pytest.monkeypatch import MonkeyPatch from _pytest.monkeypatch import MonkeyPatch
from httpie.compat import is_windows from httpie.compat import is_windows
from httpie.constants import UTF8
from httpie.config import ( from httpie.config import (
Config, DEFAULT_CONFIG_DIRNAME, DEFAULT_RELATIVE_LEGACY_CONFIG_DIR, Config, DEFAULT_CONFIG_DIRNAME, DEFAULT_RELATIVE_LEGACY_CONFIG_DIR,
DEFAULT_RELATIVE_XDG_CONFIG_HOME, DEFAULT_WINDOWS_CONFIG_DIR, DEFAULT_RELATIVE_XDG_CONFIG_HOME, DEFAULT_WINDOWS_CONFIG_DIR,
@ -25,7 +26,7 @@ def test_default_options(httpbin):
def test_config_file_not_valid(httpbin): def test_config_file_not_valid(httpbin):
env = MockEnvironment() env = MockEnvironment()
env.create_temp_config_dir() env.create_temp_config_dir()
(env.config_dir / Config.FILENAME).write_text('{invalid json}') (env.config_dir / Config.FILENAME).write_text('{invalid json}', encoding=UTF8)
r = http(httpbin + '/get', env=env) r = http(httpbin + '/get', env=env)
assert HTTP_OK in r assert HTTP_OK in r
assert 'http: warning' in r.stderr assert 'http: warning' in r.stderr

View File

@ -66,4 +66,4 @@ def test_rst_file_syntax(filename):
shell=True, shell=True,
) )
err = p.communicate()[1] err = p.communicate()[1]
assert p.returncode == 0, err.decode('utf8') assert p.returncode == 0, err.decode()

View File

@ -66,7 +66,7 @@ class TestDownloadUtils:
) )
assert 'foo.html' == filename_from_url( assert 'foo.html' == filename_from_url(
url='http://example.org/foo', url='http://example.org/foo',
content_type='text/html; charset=utf8' content_type='text/html; charset=UTF-8'
) )
assert 'foo' == filename_from_url( assert 'foo' == filename_from_url(
url='http://example.org/foo', url='http://example.org/foo',

View File

@ -9,6 +9,7 @@ import httpie.__main__
from .fixtures import FILE_CONTENT, FILE_PATH from .fixtures import FILE_CONTENT, FILE_PATH
from httpie.cli.exceptions import ParseError from httpie.cli.exceptions import ParseError
from httpie.context import Environment from httpie.context import Environment
from httpie.constants import UTF8
from httpie.status import ExitStatus from httpie.status import ExitStatus
from .utils import HTTP_OK, MockEnvironment, StdinBytesIO, http from .utils import HTTP_OK, MockEnvironment, StdinBytesIO, http
@ -130,7 +131,7 @@ def test_form_POST_file_redirected_stdin(httpbin):
<https://github.com/httpie/httpie/issues/840> <https://github.com/httpie/httpie/issues/840>
""" """
with open(FILE_PATH): with open(FILE_PATH, encoding=UTF8):
r = http( r = http(
'--form', '--form',
'POST', 'POST',

View File

@ -15,6 +15,7 @@ from httpie.cli.argtypes import (
parse_format_options, parse_format_options,
) )
from httpie.cli.definition import parser from httpie.cli.definition import parser
from httpie.constants import UTF8
from httpie.output.formatters.colors import get_lexer from httpie.output.formatters.colors import get_lexer
from httpie.status import ExitStatus from httpie.status import ExitStatus
from .utils import COLOR, CRLF, HTTP_OK, MockEnvironment, http from .utils import COLOR, CRLF, HTTP_OK, MockEnvironment, http
@ -30,7 +31,7 @@ def test_output_option(tmp_path, httpbin, stdout_isatty):
assert r == '' assert r == ''
expected_body = urlopen(url).read().decode() expected_body = urlopen(url).read().decode()
actual_body = output_filename.read_text() actual_body = output_filename.read_text(encoding=UTF8)
assert actual_body == expected_body assert actual_body == expected_body
@ -125,7 +126,7 @@ class TestQuietFlag:
assert env.stdout is env.devnull assert env.stdout is env.devnull
else: else:
assert env.stdout is not env.devnull # --output swaps stdout. assert env.stdout is not env.devnull # --output swaps stdout.
assert output_path.read_text() == output assert output_path.read_text(encoding=UTF8) == output
finally: finally:
os.chdir(orig_cwd) os.chdir(orig_cwd)

View File

@ -7,6 +7,7 @@ from unittest import mock
import pytest import pytest
from .fixtures import FILE_PATH_ARG, UNICODE from .fixtures import FILE_PATH_ARG, UNICODE
from httpie.constants import UTF8
from httpie.plugins import AuthPlugin from httpie.plugins import AuthPlugin
from httpie.plugins.builtin import HTTPBasicAuth from httpie.plugins.builtin import HTTPBasicAuth
from httpie.plugins.registry import plugin_manager from httpie.plugins.registry import plugin_manager
@ -52,7 +53,7 @@ class CookieTestBase:
} }
} }
self.session_path = self.config_dir / 'test-session.json' self.session_path = self.config_dir / 'test-session.json'
self.session_path.write_text(json.dumps(orig_session)) self.session_path.write_text(json.dumps(orig_session), encoding=UTF8)
def teardown_method(self, method): def teardown_method(self, method):
shutil.rmtree(self.config_dir) shutil.rmtree(self.config_dir)
@ -192,7 +193,7 @@ class TestSession(SessionTestBase):
# FIXME: Authorization *sometimes* is not present # FIXME: Authorization *sometimes* is not present
assert (r2.json['headers']['Authorization'] assert (r2.json['headers']['Authorization']
== HTTPBasicAuth.make_header('test', UNICODE)) == HTTPBasicAuth.make_header('test', UNICODE))
# httpbin doesn't interpret utf8 headers # httpbin doesn't interpret UTF-8 headers
assert UNICODE in r2 assert UNICODE in r2
def test_session_default_header_value_overwritten(self, httpbin): def test_session_default_header_value_overwritten(self, httpbin):
@ -309,7 +310,7 @@ class TestSession(SessionTestBase):
Plugin.auth_type, Plugin.auth_type,
'--auth', 'user:password', '--auth', 'user:password',
) )
updated_session = json.loads(self.session_path.read_text()) updated_session = json.loads(self.session_path.read_text(encoding=UTF8))
assert updated_session['auth']['type'] == 'test-saved' assert updated_session['auth']['type'] == 'test-saved'
assert updated_session['auth']['raw_auth'] == "user:password" assert updated_session['auth']['raw_auth'] == "user:password"
plugin_manager.unregister(Plugin) plugin_manager.unregister(Plugin)
@ -338,7 +339,7 @@ class TestExpiredCookies(CookieTestBase):
) )
assert 'Cookie: cookie1=foo; cookie2=foo' in r assert 'Cookie: cookie1=foo; cookie2=foo' in r
updated_session = json.loads(self.session_path.read_text()) updated_session = json.loads(self.session_path.read_text(encoding=UTF8))
assert 'cookie1' in updated_session['cookies'] assert 'cookie1' in updated_session['cookies']
assert 'cookie2' not in updated_session['cookies'] assert 'cookie2' not in updated_session['cookies']
@ -432,7 +433,7 @@ class TestCookieStorage(CookieTestBase):
# Note: cookies in response are in alphabetical order # Note: cookies in response are in alphabetical order
assert f'Cookie: {expected}' in r assert f'Cookie: {expected}' in r
updated_session = json.loads(self.session_path.read_text()) updated_session = json.loads(self.session_path.read_text(encoding=UTF8))
for name, value in new_cookies_dict.items(): for name, value in new_cookies_dict.items():
assert name, value in updated_session['cookies'] assert name, value in updated_session['cookies']
assert 'Cookie' not in updated_session['headers'] assert 'Cookie' not in updated_session['headers']
@ -473,6 +474,6 @@ class TestCookieStorage(CookieTestBase):
httpbin.url + set_cookie, httpbin.url + set_cookie,
'Cookie:' + cli_cookie, 'Cookie:' + cli_cookie,
) )
updated_session = json.loads(self.session_path.read_text()) updated_session = json.loads(self.session_path.read_text(encoding=UTF8))
assert updated_session['cookies']['cookie1']['value'] == expected assert updated_session['cookies']['cookie1']['value'] == expected

View File

@ -7,13 +7,13 @@ from .fixtures import UNICODE
def test_unicode_headers(httpbin): def test_unicode_headers(httpbin):
# httpbin doesn't interpret utf8 headers # httpbin doesn't interpret UFT-8 headers
r = http(httpbin.url + '/headers', f'Test:{UNICODE}') r = http(httpbin.url + '/headers', f'Test:{UNICODE}')
assert HTTP_OK in r assert HTTP_OK in r
def test_unicode_headers_verbose(httpbin): def test_unicode_headers_verbose(httpbin):
# httpbin doesn't interpret utf8 headers # httpbin doesn't interpret UTF-8 headers
r = http('--verbose', httpbin.url + '/headers', f'Test:{UNICODE}') r = http('--verbose', httpbin.url + '/headers', f'Test:{UNICODE}')
assert HTTP_OK in r assert HTTP_OK in r
assert UNICODE in r assert UNICODE in r
@ -98,14 +98,14 @@ def test_unicode_url_verbose(httpbin):
def test_unicode_basic_auth(httpbin): def test_unicode_basic_auth(httpbin):
# it doesn't really authenticate us because httpbin # it doesn't really authenticate us because httpbin
# doesn't interpret the utf8-encoded auth # doesn't interpret the UTF-8-encoded auth
http('--verbose', '--auth', f'test:{UNICODE}', http('--verbose', '--auth', f'test:{UNICODE}',
f'{httpbin.url}/basic-auth/test/{UNICODE}') f'{httpbin.url}/basic-auth/test/{UNICODE}')
def test_unicode_digest_auth(httpbin): def test_unicode_digest_auth(httpbin):
# it doesn't really authenticate us because httpbin # it doesn't really authenticate us because httpbin
# doesn't interpret the utf8-encoded auth # doesn't interpret the UTF-8-encoded auth
http('--auth-type=digest', http('--auth-type=digest',
'--auth', f'test:{UNICODE}', '--auth', f'test:{UNICODE}',
f'{httpbin.url}/digest-auth/auth/test/{UNICODE}') f'{httpbin.url}/digest-auth/auth/test/{UNICODE}')

View File

@ -247,10 +247,10 @@ class TestRequestBodyFromFilePath:
self, httpbin): self, httpbin):
r = http('--verbose', r = http('--verbose',
'POST', httpbin.url + '/post', '@' + FILE_PATH_ARG, 'POST', httpbin.url + '/post', '@' + FILE_PATH_ARG,
'Content-Type:text/plain; charset=utf8') 'Content-Type:text/plain; charset=UTF-8')
assert HTTP_OK in r assert HTTP_OK in r
assert FILE_CONTENT in r assert FILE_CONTENT in r
assert 'Content-Type: text/plain; charset=utf8' in r assert 'Content-Type: text/plain; charset=UTF-8' in r
def test_request_body_from_file_by_path_no_field_name_allowed( def test_request_body_from_file_by_path_no_field_name_allowed(
self, httpbin): self, httpbin):

View File

@ -281,14 +281,14 @@ def http(
output = stdout.read() output = stdout.read()
devnull_output = devnull.read() devnull_output = devnull.read()
try: try:
output = output.decode('utf8') output = output.decode()
except UnicodeDecodeError: except UnicodeDecodeError:
r = BytesCLIResponse(output) r = BytesCLIResponse(output)
else: else:
r = StrCLIResponse(output) r = StrCLIResponse(output)
try: try:
devnull_output = devnull_output.decode('utf8') devnull_output = devnull_output.decode()
except Exception: except Exception:
pass pass