forked from extern/httpie-cli
6bd6648545
We were using the potential `encoding` returned by `mimetypes.guess_type()`
to expand the `Content-Type` header.
According to the RFC-7231 [1] the `Content-Type` should contain a charset
and nothing more. But as stated in the `mimetypes.guess_type()` doc [2],
the `encoding` would be the name of the program used to encode (e.g. compress
or gzip) the payload. The `encoding` is suitable for use as a `Content-Encoding`
header. See [3] for potential `encoding`s, none is a IANA registered one [4], and
so a valid charset to be used by the `Content-Type` header.
[1] https://httpwg.org/specs/rfc7231.html#header.content-type
[2] https://docs.python.org/3/library/mimetypes.html#guess_type
[3] 938e84b4fa/Lib/mimetypes.py (L416-L422)
[4] https://www.iana.org/assignments/character-sets/character-sets.xhtml
144 lines
3.4 KiB
Python
144 lines
3.4 KiB
Python
import json
|
|
import mimetypes
|
|
import time
|
|
from collections import OrderedDict
|
|
from http.cookiejar import parse_ns_headers
|
|
from pprint import pformat
|
|
from typing import List, Optional, Tuple
|
|
import re
|
|
|
|
import requests.auth
|
|
|
|
RE_COOKIE_SPLIT = re.compile(r', (?=[^ ;]+=)')
|
|
|
|
|
|
def load_json_preserve_order(s):
|
|
return json.loads(s, object_pairs_hook=OrderedDict)
|
|
|
|
|
|
def repr_dict(d: dict) -> str:
|
|
return pformat(d)
|
|
|
|
|
|
def humanize_bytes(n, precision=2):
|
|
# Author: Doug Latornell
|
|
# Licence: MIT
|
|
# URL: https://code.activestate.com/recipes/577081/
|
|
"""Return a humanized string representation of a number of bytes.
|
|
|
|
>>> humanize_bytes(1)
|
|
'1 B'
|
|
>>> humanize_bytes(1024, precision=1)
|
|
'1.0 kB'
|
|
>>> humanize_bytes(1024 * 123, precision=1)
|
|
'123.0 kB'
|
|
>>> humanize_bytes(1024 * 12342, precision=1)
|
|
'12.1 MB'
|
|
>>> humanize_bytes(1024 * 12342, precision=2)
|
|
'12.05 MB'
|
|
>>> humanize_bytes(1024 * 1234, precision=2)
|
|
'1.21 MB'
|
|
>>> humanize_bytes(1024 * 1234 * 1111, precision=2)
|
|
'1.31 GB'
|
|
>>> humanize_bytes(1024 * 1234 * 1111, precision=1)
|
|
'1.3 GB'
|
|
|
|
"""
|
|
abbrevs = [
|
|
(1 << 50, 'PB'),
|
|
(1 << 40, 'TB'),
|
|
(1 << 30, 'GB'),
|
|
(1 << 20, 'MB'),
|
|
(1 << 10, 'kB'),
|
|
(1, 'B')
|
|
]
|
|
|
|
if n == 1:
|
|
return '1 B'
|
|
|
|
for factor, suffix in abbrevs:
|
|
if n >= factor:
|
|
break
|
|
|
|
# noinspection PyUnboundLocalVariable
|
|
return f'{n / factor:.{precision}f} {suffix}'
|
|
|
|
|
|
class ExplicitNullAuth(requests.auth.AuthBase):
|
|
"""Forces requests to ignore the ``.netrc``.
|
|
<https://github.com/psf/requests/issues/2773#issuecomment-174312831>
|
|
"""
|
|
|
|
def __call__(self, r):
|
|
return r
|
|
|
|
|
|
def get_content_type(filename):
|
|
"""
|
|
Return the content type for ``filename`` in format appropriate
|
|
for Content-Type headers, or ``None`` if the file type is unknown
|
|
to ``mimetypes``.
|
|
|
|
"""
|
|
return mimetypes.guess_type(filename, strict=False)[0]
|
|
|
|
|
|
def split_cookies(cookies):
|
|
"""
|
|
When ``requests`` stores cookies in ``response.headers['Set-Cookie']``
|
|
it concatenates all of them through ``, ``.
|
|
|
|
This function splits cookies apart being careful to not to
|
|
split on ``, `` which may be part of cookie value.
|
|
"""
|
|
if not cookies:
|
|
return []
|
|
return RE_COOKIE_SPLIT.split(cookies)
|
|
|
|
|
|
def get_expired_cookies(
|
|
cookies: str,
|
|
now: float = None
|
|
) -> List[dict]:
|
|
|
|
now = now or time.time()
|
|
|
|
def is_expired(expires: Optional[float]) -> bool:
|
|
return expires is not None and expires <= now
|
|
|
|
attr_sets: List[Tuple[str, str]] = parse_ns_headers(
|
|
split_cookies(cookies)
|
|
)
|
|
|
|
cookies = [
|
|
# The first attr name is the cookie name.
|
|
dict(attrs[1:], name=attrs[0][0])
|
|
for attrs in attr_sets
|
|
]
|
|
|
|
_max_age_to_expires(cookies=cookies, now=now)
|
|
|
|
return [
|
|
{
|
|
'name': cookie['name'],
|
|
'path': cookie.get('path', '/')
|
|
}
|
|
for cookie in cookies
|
|
if is_expired(expires=cookie.get('expires'))
|
|
]
|
|
|
|
|
|
def _max_age_to_expires(cookies, now):
|
|
"""
|
|
Translate `max-age` into `expires` for Requests to take it into account.
|
|
|
|
HACK/FIXME: <https://github.com/psf/requests/issues/5743>
|
|
|
|
"""
|
|
for cookie in cookies:
|
|
if 'expires' in cookie:
|
|
continue
|
|
max_age = cookie.get('max-age')
|
|
if max_age and max_age.isdigit():
|
|
cookie['expires'] = now + float(max_age)
|