Avoid override of headers by urllib3 when unset (#1502)

* Pass SKIP_HEADER const when header is unset

* Hide SKIP_HEADER constant when displaying headers

* Test that omits User-Agent

httpie/client.py

@@ -10,6 +10,7 @@ from urllib.parse import urlparse, urlunparse
 import requests
 # noinspection PyPackageRequirements
 import urllib3
+from urllib3.util import SKIP_HEADER, SKIPPABLE_HEADERS
 
 from . import __version__
 from .adapters import HTTPieHTTPAdapter
@@ -200,6 +201,10 @@ def finalize_headers(headers: HTTPHeadersDict) -> HTTPHeadersDict:
             if isinstance(value, str):
                 # See <https://github.com/httpie/httpie/issues/212>
                 value = value.encode()
+        elif name.lower() in SKIPPABLE_HEADERS:
+            # Some headers get overwritten by urllib3 when set to `None`
+            # and should be replaced with the `SKIP_HEADER` constant.
+            value = SKIP_HEADER
         final_headers.add(name, value)
     return final_headers
 

httpie/models.py

@@ -1,6 +1,7 @@
 from time import monotonic
 
 import requests
+from urllib3.util import SKIP_HEADER, SKIPPABLE_HEADERS
 
 from enum import Enum, auto
 from typing import Iterable, Union, NamedTuple
@@ -152,6 +153,7 @@ class HTTPRequest(HTTPMessage):
         headers = [
             f'{name}: {value if isinstance(value, str) else value.decode()}'
             for name, value in headers.items()
+            if not (name.lower() in SKIPPABLE_HEADERS and value == SKIP_HEADER)
         ]
 
         headers.insert(0, request_line)

tests/test_httpie.py

@@ -196,6 +196,14 @@ def test_unset_host_header(httpbin_both):
     assert 'Host' not in r.json['headers']  # default Host unset
 
 
+def test_unset_useragent_header(httpbin_both):
+    r = http('GET', httpbin_both + '/headers')
+    assert 'User-Agent' in r.json['headers']  # default User-Agent present
+
+    r = http('GET', httpbin_both + '/headers', 'User-Agent:')
+    assert 'User-Agent' not in r.json['headers']  # default User-Agent unset
+
+
 def test_headers_empty_value(httpbin_both):
     r = http('GET', httpbin_both + '/headers')
     assert r.json['headers']['Accept']  # default Accept has value