extern/httpie-cli
1
0
Fork 1
mirror of https://github.com/httpie/cli.git synced 2025-03-13 14:28:50 +01:00
Code Issues Packages Projects Releases Wiki Activity

fixed+enabled recently disabled tests for download + compressed bodies

Browse Source
This commit is contained in:
Ahmed TAHRI 2024-06-28 00:24:53 +01:00
parent d02b88254d
commit 55aea9cce5
3 changed files with 68 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
httpie/downloads.py
View File

@ -211,12 +211,6 @@ class Downloader:
Might alter `request_headers`. Might alter `request_headers`.
""" """
# Ask the server not to encode the content so that we can resume, etc.
# TODO: Reconsider this once the underlying library can report raw download size (i.e., not decoded).
# Then it might still be needed when resuming. But in the default case, it won’t probably be necessary.
# <https://github.com/jawah/niquests/issues/127>
request_headers['Accept-Encoding'] = 'identity'
if self._resume: if self._resume:
bytes_have = os.path.getsize(self._output_file.name) bytes_have = os.path.getsize(self._output_file.name)
if bytes_have: if bytes_have:
@ -301,11 +295,11 @@ class Downloader:
def is_interrupted(self) -> bool: def is_interrupted(self) -> bool:
return self.status.is_interrupted return self.status.is_interrupted
def chunk_downloaded(self, chunk_or_new_total: Union[bytes, int]): def chunk_downloaded(self, chunk_or_new_total: Union[bytes, int]) -> None:
""" """
A download progress callback. A download progress callback.
:param chunk: A chunk of response body data that has just :param chunk_or_new_total: A chunk of response body data that has just
been downloaded and written to the output. been downloaded and written to the output.
""" """
@ -333,8 +327,7 @@ class Downloader:
return open(unique_filename, buffering=0, mode='a+b') return open(unique_filename, buffering=0, mode='a+b')
DECODED_FROM_SUFFIX = ' - decoded from {encodings}' DECODED_FROM_SUFFIX = ' - decoded using {encodings}'
DECODED_SIZE_NOTE_SUFFIX = ' - decoded size'
class DownloadStatus: class DownloadStatus:
@ -365,8 +358,14 @@ class DownloadStatus:
ProgressDisplayFull ProgressDisplayFull
) )
message = f'Downloading to {output_file.name}' message = f'Downloading to {output_file.name}'
message_suffix = ''
summary_suffix = '' summary_suffix = ''
if self.decoded_from:
encodings = ', '.join(f'`{enc}`' for enc in self.decoded_from)
message_suffix = DECODED_FROM_SUFFIX.format(encodings=encodings)
else:
message_suffix = ''
if not self.env.show_displays: if not self.env.show_displays:
progress_display_class = DummyProgressDisplay progress_display_class = DummyProgressDisplay
else: else:
@ -375,11 +374,8 @@ class DownloadStatus:
if has_reliable_total: if has_reliable_total:
progress_display_class = ProgressDisplayFull progress_display_class = ProgressDisplayFull
else: else:
if self.decoded_from:
encodings = ', '.join(f'`{enc}`' for enc in self.decoded_from)
message_suffix = DECODED_FROM_SUFFIX.format(encodings=encodings)
summary_suffix = DECODED_SIZE_NOTE_SUFFIX
progress_display_class = ProgressDisplayNoTotal progress_display_class = ProgressDisplayNoTotal
self.display = progress_display_class( self.display = progress_display_class(
env=self.env, env=self.env,
total_size=self.total_size, total_size=self.total_size,

7
httpie/output/streams.py
View File

@ -79,8 +79,15 @@ class BaseStream(metaclass=ABCMeta):
# Useful when the remote compress the body. We use the "untouched" amt of data to determine # Useful when the remote compress the body. We use the "untouched" amt of data to determine
# the download speed. # the download speed.
if hasattr(self.msg, "_orig") and hasattr(self.msg._orig, "download_progress") and self.msg._orig.download_progress: if hasattr(self.msg, "_orig") and hasattr(self.msg._orig, "download_progress") and self.msg._orig.download_progress:
# this is plan A: using public interfaces!
self.on_body_chunk_downloaded(self.msg._orig.download_progress.total) self.on_body_chunk_downloaded(self.msg._orig.download_progress.total)
elif hasattr(self.msg, "_orig") and hasattr(self.msg._orig, "raw") and hasattr(self.msg._orig.raw, "_fp_bytes_read"):
# plan B, falling back on a private property that may disapear from urllib3-future...
# this case is mandatory due to how the mocking library works. it does not use any "socket" but
# rather a simple io.BytesIO.
self.on_body_chunk_downloaded(self.msg._orig.raw._fp_bytes_read)
else: else:
# well. this case will certainly cause issues if the body is compressed.
self.on_body_chunk_downloaded(chunk) self.on_body_chunk_downloaded(chunk)
except DataSuppressedError as e: except DataSuppressedError as e:
if self.output_options.headers: if self.output_options.headers:

104
tests/test_downloads.py
View File

@ -1,6 +1,7 @@
import os import os
import tempfile import tempfile
import time import time
import zlib
from unittest import mock from unittest import mock
from urllib.request import urlopen from urllib.request import urlopen
@ -15,7 +16,7 @@ from httpie.downloads import (
ContentRangeError, ContentRangeError,
Downloader, Downloader,
PARTIAL_CONTENT, PARTIAL_CONTENT,
DECODED_SIZE_NOTE_SUFFIX, DECODED_FROM_SUFFIX,
) )
from niquests.structures import CaseInsensitiveDict from niquests.structures import CaseInsensitiveDict
from .utils import http, MockEnvironment, cd_clean_tmp_dir, DUMMY_URL from .utils import http, MockEnvironment, cd_clean_tmp_dir, DUMMY_URL
@ -232,7 +233,6 @@ class TestDownloader:
# Ensure `pre_request()` is working as expected too # Ensure `pre_request()` is working as expected too
headers = {} headers = {}
downloader.pre_request(headers) downloader.pre_request(headers)
assert headers['Accept-Encoding'] == 'identity'
assert headers['Range'] == 'bytes=3-' assert headers['Range'] == 'bytes=3-'
downloader.start( downloader.start(
@ -264,7 +264,7 @@ class TestDownloader:
@responses.activate @responses.activate
def test_incomplete_response(self): def test_incomplete_response(self):
# We have incompleteness checks in the downloader, but it might not be needed as it’s built into (ni|req)uests. # We have incompleteness checks in the downloader, but it might not be needed as it’s built into (ni|req)uests.
error_msg = 'peer closed connection without sending complete message body (received 2 bytes, expected 1 more)' error_msg = 'IncompleteRead(2 bytes read, 1 more expected)'
responses.add( responses.add(
method=responses.GET, method=responses.GET,
url=DUMMY_URL, url=DUMMY_URL,
@ -281,55 +281,53 @@ class TestDownloader:
class TestDecodedDownloads: class TestDecodedDownloads:
"""Test downloading responses with `Content-Encoding`""" """Test downloading responses with `Content-Encoding`"""
# todo: find an appropriate way to mock compressed bodies within those tests. @responses.activate
# @responses.activate def test_decoded_response_no_content_length(self):
# def test_decoded_response_no_content_length(self): responses.add(
# responses.add( method=responses.GET,
# method=responses.GET, url=DUMMY_URL,
# url=DUMMY_URL, headers={
# headers={ 'Content-Encoding': 'deflate',
# 'Content-Encoding': 'gzip, br', },
# }, body=zlib.compress(b"foobar"),
# body='123', )
# ) with cd_clean_tmp_dir():
# with cd_clean_tmp_dir(): r = http('--download', '--headers', DUMMY_URL)
# r = http('--download', '--headers', DUMMY_URL) print(r.stderr)
# print(r.stderr) assert DECODED_FROM_SUFFIX.format(encodings='`deflate`') in r.stderr
# assert DECODED_FROM_SUFFIX.format(encodings='`gzip`, `br`') in r.stderr
# assert DECODED_SIZE_NOTE_SUFFIX in r.stderr @responses.activate
# def test_decoded_response_with_content_length(self):
# @responses.activate payload = zlib.compress(b"foobar")
# def test_decoded_response_with_content_length(self):
# responses.add( responses.add(
# method=responses.GET, method=responses.GET,
# url=DUMMY_URL, url=DUMMY_URL,
# headers={ headers={
# 'Content-Encoding': 'gzip, br', 'Content-Encoding': 'deflate',
# 'Content-Length': '3', 'Content-Length': str(len(payload)),
# }, },
# body='123', body=payload,
# ) )
# with cd_clean_tmp_dir(): with cd_clean_tmp_dir():
# r = http('--download', DUMMY_URL) r = http('--download', DUMMY_URL)
# print(r.stderr) print(r.stderr)
# assert DECODED_FROM_SUFFIX.format(encodings='`gzip`, `br`') in r.stderr assert DECODED_FROM_SUFFIX.format(encodings='`deflate`') in r.stderr
# assert DECODED_SIZE_NOTE_SUFFIX in r.stderr
# @responses.activate
# @responses.activate def test_decoded_response_without_content_length(self):
# def test_decoded_response_without_content_length(self): responses.add(
# responses.add( method=responses.GET,
# method=responses.GET, url=DUMMY_URL,
# url=DUMMY_URL, headers={
# headers={ 'Content-Encoding': 'deflate',
# 'Content-Encoding': 'gzip, br', },
# }, body=zlib.compress(b'foobar'),
# body='123', )
# ) with cd_clean_tmp_dir():
# with cd_clean_tmp_dir(): r = http('--download', DUMMY_URL)
# r = http('--download', DUMMY_URL) print(r.stderr)
# print(r.stderr) assert DECODED_FROM_SUFFIX.format(encodings='`deflate`') in r.stderr
# assert DECODED_FROM_SUFFIX.format(encodings='`gzip`, `br`') in r.stderr
# assert DECODED_SIZE_NOTE_SUFFIX in r.stderr
@responses.activate @responses.activate
def test_non_decoded_response_without_content_length(self): def test_non_decoded_response_without_content_length(self):
@ -344,7 +342,6 @@ class TestDecodedDownloads:
with cd_clean_tmp_dir(): with cd_clean_tmp_dir():
r = http('--download', DUMMY_URL) r = http('--download', DUMMY_URL)
print(r.stderr) print(r.stderr)
assert DECODED_SIZE_NOTE_SUFFIX not in r.stderr
@responses.activate @responses.activate
def test_non_decoded_response_with_content_length(self): def test_non_decoded_response_with_content_length(self):
@ -358,4 +355,3 @@ class TestDecodedDownloads:
with cd_clean_tmp_dir(): with cd_clean_tmp_dir():
r = http('--download', DUMMY_URL) r = http('--download', DUMMY_URL)
print(r.stderr) print(r.stderr)
assert DECODED_SIZE_NOTE_SUFFIX not in r.stderr