From 00c859c51d725159249aa6fda812a775e352f21b Mon Sep 17 00:00:00 2001 From: Batuhan Taskaya Date: Wed, 12 Jan 2022 17:07:34 +0300 Subject: [PATCH] Add warnings when there is no incoming data from stdin (#1256) * Add warnings when there is no incoming data from stdin * Pass os.environ as well * Apply suggestions --- CHANGELOG.md | 1 + httpie/client.py | 9 +++-- httpie/core.py | 2 +- httpie/uploads.py | 51 ++++++++++++++++++++++++++ tests/test_uploads.py | 85 +++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 144 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 32324935..cd73afcc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). - Added new `pie-dark`/`pie-light` (and `pie`) styles that match with [HTTPie for Web and Desktop](https://httpie.io/product). ([#1237](https://github.com/httpie/httpie/issues/1237)) - Added support for better error handling on DNS failures. ([#1248](https://github.com/httpie/httpie/issues/1248)) - Added support for storing prompted passwords in the local sessions. ([#1098](https://github.com/httpie/httpie/issues/1098)) +- Added warnings about the `--ignore-stdin`, when there is no incoming data from stdin. ([#1255](https://github.com/httpie/httpie/issues/1255)) - Broken plugins will no longer crash the whole application. ([#1204](https://github.com/httpie/httpie/issues/1204)) - Fixed auto addition of XML declaration to every formatted XML response. ([#1156](https://github.com/httpie/httpie/issues/1156)) - Fixed highlighting when `Content-Type` specifies `charset`. ([#1242](https://github.com/httpie/httpie/issues/1242)) diff --git a/httpie/client.py b/httpie/client.py index dcf24826..58575fe6 100644 --- a/httpie/client.py +++ b/httpie/client.py @@ -3,7 +3,6 @@ import http.client import json import sys from contextlib import contextmanager -from pathlib import Path from typing import Any, Dict, Callable, Iterable from urllib.parse import urlparse, urlunparse @@ -12,6 +11,7 @@ import requests import urllib3 from . import __version__ from .adapters import HTTPieHTTPAdapter +from .context import Environment from .cli.dicts import HTTPHeadersDict from .encoding import UTF8 from .models import RequestsMessage @@ -34,15 +34,15 @@ DEFAULT_UA = f'HTTPie/{__version__}' def collect_messages( + env: Environment, args: argparse.Namespace, - config_dir: Path, request_body_read_callback: Callable[[bytes], None] = None, ) -> Iterable[RequestsMessage]: httpie_session = None httpie_session_headers = None if args.session or args.session_read_only: httpie_session = get_httpie_session( - config_dir=config_dir, + config_dir=env.config.directory, session_name=args.session or args.session_read_only, host=args.headers.get('Host'), url=args.url, @@ -50,6 +50,7 @@ def collect_messages( httpie_session_headers = httpie_session.headers request_kwargs = make_request_kwargs( + env, args=args, base_headers=httpie_session_headers, request_body_read_callback=request_body_read_callback @@ -292,6 +293,7 @@ def json_dict_to_request_body(data: Dict[str, Any]) -> str: def make_request_kwargs( + env: Environment, args: argparse.Namespace, base_headers: HTTPHeadersDict = None, request_body_read_callback=lambda chunk: chunk @@ -330,6 +332,7 @@ def make_request_kwargs( 'url': args.url, 'headers': headers, 'data': prepare_request_body( + env, data, body_read_callback=request_body_read_callback, chunked=args.chunked, diff --git a/httpie/core.py b/httpie/core.py index 38f67065..4718b924 100644 --- a/httpie/core.py +++ b/httpie/core.py @@ -188,7 +188,7 @@ def program(args: argparse.Namespace, env: Environment) -> ExitStatus: args.follow = True # --download implies --follow. downloader = Downloader(output_file=args.output_file, progress_file=env.stderr, resume=args.download_resume) downloader.pre_request(args.headers) - messages = collect_messages(args=args, config_dir=env.config.directory, + messages = collect_messages(env, args=args, request_body_read_callback=request_body_read_callback) force_separator = False prev_with_body = False diff --git a/httpie/uploads.py b/httpie/uploads.py index 0bb307e5..4fdb7922 100644 --- a/httpie/uploads.py +++ b/httpie/uploads.py @@ -1,3 +1,5 @@ +import sys +import os import zlib import functools from typing import Any, Callable, IO, Iterable, Optional, Tuple, Union, TYPE_CHECKING @@ -9,7 +11,9 @@ from requests.utils import super_len if TYPE_CHECKING: from requests_toolbelt import MultipartEncoder +from .context import Environment from .cli.dicts import MultipartRequestDataDict, RequestDataDict +from .compat import is_windows class ChunkedStream: @@ -64,13 +68,58 @@ def _wrap_function_with_callback( return wrapped +def is_stdin(file: IO) -> bool: + try: + file_no = file.fileno() + except Exception: + return False + else: + return file_no == sys.stdin.fileno() + + +READ_THRESHOLD = float(os.getenv("HTTPIE_STDIN_READ_WARN_THRESHOLD", 10.0)) + + +def observe_stdin_for_data_thread(env: Environment, file: IO) -> None: + # Windows unfortunately does not support select() operation + # on regular files, like stdin in our use case. + # https://docs.python.org/3/library/select.html#select.select + if is_windows: + return None + + # If the user configures READ_THRESHOLD to be 0, then + # disable this warning. + if READ_THRESHOLD == 0: + return None + + import select + import threading + + def worker(): + can_read, _, _ = select.select([file], [], [], READ_THRESHOLD) + if not can_read: + env.stderr.write( + f'> warning: no stdin data read in {READ_THRESHOLD}s ' + f'(perhaps you want to --ignore-stdin)\n' + f'> See: https://httpie.io/docs/cli/best-practices\n' + ) + + thread = threading.Thread( + target=worker + ) + thread.start() + + def _prepare_file_for_upload( + env: Environment, file: Union[IO, 'MultipartEncoder'], callback: CallbackT, chunked: bool = False, content_length_header_value: Optional[int] = None, ) -> Union[bytes, IO, ChunkedStream]: if not super_len(file): + if is_stdin(file): + observe_stdin_for_data_thread(env, file) # Zero-length -> assume stdin. if content_length_header_value is None and not chunked: # Read the whole stdin to determine `Content-Length`. @@ -103,6 +152,7 @@ def _prepare_file_for_upload( def prepare_request_body( + env: Environment, raw_body: Union[str, bytes, IO, 'MultipartEncoder', RequestDataDict], body_read_callback: CallbackT, offline: bool = False, @@ -125,6 +175,7 @@ def prepare_request_body( if is_file_like: return _prepare_file_for_upload( + env, body, chunked=chunked, callback=body_read_callback, diff --git a/tests/test_uploads.py b/tests/test_uploads.py index 28d427ae..613007ba 100644 --- a/tests/test_uploads.py +++ b/tests/test_uploads.py @@ -1,10 +1,16 @@ import os import json +import sys +import subprocess +import time +import contextlib +import httpie.__main__ as main import pytest from httpie.cli.exceptions import ParseError from httpie.client import FORM_CONTENT_TYPE +from httpie.compat import is_windows from httpie.status import ExitStatus from .utils import ( MockEnvironment, StdinBytesIO, http, @@ -83,6 +89,85 @@ def test_chunked_raw(httpbin_with_chunked_support): assert 'Transfer-Encoding: chunked' in r +@contextlib.contextmanager +def stdin_processes(httpbin, *args): + process_1 = subprocess.Popen( + [ + "cat" + ], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE + ) + process_2 = subprocess.Popen( + [ + sys.executable, + main.__file__, + "POST", + httpbin + "/post", + *args + ], + stdin=process_1.stdout, + stderr=subprocess.PIPE, + env={ + **os.environ, + "HTTPIE_STDIN_READ_WARN_THRESHOLD": "0.1" + } + ) + try: + yield process_1, process_2 + finally: + process_1.terminate() + process_2.terminate() + + +@pytest.mark.parametrize("wait", (True, False)) +@pytest.mark.skipif(is_windows, reason="Windows doesn't support select() calls into files") +def test_reading_from_stdin(httpbin, wait): + with stdin_processes(httpbin) as (process_1, process_2): + process_1.communicate(timeout=0.1, input=b"bleh") + # Since there is data, it doesn't matter if there + # you wait or not. + if wait: + time.sleep(0.75) + + try: + _, errs = process_2.communicate(timeout=0.25) + except subprocess.TimeoutExpired: + errs = b'' + + assert b'> warning: no stdin data read in 0.1s' not in errs + + +@pytest.mark.skipif(is_windows, reason="Windows doesn't support select() calls into files") +def test_stdin_read_warning(httpbin): + with stdin_processes(httpbin) as (process_1, process_2): + # Wait before sending any data + time.sleep(0.75) + process_1.communicate(timeout=0.1, input=b"bleh\n") + + try: + _, errs = process_2.communicate(timeout=0.25) + except subprocess.TimeoutExpired: + errs = b'' + + assert b'> warning: no stdin data read in 0.1s' in errs + + +@pytest.mark.skipif(is_windows, reason="Windows doesn't support select() calls into files") +def test_stdin_read_warning_with_quiet(httpbin): + with stdin_processes(httpbin, "-qq") as (process_1, process_2): + # Wait before sending any data + time.sleep(0.75) + process_1.communicate(timeout=0.1, input=b"bleh\n") + + try: + _, errs = process_2.communicate(timeout=0.25) + except subprocess.TimeoutExpired: + errs = b'' + + assert b'> warning: no stdin data read in 0.1s' not in errs + + class TestMultipartFormDataFileUpload: def test_non_existent_file_raises_parse_error(self, httpbin):