Remove automatic config file creation to avoid concurrency issues.

Close #788
Close #812

CHANGELOG.rst

@@ -11,6 +11,7 @@ This project adheres to `Semantic Versioning <https://semver.org/>`_.
 * Removed Python 2.7 support (`EOL Jan 2020 <https://www.python.org/doc/sunset-python-2/>`_).
 * Removed the default 30-second connection ``--timeout`` limit.
 * Removed Python’s default limit of 100 response headers.
+* Removed automatic config file creation to avoid concurrency issues.
 * Replaced the old collect-all-then-process handling of HTTP communication
   with one-by-one processing of each HTTP request or response as they become
   available. This means that you can see headers immediately,
@@ -26,7 +27,6 @@ This project adheres to `Semantic Versioning <https://semver.org/>`_.
 * Added ``tests/`` to the PyPi package for the convenience of
   downstream package maintainers.
 * Fixed an error when ``stdin`` was a closed fd.
-* Fixed an error when the config directory was not writeable.
 * Improved ``--debug`` output formatting.
 
 

README.rst

@@ -1487,7 +1487,8 @@ To create or reuse a different session, simple specify a different name:
 
     $ http --session=user2 -a user2:password example.org X-Bar:Foo
 
-Named sessions' data is stored in JSON files in the directory
+Named sessions’s data is stored in JSON files in the the ``sessions``
+subdirectory of the `config`_ directory:
 ``~/.httpie/sessions/<host>/<name>.json``
 (``%APPDATA%\httpie\sessions\<host>\<name>.json`` on Windows).
 
@@ -1517,46 +1518,61 @@ exchange once it is created, specify the session name via
 Config
 ======
 
-HTTPie uses a simple JSON config file.
+HTTPie uses a simple ``config.json`` file. The file doesn’t exist by default
+but you can create it manually.
 
 
-
+Config file directory
-Config file location
+---------------------
---------------------
-
 
 The default location of the configuration file is ``~/.httpie/config.json``
-(or ``%APPDATA%\httpie\config.json`` on Windows). The config directory
+(or ``%APPDATA%\httpie\config.json`` on Windows).
-location can be changed by setting the ``HTTPIE_CONFIG_DIR``
+
-environment variable. To view the exact location run ``http --debug``.
+The config directory can be changed by setting the ``$HTTPIE_CONFIG_DIR``
+environment variable:
+
+.. code-block:: bash
+
+    $ export HTTPIE_CONFIG_DIR=/tmp/httpie
+    $ http example.org
+
+To view the exact location run ``http --debug``.
+
 
 Configurable options
 --------------------
 
-The JSON file contains an object with the following keys:
+Currently HTTPie offers a single configurable option:
 
 
 ``default_options``
 ~~~~~~~~~~~~~~~~~~~
 
-
 An ``Array`` (by default empty) of default options that should be applied to
 every invocation of HTTPie.
 
-For instance, you can use this option to change the default style and output
+For instance, you can use this config option to change your default color theme:
-options: ``"default_options": ["--style=fruity", "--body"]`` Another useful
-default option could be ``"--session=default"`` to make HTTPie always
-use `sessions`_ (one named ``default`` will automatically be used).
-Or you could change the implicit request content type from JSON to form by
-adding ``--form`` to the list.
 
 
-``__meta__``
+.. code-block:: bash
-~~~~~~~~~~~~
 
-HTTPie automatically stores some of its metadata here. Please do not change.
+    $ cat ~/.httpie/config.json
 
 
+.. code-block:: json
+
+    {
+        "default_options": [
+          "--style=fruity"
+        ]
+    }
+
+
+Even though it is technically possible to include there any of HTTPie’s
+options, it is not recommended to modify the default behaviour in a way
+that would break your compatibility with the wider world as that can
+generate a lot of confusion.
+
 
 Un-setting previously specified options
 ---------------------------------------

httpie/cli/argparser.py

@@ -61,7 +61,6 @@ class HTTPieArgumentParser(argparse.ArgumentParser):
     def parse_args(
         self,
         env: Environment,
-        program_name='http',
         args=None,
         namespace=None
     ) -> argparse.Namespace:
@@ -89,7 +88,7 @@ class HTTPieArgumentParser(argparse.ArgumentParser):
         if self.has_stdin_data:
             self._body_from_file(self.env.stdin)
         if not URL_SCHEME_RE.match(self.args.url):
-            if os.path.basename(program_name) == 'https':
+            if os.path.basename(env.program_name) == 'https':
                 scheme = 'https://'
             else:
                 scheme = self.args.default_scheme + "://"

httpie/client.py

@@ -35,7 +35,7 @@ DEFAULT_UA = f'HTTPie/{__version__}'
 
 def collect_messages(
     args: argparse.Namespace,
-    config_dir: Path
+    config_dir: Path,
 ) -> Iterable[Union[requests.PreparedRequest, requests.Response]]:
     httpie_session = None
     httpie_session_headers = None

httpie/config.py

@@ -15,42 +15,43 @@ DEFAULT_CONFIG_DIR = Path(os.environ.get(
 ))
 
 
+class ConfigFileError(Exception):
+    pass
+
+
 class BaseConfigDict(dict):
     name = None
     helpurl = None
     about = None
 
-    def _get_path(self) -> Path:
+    def __init__(self, path: Path):
-        """Return the config file path without side-effects."""
+        super().__init__()
-        raise NotImplementedError()
+        self.path = path
 
-    def path(self) -> Path:
+    def ensure_directory(self):
-        """Return the config file path creating basedir, if needed."""
-        path = self._get_path()
         try:
-            path.parent.mkdir(mode=0o700, parents=True)
+            self.path.parent.mkdir(mode=0o700, parents=True)
         except OSError as e:
             if e.errno != errno.EEXIST:
                 raise
-        return path
 
     def is_new(self) -> bool:
-        return not self._get_path().exists()
+        return not self.path.exists()
 
     def load(self):
+        config_type = type(self).__name__.lower()
         try:
-            with self.path().open('rt') as f:
+            with self.path.open('rt') as f:
                 try:
                     data = json.load(f)
                 except ValueError as e:
-                    raise ValueError(
+                    raise ConfigFileError(
-                        'Invalid %s JSON: %s [%s]' %
+                        f'invalid {config_type} file: {e} [{self.path}]'
-                        (type(self).__name__, str(e), self.path())
                     )
                 self.update(data)
         except IOError as e:
             if e.errno != errno.ENOENT:
-                raise
+                raise ConfigFileError(f'cannot read {config_type} file: {e}')
 
     def save(self, fail_silently=False):
         self['__meta__'] = {
@@ -62,9 +63,17 @@ class BaseConfigDict(dict):
         if self.about:
             self['__meta__']['about'] = self.about
 
+        self.ensure_directory()
+
         try:
-            with self.path().open('w') as f:
+            with self.path.open('w') as f:
-                json.dump(self, f, indent=4, sort_keys=True, ensure_ascii=True)
+                json.dump(
+                    obj=self,
+                    fp=f,
+                    indent=4,
+                    sort_keys=True,
+                    ensure_ascii=True,
+                )
                 f.write('\n')
         except IOError:
             if not fail_silently:
@@ -72,27 +81,22 @@ class BaseConfigDict(dict):
 
     def delete(self):
         try:
-            self.path().unlink()
+            self.path.unlink()
         except OSError as e:
             if e.errno != errno.ENOENT:
                 raise
 
 
 class Config(BaseConfigDict):
-    name = 'config'
+    FILENAME = 'config.json'
-    helpurl = 'https://httpie.org/doc#config'
-    about = 'HTTPie configuration file'
     DEFAULTS = {
         'default_options': []
     }
 
     def __init__(self, directory: Union[str, Path] = DEFAULT_CONFIG_DIR):
-        super().__init__()
-        self.update(self.DEFAULTS)
         self.directory = Path(directory)
-
+        super().__init__(path=self.directory / self.FILENAME)
-    def _get_path(self) -> Path:
+        self.update(self.DEFAULTS)
-        return self.directory / (self.name + '.json')
 
     @property
     def default_options(self) -> list:

httpie/context.py

@@ -1,3 +1,4 @@
+import os
 import sys
 from pathlib import Path
 from typing import Union, IO, Optional
@@ -9,7 +10,7 @@ except ImportError:
     curses = None  # Compiled w/o curses
 
 from httpie.compat import is_windows
-from httpie.config import DEFAULT_CONFIG_DIR, Config
+from httpie.config import DEFAULT_CONFIG_DIR, Config, ConfigFileError
 
 from httpie.utils import repr_dict
 
@@ -35,6 +36,7 @@ class Environment:
     stderr: IO = sys.stderr
     stderr_isatty: bool = stderr.isatty()
     colors = 256
+    program_name: str = 'http'
     if not is_windows:
         if curses:
             try:
@@ -79,16 +81,6 @@ class Environment:
             self.stdout_encoding = getattr(
                 actual_stdout, 'encoding', None) or 'utf8'
 
-    @property
-    def config(self) -> Config:
-        if not hasattr(self, '_config'):
-            self._config = Config(directory=self.config_dir)
-            if self._config.is_new():
-                self._config.save(fail_silently=True)
-            else:
-                self._config.load()
-        return self._config
-
     def __str__(self):
         defaults = dict(type(self).__dict__)
         actual = dict(defaults)
@@ -102,3 +94,21 @@ class Environment:
 
     def __repr__(self):
         return f'<{type(self).__name__} {self}>'
+
+    _config: Config = None
+
+    @property
+    def config(self) -> Config:
+        config = self._config
+        if not config:
+            self._config = config = Config(directory=self.config_dir)
+            if not config.is_new():
+                try:
+                    config.load()
+                except ConfigFileError as e:
+                    self.log_error(e, level='warning')
+        return config
+
+    def log_error(self, msg, level='error'):
+        assert level in ['error', 'warning']
+        self.stderr.write(f'\n{self.program_name}: {level}: {msg}\n\n')

httpie/core.py

@@ -1,25 +1,25 @@
 import argparse
+import os
 import platform
 import sys
-from typing import Callable, List, Union
+from typing import List, Union
 
 import requests
 from pygments import __version__ as pygments_version
 from requests import __version__ as requests_version
 
 from httpie import __version__ as httpie_version
-from httpie.status import ExitStatus, http_status_to_exit_status
 from httpie.client import collect_messages
 from httpie.context import Environment
 from httpie.downloads import Downloader
 from httpie.output.writer import write_message, write_stream
 from httpie.plugins import plugin_manager
+from httpie.status import ExitStatus, http_status_to_exit_status
 
 
 def main(
     args: List[Union[str, bytes]] = sys.argv,
     env=Environment(),
-    custom_log_error: Callable = None
 ) -> ExitStatus:
     """
     The main function.
@@ -30,22 +30,16 @@ def main(
     Return exit status code.
 
     """
-    args = decode_raw_args(args, env.stdin_encoding)
     program_name, *args = args
+    env.program_name = os.path.basename(program_name)
+    args = decode_raw_args(args, env.stdin_encoding)
     plugin_manager.load_installed_plugins()
 
-    def log_error(msg, level='error'):
-        assert level in ['error', 'warning']
-        env.stderr.write(f'\n{program_name}: {level}: {msg}\n')
-
     from httpie.cli.definition import parser
 
     if env.config.default_options:
         args = env.config.default_options + args
 
-    if custom_log_error:
-        log_error = custom_log_error
-
     include_debug_info = '--debug' in args
     include_traceback = include_debug_info or '--traceback' in args
 
@@ -59,7 +53,6 @@ def main(
     try:
         parsed_args = parser.parse_args(
             args=args,
-            program_name=program_name,
             env=env,
         )
     except KeyboardInterrupt:
@@ -78,7 +71,6 @@ def main(
             exit_status = program(
                 args=parsed_args,
                 env=env,
-                log_error=log_error,
             )
         except KeyboardInterrupt:
             env.stderr.write('\n')
@@ -93,10 +85,10 @@ def main(
                 exit_status = ExitStatus.ERROR
         except requests.Timeout:
             exit_status = ExitStatus.ERROR_TIMEOUT
-            log_error(f'Request timed out ({parsed_args.timeout}s).')
+            env.log_error(f'Request timed out ({parsed_args.timeout}s).')
         except requests.TooManyRedirects:
             exit_status = ExitStatus.ERROR_TOO_MANY_REDIRECTS
-            log_error(
+            env.log_error(
                 f'Too many redirects'
                 f' (--max-redirects=parsed_args.max_redirects).'
             )
@@ -110,7 +102,7 @@ def main(
                         f'{msg} while doing a {request.method}'
                         f' request to URL: {request.url}'
                     )
-            log_error(f'{type(e).__name__}: {msg}')
+            env.log_error(f'{type(e).__name__}: {msg}')
             if include_traceback:
                 raise
             exit_status = ExitStatus.ERROR
@@ -121,7 +113,6 @@ def main(
 def program(
     args: argparse.Namespace,
     env: Environment,
-    log_error: Callable
 ) -> ExitStatus:
     """
     The main program without error handling.
@@ -159,8 +150,9 @@ def program(
                         http_status=message.status_code,
                         follow=args.follow
                     )
-                    if not env.stdout_isatty and exit_status != ExitStatus.SUCCESS:
+                    if (not env.stdout_isatty
-                        log_error(
+                            and exit_status != ExitStatus.SUCCESS):
+                        env.log_error(
                             f'HTTP {message.raw.status} {message.raw.reason}',
                             level='warning'
                         )
@@ -179,7 +171,8 @@ def program(
             downloader.finish()
             if downloader.interrupted:
                 exit_status = ExitStatus.ERROR
-                log_error('Incomplete download: size=%d; downloaded=%d' % (
+                env.log_error(
+                    'Incomplete download: size=%d; downloaded=%d' % (
                         downloader.status.total_size,
                         downloader.status.downloaded
                     ))
@@ -196,11 +189,11 @@ def program(
 
 def print_debug_info(env: Environment):
     env.stderr.writelines([
-        'HTTPie %s\n' % httpie_version,
+        f'HTTPie {httpie_version}\n',
-        'Requests %s\n' % requests_version,
+        f'Requests {requests_version}\n',
-        'Pygments %s\n' % pygments_version,
+        f'Pygments {pygments_version}\n',
-        'Python %s\n%s\n' % (sys.version, sys.executable),
+        f'Python {sys.version}\n{sys.executable}\n',
-        '%s %s' % (platform.system(), platform.release()),
+        f'{platform.system()} {platform.release()}',
     ])
     env.stderr.write('\n\n')
     env.stderr.write(repr(env))

httpie/sessions.py

@@ -1,4 +1,5 @@
-"""Persistent, JSON-serialized sessions.
+"""
+Persistent, JSON-serialized sessions.
 
 """
 import os
@@ -53,8 +54,7 @@ class Session(BaseConfigDict):
     about = 'HTTPie session file'
 
     def __init__(self, path: Union[str, Path]):
-        super().__init__()
+        super().__init__(path=Path(path))
-        self._path = Path(path)
         self['headers'] = {}
         self['cookies'] = {}
         self['auth'] = {
@@ -63,9 +63,6 @@ class Session(BaseConfigDict):
             'password': None
         }
 
-    def _get_path(self) -> Path:
-        return self._path
-
     def update_headers(self, request_headers: RequestHeadersDict):
         """
         Update the session headers with the request ones while ignoring

tests/test_config.py

@@ -1,6 +1,5 @@
-from httpie import __version__
+from httpie.config import Config
-from utils import MockEnvironment, http, HTTP_OK
+from utils import HTTP_OK, MockEnvironment, http
-from httpie.context import Environment
 
 
 def test_default_options(httpbin):
@@ -8,15 +7,33 @@ def test_default_options(httpbin):
     env.config['default_options'] = ['--form']
     env.config.save()
     r = http(httpbin.url + '/post', 'foo=bar', env=env)
-    assert r.json['form'] == {"foo": "bar"}
+    assert r.json['form'] == {
+        "foo": "bar"
+    }
 
 
-def test_config_dir_not_writeable(httpbin):
+def test_config_file_not_valid(httpbin):
-    r = http(httpbin + '/get', env=MockEnvironment(
+    env = MockEnvironment()
-        config_dir='/',
+    env.create_temp_config_dir()
-        create_temp_config_dir=False,
+    with (env.config_dir / Config.FILENAME).open('w') as f:
-    ))
+        f.write('{invalid json}')
+    r = http(httpbin + '/get', env=env)
     assert HTTP_OK in r
+    assert 'http: warning' in r.stderr
+    assert 'invalid config file' in r.stderr
+
+
+def test_config_file_not_inaccessible(httpbin):
+    env = MockEnvironment()
+    env.create_temp_config_dir()
+    config_path = env.config_dir / Config.FILENAME
+    assert not config_path.exists()
+    config_path.touch(0o000)
+    assert config_path.exists()
+    r = http(httpbin + '/get', env=env)
+    assert HTTP_OK in r
+    assert 'http: warning' in r.stderr
+    assert 'cannot read config file' in r.stderr
 
 
 def test_default_options_overwrite(httpbin):
@@ -24,9 +41,6 @@ def test_default_options_overwrite(httpbin):
     env.config['default_options'] = ['--form']
     env.config.save()
     r = http('--json', httpbin.url + '/post', 'foo=bar', env=env)
-    assert r.json['json'] == {"foo": "bar"}
+    assert r.json['json'] == {
-
+        "foo": "bar"
-
+    }
-def test_current_version():
-    version = MockEnvironment().config['__meta__']['httpie']
-    assert version == __version__

tests/test_errors.py

@@ -4,37 +4,30 @@ from requests import Request
 from requests.exceptions import ConnectionError
 
 from httpie.status import ExitStatus
-from httpie.core import main
 from utils import HTTP_OK, http
 
 
-error_msg = None
-
-
 @mock.patch('httpie.core.program')
-def test_error(get_response):
+def test_error(program):
-    def error(msg, *args, **kwargs):
-        global error_msg
-        error_msg = msg % args
-
     exc = ConnectionError('Connection aborted')
     exc.request = Request(method='GET', url='http://www.google.com')
-    get_response.side_effect = exc
+    program.side_effect = exc
-    ret = main(['http', '--ignore-stdin', 'www.google.com'], custom_log_error=error)
+    r = http('www.google.com', tolerate_error_exit_status=True)
-    assert ret == ExitStatus.ERROR
+    assert r.exit_status == ExitStatus.ERROR
-    assert error_msg == (
+    assert (
                'ConnectionError: '
                'Connection aborted while doing a GET request to URL: '
-        'http://www.google.com')
+               'http://www.google.com'
+           ) in r.stderr
 
 
 @mock.patch('httpie.core.program')
-def test_error_traceback(get_response):
+def test_error_traceback(program):
     exc = ConnectionError('Connection aborted')
     exc.request = Request(method='GET', url='http://www.google.com')
-    get_response.side_effect = exc
+    program.side_effect = exc
     with raises(ConnectionError):
-        main(['http', '--ignore-stdin', '--traceback', 'www.google.com'])
+        http('--traceback', 'www.google.com')
 
 
 def test_max_headers_limit(httpbin_both):

tests/utils.py

@@ -6,7 +6,7 @@ import time
 import json
 import tempfile
 from pathlib import Path
-from typing import Optional
+from typing import Optional, Union
 
 from httpie.status import ExitStatus
 from httpie.config import Config
@@ -18,6 +18,7 @@ TESTS_ROOT = os.path.abspath(os.path.dirname(__file__))
 CRLF = '\r\n'
 COLOR = '\x1b['
 HTTP_OK = '200 OK'
+# noinspection GrazieInspection
 HTTP_OK_COLOR = (
     'HTTP\x1b[39m\x1b[38;5;245m/\x1b[39m\x1b'
     '[38;5;37m1.1\x1b[39m\x1b[38;5;245m \x1b[39m\x1b[38;5;37m200'
@@ -62,9 +63,12 @@ class MockEnvironment(Environment):
     def config(self) -> Config:
         if (self._create_temp_config_dir
                 and self._temp_dir not in self.config_dir.parents):
+            self.create_temp_config_dir()
+        return super().config
+
+    def create_temp_config_dir(self):
         self.config_dir = mk_config_dir()
         self._delete_config_dir = True
-        return super().config
 
     def cleanup(self):
         self.stdout.close()
@@ -75,6 +79,7 @@ class MockEnvironment(Environment):
             rmtree(self.config_dir)
 
     def __del__(self):
+        # noinspection PyBroadException
         try:
             self.cleanup()
         except Exception:
@@ -113,8 +118,8 @@ class StrCLIResponse(str, BaseCLIResponse):
     @property
     def json(self) -> Optional[dict]:
         """
-        Return deserialized JSON body, if one included in the output
+        Return deserialized the request or response JSON body,
-        and is parsable.
+        if one (and only one) included in the output and is parsable.
 
         """
         if not hasattr(self, '_json'):
@@ -147,7 +152,12 @@ class ExitStatusError(Exception):
     pass
 
 
-def http(*args, program_name='http', **kwargs):
+def http(
+    *args,
+    program_name='http',
+    tolerate_error_exit_status=False,
+    **kwargs,
+) -> Union[StrCLIResponse, BytesCLIResponse]:
     # noinspection PyUnresolvedReferences
     """
     Run HTTPie and capture stderr/out and exit status.
@@ -188,7 +198,6 @@ def http(*args, program_name='http', **kwargs):
         True
 
     """
-    tolerate_error_exit_status = kwargs.pop('tolerate_error_exit_status', False)
     env = kwargs.get('env')
     if not env:
         env = kwargs['env'] = MockEnvironment()
@@ -200,7 +209,8 @@ def http(*args, program_name='http', **kwargs):
     args_with_config_defaults = args + env.config.default_options
     add_to_args = []
     if '--debug' not in args_with_config_defaults:
-        if not tolerate_error_exit_status and '--traceback' not in args_with_config_defaults:
+        if (not tolerate_error_exit_status
+                and '--traceback' not in args_with_config_defaults):
             add_to_args.append('--traceback')
         if not any('--timeout' in arg for arg in args_with_config_defaults):
             add_to_args.append('--timeout=3')
@@ -228,7 +238,8 @@ def http(*args, program_name='http', **kwargs):
             sys.stderr.write(stderr.read())
             raise
         else:
-            if not tolerate_error_exit_status and exit_status != ExitStatus.SUCCESS:
+            if (not tolerate_error_exit_status
+                    and exit_status != ExitStatus.SUCCESS):
                 dump_stderr()
                 raise ExitStatusError(
                     'httpie.core.main() unexpectedly returned'