From dd2c9513f3d0ede22252435a791285c073f127da Mon Sep 17 00:00:00 2001 From: Batuhan Taskaya Date: Thu, 14 Apr 2022 18:11:12 +0300 Subject: [PATCH] Single binary executables (#1330) * Single binary executables / DEB packages. * Attach single binary executables to the releases --- .../workflows/release-linux-standalone.yml | 68 ++++++++++++ .gitignore | 4 +- docs/markdownlint.rb | 3 + docs/packaging/README.md | 25 ++--- docs/packaging/brew/README.md | 20 ++-- docs/packaging/brew/brew-deps.py | 81 -------------- docs/packaging/brew/httpie.rb | 24 ++--- docs/packaging/brew/update.sh | 6 ++ docs/packaging/linux-debian/README.md | 21 ++-- docs/packaging/linux-fedora/httpie.spec.txt | 6 +- docs/packaging/linux-fedora/update.sh | 6 ++ docs/packaging/snapcraft/README.md | 11 +- docs/packaging/windows-chocolatey/README.md | 7 +- extras/packaging/linux/Dockerfile | 32 ++++++ extras/packaging/linux/README.md | 52 +++++++++ extras/packaging/linux/build.py | 100 ++++++++++++++++++ .../packaging/linux/get_release_artifacts.sh | 22 ++++ .../packaging/linux/scripts/hooks/hook-pip.py | 14 +++ extras/packaging/linux/scripts/http_cli.py | 5 + extras/packaging/linux/scripts/httpie_cli.py | 5 + httpie/compat.py | 3 + httpie/manager/compat.py | 69 ++++++++++++ httpie/manager/tasks/plugins.py | 63 +++++------ httpie/plugins/manager.py | 18 ++-- httpie/utils.py | 23 +++- setup.py | 1 + 26 files changed, 510 insertions(+), 179 deletions(-) create mode 100644 .github/workflows/release-linux-standalone.yml delete mode 100755 docs/packaging/brew/brew-deps.py create mode 100755 docs/packaging/brew/update.sh create mode 100755 docs/packaging/linux-fedora/update.sh create mode 100644 extras/packaging/linux/Dockerfile create mode 100644 extras/packaging/linux/README.md create mode 100644 extras/packaging/linux/build.py create mode 100755 extras/packaging/linux/get_release_artifacts.sh create mode 100644 extras/packaging/linux/scripts/hooks/hook-pip.py create mode 100644 extras/packaging/linux/scripts/http_cli.py create mode 100644 extras/packaging/linux/scripts/httpie_cli.py create mode 100644 httpie/manager/compat.py diff --git a/.github/workflows/release-linux-standalone.yml b/.github/workflows/release-linux-standalone.yml new file mode 100644 index 00000000..967987d9 --- /dev/null +++ b/.github/workflows/release-linux-standalone.yml @@ -0,0 +1,68 @@ +name: Release as Standalone Linux Package + +on: + workflow_dispatch: + inputs: + branch: + description: "The branch, tag or SHA to release from" + required: true + default: "master" + + release: + types: [released, prereleased] + + +jobs: + binary-build-and-release: + name: Build and Release + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + ref: ${{ github.event.inputs.branch }} + + - uses: actions/setup-python@v3 + with: + python-version: 3.9 + + - name: Build Artifacts + run: | + cd extras/packaging/linux + ./get_release_artifacts.sh + + - uses: actions/upload-artifact@v3 + with: + name: http + path: extras/packaging/linux/artifacts/dist/http + + - uses: actions/upload-artifact@v3 + with: + name: httpie.deb + path: extras/packaging/linux/artifacts/dist/*.deb + + - uses: actions/upload-artifact@v3 + with: + name: httpie.rpm + path: extras/packaging/linux/artifacts/dist/*.rpm + + - name: Publish Debian Package + if: github.event_name == 'release' + uses: actions/upload-release-asset@v1.0.2 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ github.event.release.upload_url }} + asset_path: extras/packaging/linux/artifacts/dist/httpie-${{ github.event.release.tag_name }}.deb + asset_name: httpie-${{ github.event.release.tag_name }}.deb + asset_content_type: binary/octet-stream + + - name: Publish Single Executable + if: github.event_name == 'release' + uses: actions/upload-release-asset@v1.0.2 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ github.event.release.upload_url }} + asset_path: extras/packaging/linux/artifacts/dist/http + asset_name: http + asset_content_type: binary/octet-stream diff --git a/.gitignore b/.gitignore index 4f4d9454..aee3301c 100644 --- a/.gitignore +++ b/.gitignore @@ -43,8 +43,8 @@ MANIFEST # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest *.spec +*.manifest # Installer logs pip-log.txt @@ -151,3 +151,5 @@ dmypy.json # Windows Chocolatey *.nupkg + +artifacts/ diff --git a/docs/markdownlint.rb b/docs/markdownlint.rb index 0bf3f87b..cf2878c7 100644 --- a/docs/markdownlint.rb +++ b/docs/markdownlint.rb @@ -11,6 +11,9 @@ all # Because we use HTML to hide them on the website. exclude_rule 'MD002' +# MD007 Allow unordered list indentation +exclude_rule 'MD007' + # MD013 Line length exclude_rule 'MD013' diff --git a/docs/packaging/README.md b/docs/packaging/README.md index c8d18b28..5354b5e5 100644 --- a/docs/packaging/README.md +++ b/docs/packaging/README.md @@ -12,16 +12,18 @@ You are looking at the HTTPie packaging documentation, where you will find valua The overall release process starts simple: -1. Do the [PyPI](https://pypi.org/project/httpie/) publication. -2. Then, handle company-related tasks. -3. Finally, follow OS-specific steps, described in documents below, to send patches downstream. +1. Bump the version identifiers in the following places: + - `httpie/__init__.py` + - `docs/packaging/windows-chocolatey/httpie.nuspec` + - `CHANGELOG.md` +2. Commit your changes and make a PR against the `master`. +3. Merge the PR, and tag the last commit with your version identifier. +4. Make a GitHub release (by copying the text in `CHANGELOG.md`) +5. Push that release to PyPI (dispatch the `Release PyPI` GitHub action). +6. Once PyPI is ready, push the release to the Snap, Homebrew and Chocolatey with their respective actions. +7. Go to the [`httpie/debian.httpie.io`](https://github.com/httpie/debian.httpie.io) repo and trigger the package index workflow. -## First, PyPI - -Let's do the release on [PyPi](https://pypi.org/project/httpie/). -That is done quite easily by manually triggering the [release workflow](https://github.com/httpie/httpie/actions/workflows/release.yml). - -## Then, company-specific tasks +## Company-specific tasks - Blank the `master_and_released_docs_differ_after` value in [config.json](https://github.com/httpie/httpie/blob/master/docs/config.json). - Update the [contributors list](../contributors). @@ -36,10 +38,9 @@ A more complete state of deployment can be found on [repology](https://repology. | -------------------------------------------: | -------------- | | [Arch Linux, and derived](linux-arch/) | trusted person | | [CentOS, RHEL, and derived](linux-centos/) | trusted person | -| [Debian, Ubuntu, and derived](linux-debian/) | trusted person | | [Fedora](linux-fedora/) | trusted person | -| :construction: [Homebrew, Linuxbrew](brew/) | **HTTPie** | -| :construction: [MacPorts](mac-ports/) | **HTTPie** | +| [Debian, Ubuntu, and derived](linux-debian/) | **HTTPie** | +| [Homebrew, Linuxbrew](brew/) | **HTTPie** | | [Snapcraft](snapcraft/) | **HTTPie** | | [Windows — Chocolatey](windows-chocolatey/) | **HTTPie** | diff --git a/docs/packaging/brew/README.md b/docs/packaging/brew/README.md index 2acb3299..6c709499 100644 --- a/docs/packaging/brew/README.md +++ b/docs/packaging/brew/README.md @@ -13,21 +13,19 @@ We will discuss setting up the environment, installing development tools, instal ## Overall process -:construction: Work in progress. +The brew deployment is completely automated, and only requires a trigger to [`Release on Homebrew`](https://github.com/httpie/httpie/actions/workflows/release-brew.yml) action +from the release manager. -First, update the current Formula: +If it is needed to be done manually, the following command can be used: -```bash -make brew-deps -# Copy-paste content into downstream/mac/brew/httpie.rb -git add downstream/mac/brew/httpie.rb -git commit -s -m 'Update brew formula to XXX' +```console +$ brew bump-formula-pr httpie --version={TARGET_VERSION} ``` -That [GitHub workflow](https://github.com/httpie/httpie/actions/workflows/test-package-mac-brew.yml) will test the formula when `downstream/mac/brew/httpie.rb` is changed in a pull request. - -Then, open a pull request with those changes to the [downstream file](https://github.com/Homebrew/homebrew-core/blob/master/Formula/httpie.rb). +which will bump the formala, and create a PR against the package index. ## Hacking -:construction: Work in progress. +Make your changes, test the formula through the [`Test Brew Package`](https://github.com/httpie/httpie/actions/workflows/test-package-mac-brew.yml) action +and then finally submit your patch to [`homebrew-core`](https://github.com/Homebrew/homebrew-core`) + diff --git a/docs/packaging/brew/brew-deps.py b/docs/packaging/brew/brew-deps.py deleted file mode 100755 index 3e456839..00000000 --- a/docs/packaging/brew/brew-deps.py +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/env python3 -""" -Generate Ruby code with URLs and file hashes for packages from PyPi -(i.e., httpie itself as well as its dependencies) to be included -in the Homebrew formula after a new release of HTTPie has been published -on PyPi. - - - -""" -import hashlib -import requests - - -VERSIONS = { - # By default, we use the latest packages. But sometimes Requests has a maximum supported versions. - # Take a look here before making a release: - 'idna': '3.2', -} - - -# Note: Keep that list sorted. -PACKAGES = [ - 'certifi', - 'charset-normalizer', - 'defusedxml', - 'httpie', - 'idna', - 'Pygments', - 'PySocks', - 'requests', - 'requests-toolbelt', - 'urllib3', - 'multidict', -] - - -def get_package_meta(package_name): - api_url = f'https://pypi.org/pypi/{package_name}/json' - resp = requests.get(api_url).json() - hasher = hashlib.sha256() - version = VERSIONS.get(package_name) - if package_name not in VERSIONS: - # Latest version - release_bundle = resp['urls'] - else: - release_bundle = resp['releases'][version] - - for release in release_bundle: - download_url = release['url'] - if download_url.endswith('.tar.gz'): - hasher.update(requests.get(download_url).content) - return { - 'name': package_name, - 'url': download_url, - 'sha256': hasher.hexdigest(), - } - else: - raise RuntimeError(f'{package_name}: download not found: {resp}') - - -def main(): - package_meta_map = { - package_name: get_package_meta(package_name) - for package_name in PACKAGES - } - httpie_meta = package_meta_map.pop('httpie') - print() - print(' url "{url}"'.format(url=httpie_meta['url'])) - print(' sha256 "{sha256}"'.format(sha256=httpie_meta['sha256'])) - print() - for dep_meta in package_meta_map.values(): - print(' resource "{name}" do'.format(name=dep_meta['name'])) - print(' url "{url}"'.format(url=dep_meta['url'])) - print(' sha256 "{sha256}"'.format(sha256=dep_meta['sha256'])) - print(' end') - print('') - - -if __name__ == '__main__': - main() diff --git a/docs/packaging/brew/httpie.rb b/docs/packaging/brew/httpie.rb index ed09cb53..6a6257ed 100644 --- a/docs/packaging/brew/httpie.rb +++ b/docs/packaging/brew/httpie.rb @@ -3,18 +3,18 @@ class Httpie < Formula desc "User-friendly cURL replacement (command-line HTTP client)" homepage "https://httpie.io/" - url "https://files.pythonhosted.org/packages/7b/f9/13070f19226b7db3641fb787df36bb715063abe1b8ca03fbaeca0f465d27/httpie-3.0.1.tar.gz" - sha256 "0e9bc93ebdcdd2d32ec24b8fa46cf7e4fde9eec7a6bd0c5d0ef224f25d7466b2" + url "https://files.pythonhosted.org/packages/32/85/bb095699be20cc98731261cb80884e9458178f8fef2a38273530ce77c0a5/httpie-3.1.0.tar.gz" + sha256 "2e4a2040b84a912e65c01fb34f7aafe88cad2a3af2da8c685ca65080f376feda" license "BSD-3-Clause" head "https://github.com/httpie/httpie.git", branch: "master" bottle do - sha256 cellar: :any_skip_relocation, arm64_monterey: "9d285fcfb55ce8ed787d1b01966d51e6e07f7e77c44a204695a2d6eee9c8698d" - sha256 cellar: :any_skip_relocation, arm64_big_sur: "743a282b475e87a4eaf11e545f761aef1b8e4bfe49eaee47251d7629a35a8ced" - sha256 cellar: :any_skip_relocation, monterey: "5d63ea4f47b2028b2ba68abe12a4176934193e058edd869270221b41cc946c76" - sha256 cellar: :any_skip_relocation, big_sur: "5a53221a680a35d1aa00cbadde279dbe4f562d22ed207c15bd4221cb8c3180f1" - sha256 cellar: :any_skip_relocation, catalina: "5feadb6d76f55d6f9681682e221008c282dccf0e46ae22a959b4bad2efde204a" - sha256 cellar: :any_skip_relocation, x86_64_linux: "d530ddbec49588b0d481f156d35f7e5bb7d3b6427d203f04750e55cd3eecc303" + sha256 cellar: :any_skip_relocation, arm64_monterey: "9bb6e8c1ef5ba8b019ddedd7e908dd2174da695351aa9a238dfb28b0f57ef005" + sha256 cellar: :any_skip_relocation, arm64_big_sur: "47ffccd3241155d863e1b4f6259d538a34d42a0cdeed8152bda257ee607b51be" + sha256 cellar: :any_skip_relocation, monterey: "dc4a04cb05a9cd1bfa6a632a0e4a21975905954af54ece41f9050c52474267be" + sha256 cellar: :any_skip_relocation, big_sur: "ae469e37864e967e0fd99fba15a78e719dcb351b462f98f3843c78ed1473df6d" + sha256 cellar: :any_skip_relocation, catalina: "291a3eaecb2a2cc845c1652686a9a14b21053d7e3a7d0115245b2150ca2e199e" + sha256 cellar: :any_skip_relocation, x86_64_linux: "710836e27c44c8e3ad181d668f4a9f78c4cb4c355d7b148a397599a7cd42713d" end depends_on "python@3.10" @@ -25,8 +25,8 @@ class Httpie < Formula end resource "charset-normalizer" do - url "https://files.pythonhosted.org/packages/48/44/76b179e0d1afe6e6a91fd5661c284f60238987f3b42b676d141d01cd5b97/charset-normalizer-2.0.10.tar.gz" - sha256 "876d180e9d7432c5d1dfd4c5d26b72f099d503e8fcc0feb7532c9289be60fcbd" + url "https://files.pythonhosted.org/packages/56/31/7bcaf657fafb3c6db8c787a865434290b726653c912085fbd371e9b92e1c/charset-normalizer-2.0.12.tar.gz" + sha256 "2857e29ff0d34db842cd7ca3230549d1a697f96ee6d3fb071cfa6c7393832597" end resource "defusedxml" do @@ -40,8 +40,8 @@ class Httpie < Formula end resource "multidict" do - url "https://files.pythonhosted.org/packages/8e/7c/e12a69795b7b7d5071614af2c691c97fbf16a2a513c66ec52dd7d0a115bb/multidict-5.2.0.tar.gz" - sha256 "0dd1c93edb444b33ba2274b66f63def8a327d607c6c790772f448a53b6ea59ce" + url "https://files.pythonhosted.org/packages/fa/a7/71c253cdb8a1528802bac7503bf82fe674367e4055b09c28846fdfa4ab90/multidict-6.0.2.tar.gz" + sha256 "5ff3bd75f38e4c43f1f470f2df7a4d430b821c4ce22be384e1459cb57d6bb013" end resource "Pygments" do diff --git a/docs/packaging/brew/update.sh b/docs/packaging/brew/update.sh new file mode 100755 index 00000000..7c26ae0b --- /dev/null +++ b/docs/packaging/brew/update.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +set -xe + +rm -f httpie.rb +http --download https://raw.githubusercontent.com/Homebrew/homebrew-core/master/Formula/httpie.rb diff --git a/docs/packaging/linux-debian/README.md b/docs/packaging/linux-debian/README.md index f43ff3c4..10c409f7 100644 --- a/docs/packaging/linux-debian/README.md +++ b/docs/packaging/linux-debian/README.md @@ -11,19 +11,16 @@ Welcome to the documentation about **packaging HTTPie for Debian GNU/Linux**. This document contains technical details, where we describe how to create a patch for the latest HTTPie version for Debian GNU/Linux. They apply to Ubuntu as well, and any Debian-derived distributions like MX Linux, Linux Mint, deepin, Pop!_OS, KDE neon, Zorin OS, elementary OS, Kubuntu, Devuan, Linux Lite, Peppermint OS, Lubuntu, antiX, Xubuntu, etc. We will discuss setting up the environment, installing development tools, installing and testing changes before submitting a patch downstream. -The current maintainer is Bartosz Fenski. +We create the standalone binaries (see this [for more details](../../../extras/packaging/linux/)) and package them with +[FPM](https://github.com/jordansissel/fpm)'s `dir` mode. The core `http`/`https` commands don't have any dependencies, but the `httpie` +command (due to the underlying `httpie cli plugins` interface) explicitly depends to the system Python (through `python3`/`python3-pip`). ## Overall process -Open a new bug on the Debian Bug Tracking System by sending an email: +The [`Release as Standalone Linux Binary`](https://github.com/httpie/httpie/actions/workflows/release-linux-standalone.yml) will be automatically +triggered when a new release is created, and it will submit the `.deb` package as a release asset. -- To: `Debian Bug Tracking System ` -- Subject: `httpie: Version XXX available` -- Message template (examples [1](https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=993937), and [2](https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=996479)): - - ```email - Package: httpie - Severity: normal - - - ``` +For making that asset available for all debian users, the release manager needs to go to the [`httpie/debian.httpie.io`](https://github.com/httpie/debian.httpie.io) repo +and trigger the [`Update Index`](https://github.com/httpie/debian.httpie.io/actions/workflows/update-index.yml) action. It will automatically +scrape all new debian packages from the release assets, properly update the indexes and create a new PR ([an example](https://github.com/httpie/debian.httpie.io/pull/1)) +which then will become active when merged. diff --git a/docs/packaging/linux-fedora/httpie.spec.txt b/docs/packaging/linux-fedora/httpie.spec.txt index f18f4878..5ac2097e 100644 --- a/docs/packaging/linux-fedora/httpie.spec.txt +++ b/docs/packaging/linux-fedora/httpie.spec.txt @@ -1,5 +1,5 @@ Name: httpie -Version: 3.0.2 +Version: 3.1.0 Release: 1%{?dist} Summary: A Curl-like tool for humans @@ -78,6 +78,10 @@ help2man %{buildroot}%{_bindir}/httpie > %{buildroot}%{_mandir}/man1/httpie.1 %changelog +* Tue Mar 08 2022 Miro Hrončok - 3.1.0-1 +- Update to 3.1.0 +- Fixes: rhbz#2061597 + * Mon Jan 24 2022 Miro Hrončok - 3.0.2-1 - Update to 3.0.2 - Fixes: rhbz#2044572 diff --git a/docs/packaging/linux-fedora/update.sh b/docs/packaging/linux-fedora/update.sh new file mode 100755 index 00000000..773bef33 --- /dev/null +++ b/docs/packaging/linux-fedora/update.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +set -xe + +rm -f httpie.spec.txt +https --download src.fedoraproject.org/rpms/httpie/raw/rawhide/f/httpie.spec -o httpie.spec.txt diff --git a/docs/packaging/snapcraft/README.md b/docs/packaging/snapcraft/README.md index b22a00ef..fda65341 100644 --- a/docs/packaging/snapcraft/README.md +++ b/docs/packaging/snapcraft/README.md @@ -13,7 +13,16 @@ We will discuss setting up the environment, installing development tools, instal ## Overall process -Trigger a new [build](https://snapcraft.io/httpie/builds), then [promote it](https://snapcraft.io/httpie/releases). If more management is needed: [revisions supervision](https://dashboard.snapcraft.io/snaps/httpie/revisions/). +Trigger the [`Release on Snap`](https://github.com/httpie/httpie/actions/workflows/release-snap.yml) action, which will +create a snap package for HTTPie and then push it to Snap Store in the following channels: + +- Edge +- Beta +- Candidate +- Stable + +If a push to any of them fail, all the release tasks for the following channels will be cancelled so that the +release manager can look into the underlying cause. ## Hacking diff --git a/docs/packaging/windows-chocolatey/README.md b/docs/packaging/windows-chocolatey/README.md index 588fd7e7..50ee9d70 100644 --- a/docs/packaging/windows-chocolatey/README.md +++ b/docs/packaging/windows-chocolatey/README.md @@ -13,13 +13,18 @@ We will discuss setting up the environment, installing development tools, instal ## Overall process -After having successfully [built and tested](#hacking) the package, push it: +After having successfully [built and tested](#hacking) the package, either trigger the +[`Release on Chocolatey`](https://github.com/httpie/httpie/actions/workflows/release-choco.yml) action +to push it to the `Chocolatey` store or use the CLI: ```bash # Replace 2.5.0 with the correct version choco push httpie.2.5.0.nupkg -s https://push.chocolatey.org/ --api-key=API_KEY ``` +Be aware that it might take multiple days until the release is approved, sine it goes through multiple +sets of reviews (some of them are done manually). + ## Hacking ```bash diff --git a/extras/packaging/linux/Dockerfile b/extras/packaging/linux/Dockerfile new file mode 100644 index 00000000..bd554dd3 --- /dev/null +++ b/extras/packaging/linux/Dockerfile @@ -0,0 +1,32 @@ +# Use the oldest (but still supported) Ubuntu as the base for PyInstaller +# packages. This will prevent stuff like glibc from conflicting. +FROM ubuntu:18.04 + +RUN apt-get update +RUN apt-get install -y software-properties-common binutils +RUN apt-get install -y ruby-dev +RUN gem install fpm + +# Use deadsnakes for the latest Pythons (e.g 3.9) +RUN add-apt-repository ppa:deadsnakes/ppa +RUN apt-get update && apt-get install -y python3.9 python3.9-dev python3.9-venv + +# Install rpm as well, since we are going to build fedora dists too +RUN apt-get install -y rpm + +ADD . /app +WORKDIR /app/extras/packaging/linux + +ENV VIRTUAL_ENV=/opt/venv +RUN python3.9 -m venv $VIRTUAL_ENV +ENV PATH="$VIRTUAL_ENV/bin:$PATH" + +# Ensure that pip is renewed, otherwise we would be using distro-provided pip +# which strips vendored packages and doesn't work with PyInstaller. +RUN python -m pip install /app +RUN python -m pip install pyinstaller wheel +RUN python -m pip install --force-reinstall --upgrade pip + +RUN python build.py + +ENTRYPOINT ["mv", "/app/extras/packaging/linux/dist/", "/artifacts"] diff --git a/extras/packaging/linux/README.md b/extras/packaging/linux/README.md new file mode 100644 index 00000000..a62e4dc5 --- /dev/null +++ b/extras/packaging/linux/README.md @@ -0,0 +1,52 @@ +# Standalone Linux Packages + +![packaging.png](https://user-images.githubusercontent.com/47358913/159950478-2d090d1b-69b9-4914-a1b4-d3e3d8e25fe0.png) + +This directory contains the build scripts for creating: + +- A self-contained binary executable for the HTTPie itself +- `httpie.deb` and `httpie.rpm` packages for Debian and Fedora. + +The process of constructing them are fully automated, and can be easily done through the [`Release as Standalone Linux Package`](https://github.com/httpie/httpie/actions/workflows/release-linux-standalone.yml) +action. Once it finishes, the release artifacts will be attached in the summary page of the triggered run. + + +## Hacking + +The main entry point for the package builder is the [`build.py`](https://github.com/httpie/httpie/blob/master/extras/packaging/linux/build.py). It +contains 2 major methods: + +- `build_binaries`, for the self-contained executables +- `build_packages`, for the OS-specific packages (which wrap the binaries) + +### `build_binaries` + +We use [PyInstaller](https://pyinstaller.readthedocs.io/en/stable/) for the binaries. Normally pyinstaller offers two different modes: + +- Single directory (harder to distribute, low redundancy. Library files are shared accross different executables) +- Single binary (easier to distribute, higher redundancy. Same libraries are statically linked to different executables, so higher total size) + +Since our binary size (in total 20 MiBs) is not that big, we have decided to choose the single binary mode for the sake of easier distribution. + +We also disable `UPX`, which is a runtime decompression method since it adds some startup cost. + +### `build_packages` + +We build our OS-specific packages with [FPM](https://github.com/jordansissel/fpm) which offers a really nice abstraction. We use the `dir` mode, +and package `http`, `https` and `httpie` commands. More can be added to the `files` option. + +Since the `httpie` depends on having a pip executable, we explicitly depend on the system Python even though the core does not use it. + +### Docker Image + +This directory also contains a [docker image](https://github.com/httpie/httpie/blob/master/extras/packaging/linux/Dockerfile) which helps +building our standalone binaries in an isolated environment with the lowest possible library versions. This is important, since even though +the executables are standalone they still depend on some main system C libraries (like `glibc`) so we need to create our executables inside +an environment with a very old (but not deprecated) glibc version. It makes us soundproof for all active Ubuntu/Debian versions. + +It also contains the Python version we package our HTTPie with, so it is the place if you need to change it. + +### `./get_release_artifacts.sh` + +If you make a change in the `build.py`, run the following script to test it out. It will return multiple files under `artifacts/dist` which +then you can test out and ensure their quality (it is also the script that we use in our automation). diff --git a/extras/packaging/linux/build.py b/extras/packaging/linux/build.py new file mode 100644 index 00000000..534708bb --- /dev/null +++ b/extras/packaging/linux/build.py @@ -0,0 +1,100 @@ +import stat +import subprocess +from pathlib import Path +from typing import Iterator, Tuple + +BUILD_DIR = Path(__file__).parent +HTTPIE_DIR = BUILD_DIR.parent.parent.parent + +SCRIPT_DIR = BUILD_DIR / Path('scripts') +HOOKS_DIR = SCRIPT_DIR / 'hooks' + +DIST_DIR = BUILD_DIR / 'dist' + +TARGET_SCRIPTS = { + SCRIPT_DIR / 'http_cli.py': [], + SCRIPT_DIR / 'httpie_cli.py': ['--hidden-import=pip'], +} + + +def build_binaries() -> Iterator[Tuple[str, Path]]: + for target_script, extra_args in TARGET_SCRIPTS.items(): + subprocess.check_call( + [ + 'pyinstaller', + '--onefile', + '--noupx', + '-p', + HTTPIE_DIR, + '--additional-hooks-dir', + HOOKS_DIR, + *extra_args, + target_script, + ] + ) + + for executable_path in DIST_DIR.iterdir(): + if executable_path.suffix: + continue + stat_r = executable_path.stat() + executable_path.chmod(stat_r.st_mode | stat.S_IEXEC) + yield executable_path.stem, executable_path + + +def build_packages(http_binary: Path, httpie_binary: Path) -> None: + import httpie + + # Mapping of src_file -> dst_file + files = [ + (http_binary, '/usr/bin/http'), + (http_binary, '/usr/bin/https'), + (httpie_binary, '/usr/bin/httpie'), + ] + # A list of additional dependencies + deps = [ + 'python3 >= 3.7', + 'python3-pip' + ] + + processed_deps = [ + f'--depends={dep}' + for dep in deps + ] + processed_files = [ + '='.join([str(src.resolve()), dst]) for src, dst in files + ] + for target in ['deb', 'rpm']: + subprocess.check_call( + [ + 'fpm', + '--force', + '-s', + 'dir', + '-t', + target, + '--name', + 'httpie', + '--version', + httpie.__version__, + '--description', + httpie.__doc__.strip(), + '--license', + httpie.__licence__, + *processed_deps, + *processed_files, + ], + cwd=DIST_DIR, + ) + + +def main(): + binaries = dict(build_binaries()) + build_packages(binaries['http_cli'], binaries['httpie_cli']) + + # Rename http_cli/httpie_cli to http/httpie + binaries['http_cli'].rename('http') + binaries['httpie_cli'].rename('httpie') + + +if __name__ == '__main__': + main() diff --git a/extras/packaging/linux/get_release_artifacts.sh b/extras/packaging/linux/get_release_artifacts.sh new file mode 100755 index 00000000..b56e8b83 --- /dev/null +++ b/extras/packaging/linux/get_release_artifacts.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +set -xe + +REPO_ROOT=../../../ +ARTIFACTS_DIR=$(pwd)/artifacts + +# Reset the ARTIFACTS_DIR. +rm -rf $ARTIFACTS_DIR +mkdir -p $ARTIFACTS_DIR + +# Operate on the repository root to have the proper +# docker context. +pushd $REPO_ROOT + +# Build the PyInstaller image +docker build -t pyinstaller-httpie -f extras/packaging/linux/Dockerfile . + +# Copy the artifacts to the designated directory. +docker run --rm -i -v $ARTIFACTS_DIR:/artifacts pyinstaller-httpie:latest + +popd diff --git a/extras/packaging/linux/scripts/hooks/hook-pip.py b/extras/packaging/linux/scripts/hooks/hook-pip.py new file mode 100644 index 00000000..1dac8fb1 --- /dev/null +++ b/extras/packaging/linux/scripts/hooks/hook-pip.py @@ -0,0 +1,14 @@ +from pathlib import Path +from PyInstaller.utils.hooks import collect_all + +def hook(hook_api): + for pkg in [ + 'pip', + 'setuptools', + 'distutils', + 'pkg_resources' + ]: + datas, binaries, hiddenimports = collect_all(pkg) + hook_api.add_datas(datas) + hook_api.add_binaries(binaries) + hook_api.add_imports(*hiddenimports) diff --git a/extras/packaging/linux/scripts/http_cli.py b/extras/packaging/linux/scripts/http_cli.py new file mode 100644 index 00000000..12ac773f --- /dev/null +++ b/extras/packaging/linux/scripts/http_cli.py @@ -0,0 +1,5 @@ +from httpie.__main__ import main + +if __name__ == '__main__': + import sys + sys.exit(main()) diff --git a/extras/packaging/linux/scripts/httpie_cli.py b/extras/packaging/linux/scripts/httpie_cli.py new file mode 100644 index 00000000..1e979c95 --- /dev/null +++ b/extras/packaging/linux/scripts/httpie_cli.py @@ -0,0 +1,5 @@ +from httpie.manager.__main__ import main + +if __name__ == '__main__': + import sys + sys.exit(main()) diff --git a/httpie/compat.py b/httpie/compat.py index 133d2a52..fcf167ca 100644 --- a/httpie/compat.py +++ b/httpie/compat.py @@ -12,7 +12,10 @@ cookiejar.DefaultCookiePolicy = HTTPieCookiePolicy is_windows = 'win32' in str(sys.platform).lower() +is_frozen = getattr(sys, 'frozen', False) +MIN_SUPPORTED_PY_VERSION = (3, 7) +MAX_SUPPORTED_PY_VERSION = (3, 11) try: from functools import cached_property diff --git a/httpie/manager/compat.py b/httpie/manager/compat.py new file mode 100644 index 00000000..0f787eb8 --- /dev/null +++ b/httpie/manager/compat.py @@ -0,0 +1,69 @@ +import sys +import shutil +import subprocess + +from contextlib import suppress +from typing import List, Optional +from httpie.compat import is_frozen + + +class PipError(Exception): + """An exception that occurs when pip exits with an error status code.""" + + def __init__(self, stdout, stderr): + self.stdout = stdout + self.stderr = stderr + + +def _discover_system_pip() -> List[str]: + # When we are running inside of a frozen binary, we need the system + # pip to install plugins since there is no way for us to execute any + # code outside of the HTTPie. + # + # We explicitly depend on system pip, so the SystemError should not + # be executed (except for broken installations). + def _check_pip_version(pip_location: Optional[str]) -> bool: + if not pip_location: + return False + + with suppress(subprocess.CalledProcessError): + stdout = subprocess.check_output([pip_location, "--version"], text=True) + return "python 3" in stdout + + targets = [ + "pip", + "pip3" + ] + for target in targets: + pip_location = shutil.which(target) + if _check_pip_version(pip_location): + return pip_location + + raise SystemError("Couldn't find 'pip' executable. Please ensure that pip in your system is available.") + + +def _run_pip_subprocess(pip_executable: List[str], args: List[str]) -> bytes: + import subprocess + + cmd = [*pip_executable, *args] + try: + process = subprocess.run( + cmd, + check=True, + shell=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ) + except subprocess.CalledProcessError as error: + raise PipError(error.stdout, error.stderr) from error + else: + return process.stdout + + +def run_pip(args: List[str]) -> bytes: + if is_frozen: + pip_executable = [_discover_system_pip()] + else: + pip_executable = [sys.executable, '-m', 'pip'] + + return _run_pip_subprocess(pip_executable, args) diff --git a/httpie/manager/tasks/plugins.py b/httpie/manager/tasks/plugins.py index e6b1ee23..a0180537 100644 --- a/httpie/manager/tasks/plugins.py +++ b/httpie/manager/tasks/plugins.py @@ -1,20 +1,19 @@ import argparse import os +import textwrap import re import shutil -import subprocess -import sys -import textwrap from collections import defaultdict from contextlib import suppress from pathlib import Path from typing import List, Optional, Tuple +from httpie.manager.compat import PipError, run_pip +from httpie.manager.cli import parser, missing_subcommand from httpie.compat import get_dist_name, importlib_metadata from httpie.context import Environment -from httpie.manager.cli import missing_subcommand, parser from httpie.status import ExitStatus -from httpie.utils import as_site +from httpie.utils import get_site_paths PEP_503 = re.compile(r"[-_.]+") @@ -58,46 +57,37 @@ class PluginInstaller: self.env.stderr.write(message + '\n') return ExitStatus.ERROR - def pip(self, *args, **kwargs) -> subprocess.CompletedProcess: - options = { - 'check': True, - 'shell': False, - 'stdout': self.env.stdout, - 'stderr': subprocess.PIPE, - } - options.update(kwargs) - - cmd = [sys.executable, '-m', 'pip', *args] - return subprocess.run( - cmd, - **options - ) - - def _install(self, targets: List[str], mode='install', **process_options) -> Tuple[ - Optional[bytes], ExitStatus + def _install(self, targets: List[str], mode='install') -> Tuple[ + bytes, ExitStatus ]: pip_args = [ 'install', + '--prefer-binary', f'--prefix={self.dir}', '--no-warn-script-location', ] if mode == 'upgrade': pip_args.append('--upgrade') + pip_args.extend(targets) try: - process = self.pip( - *pip_args, - *targets, - **process_options, - ) - except subprocess.CalledProcessError as error: + stdout = run_pip(pip_args) + except PipError as pip_error: + error = pip_error + stdout = pip_error.stdout + else: + error = None + + self.env.stdout.write(stdout.decode()) + + if error: reason = None if error.stderr: stderr = error.stderr.decode() if self.debug: self.env.stderr.write('Command failed: ') - self.env.stderr.write(' '.join(error.cmd) + '\n') + self.env.stderr.write('pip ' + ' '.join(pip_args) + '\n') self.env.stderr.write(textwrap.indent(' ', stderr)) last_line = stderr.strip().splitlines()[-1] @@ -108,7 +98,6 @@ class PluginInstaller: stdout = error.stdout exit_status = self.fail(mode, ', '.join(targets), reason) else: - stdout = process.stdout exit_status = ExitStatus.SUCCESS return stdout, exit_status @@ -124,10 +113,11 @@ class PluginInstaller: # existing metadata for old versions manually. # [0]: https://github.com/pypa/pip/issues/10727 result_deps = defaultdict(list) - for child in as_site(self.dir).iterdir(): - if child.suffix in {'.dist-info', '.egg-info'}: - name, _, version = child.stem.rpartition('-') - result_deps[name].append((version, child)) + for site_dir in get_site_paths(self.dir): + for child in site_dir.iterdir(): + if child.suffix in {'.dist-info', '.egg-info'}: + name, _, version = child.stem.rpartition('-') + result_deps[name].append((version, child)) for target in targets: name, _, version = target.rpartition('-') @@ -145,15 +135,12 @@ class PluginInstaller: raw_stdout, exit_status = self._install( targets, - mode='upgrade', - stdout=subprocess.PIPE + mode='upgrade' ) if not raw_stdout: return exit_status stdout = raw_stdout.decode() - self.env.stdout.write(stdout) - installation_line = stdout.splitlines()[-1] if installation_line.startswith('Successfully installed'): self._clear_metadata(installation_line.split()[2:]) diff --git a/httpie/plugins/manager.py b/httpie/plugins/manager.py index 7a60a5c0..0a25f320 100644 --- a/httpie/plugins/manager.py +++ b/httpie/plugins/manager.py @@ -4,13 +4,13 @@ import warnings from itertools import groupby from operator import attrgetter -from typing import Dict, List, Type, Iterator, Optional, ContextManager +from typing import Dict, List, Type, Iterator, Iterable, Optional, ContextManager from pathlib import Path from contextlib import contextmanager, nullcontext from ..compat import importlib_metadata, find_entry_points, get_dist_name -from ..utils import repr_dict, as_site +from ..utils import repr_dict, get_site_paths from . import AuthPlugin, ConverterPlugin, FormatterPlugin, TransportPlugin from .base import BasePlugin @@ -25,20 +25,24 @@ ENTRY_POINT_NAMES = list(ENTRY_POINT_CLASSES.keys()) @contextmanager -def _load_directory(plugins_dir: Path) -> Iterator[None]: - plugins_path = os.fspath(plugins_dir) - sys.path.insert(0, plugins_path) +def _load_directories(site_dirs: Iterable[Path]) -> Iterator[None]: + plugin_dirs = [ + os.fspath(site_dir) + for site_dir in site_dirs + ] + sys.path.extend(plugin_dirs) try: yield finally: - sys.path.remove(plugins_path) + for plugin_dir in plugin_dirs: + sys.path.remove(plugin_dir) def enable_plugins(plugins_dir: Optional[Path]) -> ContextManager[None]: if plugins_dir is None: return nullcontext() else: - return _load_directory(as_site(plugins_dir)) + return _load_directories(get_site_paths(plugins_dir)) class PluginManager(list): diff --git a/httpie/utils.py b/httpie/utils.py index 4fffb282..5f2b15fd 100644 --- a/httpie/utils.py +++ b/httpie/utils.py @@ -214,14 +214,33 @@ def parse_content_type_header(header): return content_type, params_dict -def as_site(path: Path) -> Path: +def as_site(path: Path, **extra_vars) -> Path: site_packages_path = sysconfig.get_path( 'purelib', - vars={'base': str(path)} + vars={'base': str(path), **extra_vars} ) return Path(site_packages_path) +def get_site_paths(path: Path) -> Iterable[Path]: + from httpie.compat import ( + MIN_SUPPORTED_PY_VERSION, + MAX_SUPPORTED_PY_VERSION, + is_frozen + ) + + if is_frozen: + [major, min_minor] = MIN_SUPPORTED_PY_VERSION + [major, max_minor] = MAX_SUPPORTED_PY_VERSION + for minor in range(min_minor, max_minor + 1): + yield as_site( + path, + py_version_short=f'{major}.{minor}' + ) + else: + yield as_site(path) + + def split(iterable: Iterable[T], key: Callable[[T], bool]) -> Tuple[List[T], List[T]]: left, right = [], [] for item in iterable: diff --git a/setup.py b/setup.py index 439c2357..4929895a 100644 --- a/setup.py +++ b/setup.py @@ -30,6 +30,7 @@ dev_require = [ 'Jinja2' ] install_requires = [ + 'pip', 'charset_normalizer>=2.0.0', 'defusedxml>=0.6.0', 'requests[socks]>=2.22.0',