forked from extern/httpie-cli
Single binary executables (#1330)
* Single binary executables / DEB packages. * Attach single binary executables to the releases
This commit is contained in:
Batuhan Taskaya
GitHub
parent
278dfc487d
commit
dd2c9513f3
68
.github/workflows/release-linux-standalone.yml
vendored
Normal file
68
.github/workflows/release-linux-standalone.yml
vendored
Normal file
@ -0,0 +1,68 @@
|
||||
name: Release as Standalone Linux Package
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
branch:
|
||||
description: "The branch, tag or SHA to release from"
|
||||
required: true
|
||||
default: "master"
|
||||
|
||||
release:
|
||||
types: [released, prereleased]
|
||||
|
||||
|
||||
jobs:
|
||||
binary-build-and-release:
|
||||
name: Build and Release
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
with:
|
||||
ref: ${{ github.event.inputs.branch }}
|
||||
|
||||
- uses: actions/setup-python@v3
|
||||
with:
|
||||
python-version: 3.9
|
||||
|
||||
- name: Build Artifacts
|
||||
run: |
|
||||
cd extras/packaging/linux
|
||||
./get_release_artifacts.sh
|
||||
|
||||
- uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: http
|
||||
path: extras/packaging/linux/artifacts/dist/http
|
||||
|
||||
- uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: httpie.deb
|
||||
path: extras/packaging/linux/artifacts/dist/*.deb
|
||||
|
||||
- uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: httpie.rpm
|
||||
path: extras/packaging/linux/artifacts/dist/*.rpm
|
||||
|
||||
- name: Publish Debian Package
|
||||
if: github.event_name == 'release'
|
||||
uses: actions/upload-release-asset@v1.0.2
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
with:
|
||||
upload_url: ${{ github.event.release.upload_url }}
|
||||
asset_path: extras/packaging/linux/artifacts/dist/httpie-${{ github.event.release.tag_name }}.deb
|
||||
asset_name: httpie-${{ github.event.release.tag_name }}.deb
|
||||
asset_content_type: binary/octet-stream
|
||||
|
||||
- name: Publish Single Executable
|
||||
if: github.event_name == 'release'
|
||||
uses: actions/upload-release-asset@v1.0.2
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
with:
|
||||
upload_url: ${{ github.event.release.upload_url }}
|
||||
asset_path: extras/packaging/linux/artifacts/dist/http
|
||||
asset_name: http
|
||||
asset_content_type: binary/octet-stream
|
||||
4
.gitignore
vendored
4
.gitignore
vendored
@ -43,8 +43,8 @@ MANIFEST
|
||||
# PyInstaller
|
||||
# Usually these files are written by a python script from a template
|
||||
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
||||
*.manifest
|
||||
*.spec
|
||||
*.manifest
|
||||
|
||||
# Installer logs
|
||||
pip-log.txt
|
||||
@ -151,3 +151,5 @@ dmypy.json
|
||||
|
||||
# Windows Chocolatey
|
||||
*.nupkg
|
||||
|
||||
artifacts/
|
||||
|
||||
@ -11,6 +11,9 @@ all
|
||||
# Because we use HTML to hide them on the website.
|
||||
exclude_rule 'MD002'
|
||||
|
||||
# MD007 Allow unordered list indentation
|
||||
exclude_rule 'MD007'
|
||||
|
||||
# MD013 Line length
|
||||
exclude_rule 'MD013'
|
||||
|
||||
|
||||
@ -12,16 +12,18 @@ You are looking at the HTTPie packaging documentation, where you will find valua
|
||||
|
||||
The overall release process starts simple:
|
||||
|
||||
1. Do the [PyPI](https://pypi.org/project/httpie/) publication.
|
||||
2. Then, handle company-related tasks.
|
||||
3. Finally, follow OS-specific steps, described in documents below, to send patches downstream.
|
||||
1. Bump the version identifiers in the following places:
|
||||
- `httpie/__init__.py`
|
||||
- `docs/packaging/windows-chocolatey/httpie.nuspec`
|
||||
- `CHANGELOG.md`
|
||||
2. Commit your changes and make a PR against the `master`.
|
||||
3. Merge the PR, and tag the last commit with your version identifier.
|
||||
4. Make a GitHub release (by copying the text in `CHANGELOG.md`)
|
||||
5. Push that release to PyPI (dispatch the `Release PyPI` GitHub action).
|
||||
6. Once PyPI is ready, push the release to the Snap, Homebrew and Chocolatey with their respective actions.
|
||||
7. Go to the [`httpie/debian.httpie.io`](https://github.com/httpie/debian.httpie.io) repo and trigger the package index workflow.
|
||||
|
||||
## First, PyPI
|
||||
|
||||
Let's do the release on [PyPi](https://pypi.org/project/httpie/).
|
||||
That is done quite easily by manually triggering the [release workflow](https://github.com/httpie/httpie/actions/workflows/release.yml).
|
||||
|
||||
## Then, company-specific tasks
|
||||
## Company-specific tasks
|
||||
|
||||
- Blank the `master_and_released_docs_differ_after` value in [config.json](https://github.com/httpie/httpie/blob/master/docs/config.json).
|
||||
- Update the [contributors list](../contributors).
|
||||
@ -36,10 +38,9 @@ A more complete state of deployment can be found on [repology](https://repology.
|
||||
| -------------------------------------------: | -------------- |
|
||||
| [Arch Linux, and derived](linux-arch/) | trusted person |
|
||||
| [CentOS, RHEL, and derived](linux-centos/) | trusted person |
|
||||
| [Debian, Ubuntu, and derived](linux-debian/) | trusted person |
|
||||
| [Fedora](linux-fedora/) | trusted person |
|
||||
| :construction: [Homebrew, Linuxbrew](brew/) | **HTTPie** |
|
||||
| :construction: [MacPorts](mac-ports/) | **HTTPie** |
|
||||
| [Debian, Ubuntu, and derived](linux-debian/) | **HTTPie** |
|
||||
| [Homebrew, Linuxbrew](brew/) | **HTTPie** |
|
||||
| [Snapcraft](snapcraft/) | **HTTPie** |
|
||||
| [Windows — Chocolatey](windows-chocolatey/) | **HTTPie** |
|
||||
|
||||
|
||||
@ -13,21 +13,19 @@ We will discuss setting up the environment, installing development tools, instal
|
||||
|
||||
## Overall process
|
||||
|
||||
:construction: Work in progress.
|
||||
The brew deployment is completely automated, and only requires a trigger to [`Release on Homebrew`](https://github.com/httpie/httpie/actions/workflows/release-brew.yml) action
|
||||
from the release manager.
|
||||
|
||||
First, update the current Formula:
|
||||
If it is needed to be done manually, the following command can be used:
|
||||
|
||||
```bash
|
||||
make brew-deps
|
||||
# Copy-paste content into downstream/mac/brew/httpie.rb
|
||||
git add downstream/mac/brew/httpie.rb
|
||||
git commit -s -m 'Update brew formula to XXX'
|
||||
```console
|
||||
$ brew bump-formula-pr httpie --version={TARGET_VERSION}
|
||||
```
|
||||
|
||||
That [GitHub workflow](https://github.com/httpie/httpie/actions/workflows/test-package-mac-brew.yml) will test the formula when `downstream/mac/brew/httpie.rb` is changed in a pull request.
|
||||
|
||||
Then, open a pull request with those changes to the [downstream file](https://github.com/Homebrew/homebrew-core/blob/master/Formula/httpie.rb).
|
||||
which will bump the formala, and create a PR against the package index.
|
||||
|
||||
## Hacking
|
||||
|
||||
:construction: Work in progress.
|
||||
Make your changes, test the formula through the [`Test Brew Package`](https://github.com/httpie/httpie/actions/workflows/test-package-mac-brew.yml) action
|
||||
and then finally submit your patch to [`homebrew-core`](https://github.com/Homebrew/homebrew-core`)
|
||||
|
||||
|
||||
@ -1,81 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Generate Ruby code with URLs and file hashes for packages from PyPi
|
||||
(i.e., httpie itself as well as its dependencies) to be included
|
||||
in the Homebrew formula after a new release of HTTPie has been published
|
||||
on PyPi.
|
||||
|
||||
<https://github.com/Homebrew/homebrew-core/blob/master/Formula/httpie.rb>
|
||||
|
||||
"""
|
||||
import hashlib
|
||||
import requests
|
||||
|
||||
|
||||
VERSIONS = {
|
||||
# By default, we use the latest packages. But sometimes Requests has a maximum supported versions.
|
||||
# Take a look here before making a release: <https://github.com/psf/requests/blob/master/setup.py>
|
||||
'idna': '3.2',
|
||||
}
|
||||
|
||||
|
||||
# Note: Keep that list sorted.
|
||||
PACKAGES = [
|
||||
'certifi',
|
||||
'charset-normalizer',
|
||||
'defusedxml',
|
||||
'httpie',
|
||||
'idna',
|
||||
'Pygments',
|
||||
'PySocks',
|
||||
'requests',
|
||||
'requests-toolbelt',
|
||||
'urllib3',
|
||||
'multidict',
|
||||
]
|
||||
|
||||
|
||||
def get_package_meta(package_name):
|
||||
api_url = f'https://pypi.org/pypi/{package_name}/json'
|
||||
resp = requests.get(api_url).json()
|
||||
hasher = hashlib.sha256()
|
||||
version = VERSIONS.get(package_name)
|
||||
if package_name not in VERSIONS:
|
||||
# Latest version
|
||||
release_bundle = resp['urls']
|
||||
else:
|
||||
release_bundle = resp['releases'][version]
|
||||
|
||||
for release in release_bundle:
|
||||
download_url = release['url']
|
||||
if download_url.endswith('.tar.gz'):
|
||||
hasher.update(requests.get(download_url).content)
|
||||
return {
|
||||
'name': package_name,
|
||||
'url': download_url,
|
||||
'sha256': hasher.hexdigest(),
|
||||
}
|
||||
else:
|
||||
raise RuntimeError(f'{package_name}: download not found: {resp}')
|
||||
|
||||
|
||||
def main():
|
||||
package_meta_map = {
|
||||
package_name: get_package_meta(package_name)
|
||||
for package_name in PACKAGES
|
||||
}
|
||||
httpie_meta = package_meta_map.pop('httpie')
|
||||
print()
|
||||
print(' url "{url}"'.format(url=httpie_meta['url']))
|
||||
print(' sha256 "{sha256}"'.format(sha256=httpie_meta['sha256']))
|
||||
print()
|
||||
for dep_meta in package_meta_map.values():
|
||||
print(' resource "{name}" do'.format(name=dep_meta['name']))
|
||||
print(' url "{url}"'.format(url=dep_meta['url']))
|
||||
print(' sha256 "{sha256}"'.format(sha256=dep_meta['sha256']))
|
||||
print(' end')
|
||||
print('')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@ -3,18 +3,18 @@ class Httpie < Formula
|
||||
|
||||
desc "User-friendly cURL replacement (command-line HTTP client)"
|
||||
homepage "https://httpie.io/"
|
||||
url "https://files.pythonhosted.org/packages/7b/f9/13070f19226b7db3641fb787df36bb715063abe1b8ca03fbaeca0f465d27/httpie-3.0.1.tar.gz"
|
||||
sha256 "0e9bc93ebdcdd2d32ec24b8fa46cf7e4fde9eec7a6bd0c5d0ef224f25d7466b2"
|
||||
url "https://files.pythonhosted.org/packages/32/85/bb095699be20cc98731261cb80884e9458178f8fef2a38273530ce77c0a5/httpie-3.1.0.tar.gz"
|
||||
sha256 "2e4a2040b84a912e65c01fb34f7aafe88cad2a3af2da8c685ca65080f376feda"
|
||||
license "BSD-3-Clause"
|
||||
head "https://github.com/httpie/httpie.git", branch: "master"
|
||||
|
||||
bottle do
|
||||
sha256 cellar: :any_skip_relocation, arm64_monterey: "9d285fcfb55ce8ed787d1b01966d51e6e07f7e77c44a204695a2d6eee9c8698d"
|
||||
sha256 cellar: :any_skip_relocation, arm64_big_sur: "743a282b475e87a4eaf11e545f761aef1b8e4bfe49eaee47251d7629a35a8ced"
|
||||
sha256 cellar: :any_skip_relocation, monterey: "5d63ea4f47b2028b2ba68abe12a4176934193e058edd869270221b41cc946c76"
|
||||
sha256 cellar: :any_skip_relocation, big_sur: "5a53221a680a35d1aa00cbadde279dbe4f562d22ed207c15bd4221cb8c3180f1"
|
||||
sha256 cellar: :any_skip_relocation, catalina: "5feadb6d76f55d6f9681682e221008c282dccf0e46ae22a959b4bad2efde204a"
|
||||
sha256 cellar: :any_skip_relocation, x86_64_linux: "d530ddbec49588b0d481f156d35f7e5bb7d3b6427d203f04750e55cd3eecc303"
|
||||
sha256 cellar: :any_skip_relocation, arm64_monterey: "9bb6e8c1ef5ba8b019ddedd7e908dd2174da695351aa9a238dfb28b0f57ef005"
|
||||
sha256 cellar: :any_skip_relocation, arm64_big_sur: "47ffccd3241155d863e1b4f6259d538a34d42a0cdeed8152bda257ee607b51be"
|
||||
sha256 cellar: :any_skip_relocation, monterey: "dc4a04cb05a9cd1bfa6a632a0e4a21975905954af54ece41f9050c52474267be"
|
||||
sha256 cellar: :any_skip_relocation, big_sur: "ae469e37864e967e0fd99fba15a78e719dcb351b462f98f3843c78ed1473df6d"
|
||||
sha256 cellar: :any_skip_relocation, catalina: "291a3eaecb2a2cc845c1652686a9a14b21053d7e3a7d0115245b2150ca2e199e"
|
||||
sha256 cellar: :any_skip_relocation, x86_64_linux: "710836e27c44c8e3ad181d668f4a9f78c4cb4c355d7b148a397599a7cd42713d"
|
||||
end
|
||||
|
||||
depends_on "python@3.10"
|
||||
@ -25,8 +25,8 @@ class Httpie < Formula
|
||||
end
|
||||
|
||||
resource "charset-normalizer" do
|
||||
url "https://files.pythonhosted.org/packages/48/44/76b179e0d1afe6e6a91fd5661c284f60238987f3b42b676d141d01cd5b97/charset-normalizer-2.0.10.tar.gz"
|
||||
sha256 "876d180e9d7432c5d1dfd4c5d26b72f099d503e8fcc0feb7532c9289be60fcbd"
|
||||
url "https://files.pythonhosted.org/packages/56/31/7bcaf657fafb3c6db8c787a865434290b726653c912085fbd371e9b92e1c/charset-normalizer-2.0.12.tar.gz"
|
||||
sha256 "2857e29ff0d34db842cd7ca3230549d1a697f96ee6d3fb071cfa6c7393832597"
|
||||
end
|
||||
|
||||
resource "defusedxml" do
|
||||
@ -40,8 +40,8 @@ class Httpie < Formula
|
||||
end
|
||||
|
||||
resource "multidict" do
|
||||
url "https://files.pythonhosted.org/packages/8e/7c/e12a69795b7b7d5071614af2c691c97fbf16a2a513c66ec52dd7d0a115bb/multidict-5.2.0.tar.gz"
|
||||
sha256 "0dd1c93edb444b33ba2274b66f63def8a327d607c6c790772f448a53b6ea59ce"
|
||||
url "https://files.pythonhosted.org/packages/fa/a7/71c253cdb8a1528802bac7503bf82fe674367e4055b09c28846fdfa4ab90/multidict-6.0.2.tar.gz"
|
||||
sha256 "5ff3bd75f38e4c43f1f470f2df7a4d430b821c4ce22be384e1459cb57d6bb013"
|
||||
end
|
||||
|
||||
resource "Pygments" do
|
||||
|
||||
6
docs/packaging/brew/update.sh
Executable file
6
docs/packaging/brew/update.sh
Executable file
@ -0,0 +1,6 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -xe
|
||||
|
||||
rm -f httpie.rb
|
||||
http --download https://raw.githubusercontent.com/Homebrew/homebrew-core/master/Formula/httpie.rb
|
||||
@ -11,19 +11,16 @@ Welcome to the documentation about **packaging HTTPie for Debian GNU/Linux**.
|
||||
This document contains technical details, where we describe how to create a patch for the latest HTTPie version for Debian GNU/Linux. They apply to Ubuntu as well, and any Debian-derived distributions like MX Linux, Linux Mint, deepin, Pop!_OS, KDE neon, Zorin OS, elementary OS, Kubuntu, Devuan, Linux Lite, Peppermint OS, Lubuntu, antiX, Xubuntu, etc.
|
||||
We will discuss setting up the environment, installing development tools, installing and testing changes before submitting a patch downstream.
|
||||
|
||||
The current maintainer is Bartosz Fenski.
|
||||
We create the standalone binaries (see this [for more details](../../../extras/packaging/linux/)) and package them with
|
||||
[FPM](https://github.com/jordansissel/fpm)'s `dir` mode. The core `http`/`https` commands don't have any dependencies, but the `httpie`
|
||||
command (due to the underlying `httpie cli plugins` interface) explicitly depends to the system Python (through `python3`/`python3-pip`).
|
||||
|
||||
## Overall process
|
||||
|
||||
Open a new bug on the Debian Bug Tracking System by sending an email:
|
||||
The [`Release as Standalone Linux Binary`](https://github.com/httpie/httpie/actions/workflows/release-linux-standalone.yml) will be automatically
|
||||
triggered when a new release is created, and it will submit the `.deb` package as a release asset.
|
||||
|
||||
- To: `Debian Bug Tracking System <submit@bugs.debian.org>`
|
||||
- Subject: `httpie: Version XXX available`
|
||||
- Message template (examples [1](https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=993937), and [2](https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=996479)):
|
||||
|
||||
```email
|
||||
Package: httpie
|
||||
Severity: normal
|
||||
|
||||
<MESSAGE>
|
||||
```
|
||||
For making that asset available for all debian users, the release manager needs to go to the [`httpie/debian.httpie.io`](https://github.com/httpie/debian.httpie.io) repo
|
||||
and trigger the [`Update Index`](https://github.com/httpie/debian.httpie.io/actions/workflows/update-index.yml) action. It will automatically
|
||||
scrape all new debian packages from the release assets, properly update the indexes and create a new PR ([an example](https://github.com/httpie/debian.httpie.io/pull/1))
|
||||
which then will become active when merged.
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
Name: httpie
|
||||
Version: 3.0.2
|
||||
Version: 3.1.0
|
||||
Release: 1%{?dist}
|
||||
Summary: A Curl-like tool for humans
|
||||
|
||||
@ -78,6 +78,10 @@ help2man %{buildroot}%{_bindir}/httpie > %{buildroot}%{_mandir}/man1/httpie.1
|
||||
|
||||
|
||||
%changelog
|
||||
* Tue Mar 08 2022 Miro Hrončok <mhroncok@redhat.com> - 3.1.0-1
|
||||
- Update to 3.1.0
|
||||
- Fixes: rhbz#2061597
|
||||
|
||||
* Mon Jan 24 2022 Miro Hrončok <mhroncok@redhat.com> - 3.0.2-1
|
||||
- Update to 3.0.2
|
||||
- Fixes: rhbz#2044572
|
||||
|
||||
6
docs/packaging/linux-fedora/update.sh
Executable file
6
docs/packaging/linux-fedora/update.sh
Executable file
@ -0,0 +1,6 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -xe
|
||||
|
||||
rm -f httpie.spec.txt
|
||||
https --download src.fedoraproject.org/rpms/httpie/raw/rawhide/f/httpie.spec -o httpie.spec.txt
|
||||
@ -13,7 +13,16 @@ We will discuss setting up the environment, installing development tools, instal
|
||||
|
||||
## Overall process
|
||||
|
||||
Trigger a new [build](https://snapcraft.io/httpie/builds), then [promote it](https://snapcraft.io/httpie/releases). If more management is needed: [revisions supervision](https://dashboard.snapcraft.io/snaps/httpie/revisions/).
|
||||
Trigger the [`Release on Snap`](https://github.com/httpie/httpie/actions/workflows/release-snap.yml) action, which will
|
||||
create a snap package for HTTPie and then push it to Snap Store in the following channels:
|
||||
|
||||
- Edge
|
||||
- Beta
|
||||
- Candidate
|
||||
- Stable
|
||||
|
||||
If a push to any of them fail, all the release tasks for the following channels will be cancelled so that the
|
||||
release manager can look into the underlying cause.
|
||||
|
||||
## Hacking
|
||||
|
||||
|
||||
@ -13,13 +13,18 @@ We will discuss setting up the environment, installing development tools, instal
|
||||
|
||||
## Overall process
|
||||
|
||||
After having successfully [built and tested](#hacking) the package, push it:
|
||||
After having successfully [built and tested](#hacking) the package, either trigger the
|
||||
[`Release on Chocolatey`](https://github.com/httpie/httpie/actions/workflows/release-choco.yml) action
|
||||
to push it to the `Chocolatey` store or use the CLI:
|
||||
|
||||
```bash
|
||||
# Replace 2.5.0 with the correct version
|
||||
choco push httpie.2.5.0.nupkg -s https://push.chocolatey.org/ --api-key=API_KEY
|
||||
```
|
||||
|
||||
Be aware that it might take multiple days until the release is approved, sine it goes through multiple
|
||||
sets of reviews (some of them are done manually).
|
||||
|
||||
## Hacking
|
||||
|
||||
```bash
|
||||
|
||||
32
extras/packaging/linux/Dockerfile
Normal file
32
extras/packaging/linux/Dockerfile
Normal file
@ -0,0 +1,32 @@
|
||||
# Use the oldest (but still supported) Ubuntu as the base for PyInstaller
|
||||
# packages. This will prevent stuff like glibc from conflicting.
|
||||
FROM ubuntu:18.04
|
||||
|
||||
RUN apt-get update
|
||||
RUN apt-get install -y software-properties-common binutils
|
||||
RUN apt-get install -y ruby-dev
|
||||
RUN gem install fpm
|
||||
|
||||
# Use deadsnakes for the latest Pythons (e.g 3.9)
|
||||
RUN add-apt-repository ppa:deadsnakes/ppa
|
||||
RUN apt-get update && apt-get install -y python3.9 python3.9-dev python3.9-venv
|
||||
|
||||
# Install rpm as well, since we are going to build fedora dists too
|
||||
RUN apt-get install -y rpm
|
||||
|
||||
ADD . /app
|
||||
WORKDIR /app/extras/packaging/linux
|
||||
|
||||
ENV VIRTUAL_ENV=/opt/venv
|
||||
RUN python3.9 -m venv $VIRTUAL_ENV
|
||||
ENV PATH="$VIRTUAL_ENV/bin:$PATH"
|
||||
|
||||
# Ensure that pip is renewed, otherwise we would be using distro-provided pip
|
||||
# which strips vendored packages and doesn't work with PyInstaller.
|
||||
RUN python -m pip install /app
|
||||
RUN python -m pip install pyinstaller wheel
|
||||
RUN python -m pip install --force-reinstall --upgrade pip
|
||||
|
||||
RUN python build.py
|
||||
|
||||
ENTRYPOINT ["mv", "/app/extras/packaging/linux/dist/", "/artifacts"]
|
||||
52
extras/packaging/linux/README.md
Normal file
52
extras/packaging/linux/README.md
Normal file
@ -0,0 +1,52 @@
|
||||
# Standalone Linux Packages
|
||||
|
||||
|
||||
|
||||
This directory contains the build scripts for creating:
|
||||
|
||||
- A self-contained binary executable for the HTTPie itself
|
||||
- `httpie.deb` and `httpie.rpm` packages for Debian and Fedora.
|
||||
|
||||
The process of constructing them are fully automated, and can be easily done through the [`Release as Standalone Linux Package`](https://github.com/httpie/httpie/actions/workflows/release-linux-standalone.yml)
|
||||
action. Once it finishes, the release artifacts will be attached in the summary page of the triggered run.
|
||||
|
||||
|
||||
## Hacking
|
||||
|
||||
The main entry point for the package builder is the [`build.py`](https://github.com/httpie/httpie/blob/master/extras/packaging/linux/build.py). It
|
||||
contains 2 major methods:
|
||||
|
||||
- `build_binaries`, for the self-contained executables
|
||||
- `build_packages`, for the OS-specific packages (which wrap the binaries)
|
||||
|
||||
### `build_binaries`
|
||||
|
||||
We use [PyInstaller](https://pyinstaller.readthedocs.io/en/stable/) for the binaries. Normally pyinstaller offers two different modes:
|
||||
|
||||
- Single directory (harder to distribute, low redundancy. Library files are shared accross different executables)
|
||||
- Single binary (easier to distribute, higher redundancy. Same libraries are statically linked to different executables, so higher total size)
|
||||
|
||||
Since our binary size (in total 20 MiBs) is not that big, we have decided to choose the single binary mode for the sake of easier distribution.
|
||||
|
||||
We also disable `UPX`, which is a runtime decompression method since it adds some startup cost.
|
||||
|
||||
### `build_packages`
|
||||
|
||||
We build our OS-specific packages with [FPM](https://github.com/jordansissel/fpm) which offers a really nice abstraction. We use the `dir` mode,
|
||||
and package `http`, `https` and `httpie` commands. More can be added to the `files` option.
|
||||
|
||||
Since the `httpie` depends on having a pip executable, we explicitly depend on the system Python even though the core does not use it.
|
||||
|
||||
### Docker Image
|
||||
|
||||
This directory also contains a [docker image](https://github.com/httpie/httpie/blob/master/extras/packaging/linux/Dockerfile) which helps
|
||||
building our standalone binaries in an isolated environment with the lowest possible library versions. This is important, since even though
|
||||
the executables are standalone they still depend on some main system C libraries (like `glibc`) so we need to create our executables inside
|
||||
an environment with a very old (but not deprecated) glibc version. It makes us soundproof for all active Ubuntu/Debian versions.
|
||||
|
||||
It also contains the Python version we package our HTTPie with, so it is the place if you need to change it.
|
||||
|
||||
### `./get_release_artifacts.sh`
|
||||
|
||||
If you make a change in the `build.py`, run the following script to test it out. It will return multiple files under `artifacts/dist` which
|
||||
then you can test out and ensure their quality (it is also the script that we use in our automation).
|
||||
100
extras/packaging/linux/build.py
Normal file
100
extras/packaging/linux/build.py
Normal file
@ -0,0 +1,100 @@
|
||||
import stat
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from typing import Iterator, Tuple
|
||||
|
||||
BUILD_DIR = Path(__file__).parent
|
||||
HTTPIE_DIR = BUILD_DIR.parent.parent.parent
|
||||
|
||||
SCRIPT_DIR = BUILD_DIR / Path('scripts')
|
||||
HOOKS_DIR = SCRIPT_DIR / 'hooks'
|
||||
|
||||
DIST_DIR = BUILD_DIR / 'dist'
|
||||
|
||||
TARGET_SCRIPTS = {
|
||||
SCRIPT_DIR / 'http_cli.py': [],
|
||||
SCRIPT_DIR / 'httpie_cli.py': ['--hidden-import=pip'],
|
||||
}
|
||||
|
||||
|
||||
def build_binaries() -> Iterator[Tuple[str, Path]]:
|
||||
for target_script, extra_args in TARGET_SCRIPTS.items():
|
||||
subprocess.check_call(
|
||||
[
|
||||
'pyinstaller',
|
||||
'--onefile',
|
||||
'--noupx',
|
||||
'-p',
|
||||
HTTPIE_DIR,
|
||||
'--additional-hooks-dir',
|
||||
HOOKS_DIR,
|
||||
*extra_args,
|
||||
target_script,
|
||||
]
|
||||
)
|
||||
|
||||
for executable_path in DIST_DIR.iterdir():
|
||||
if executable_path.suffix:
|
||||
continue
|
||||
stat_r = executable_path.stat()
|
||||
executable_path.chmod(stat_r.st_mode | stat.S_IEXEC)
|
||||
yield executable_path.stem, executable_path
|
||||
|
||||
|
||||
def build_packages(http_binary: Path, httpie_binary: Path) -> None:
|
||||
import httpie
|
||||
|
||||
# Mapping of src_file -> dst_file
|
||||
files = [
|
||||
(http_binary, '/usr/bin/http'),
|
||||
(http_binary, '/usr/bin/https'),
|
||||
(httpie_binary, '/usr/bin/httpie'),
|
||||
]
|
||||
# A list of additional dependencies
|
||||
deps = [
|
||||
'python3 >= 3.7',
|
||||
'python3-pip'
|
||||
]
|
||||
|
||||
processed_deps = [
|
||||
f'--depends={dep}'
|
||||
for dep in deps
|
||||
]
|
||||
processed_files = [
|
||||
'='.join([str(src.resolve()), dst]) for src, dst in files
|
||||
]
|
||||
for target in ['deb', 'rpm']:
|
||||
subprocess.check_call(
|
||||
[
|
||||
'fpm',
|
||||
'--force',
|
||||
'-s',
|
||||
'dir',
|
||||
'-t',
|
||||
target,
|
||||
'--name',
|
||||
'httpie',
|
||||
'--version',
|
||||
httpie.__version__,
|
||||
'--description',
|
||||
httpie.__doc__.strip(),
|
||||
'--license',
|
||||
httpie.__licence__,
|
||||
*processed_deps,
|
||||
*processed_files,
|
||||
],
|
||||
cwd=DIST_DIR,
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
binaries = dict(build_binaries())
|
||||
build_packages(binaries['http_cli'], binaries['httpie_cli'])
|
||||
|
||||
# Rename http_cli/httpie_cli to http/httpie
|
||||
binaries['http_cli'].rename('http')
|
||||
binaries['httpie_cli'].rename('httpie')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
22
extras/packaging/linux/get_release_artifacts.sh
Executable file
22
extras/packaging/linux/get_release_artifacts.sh
Executable file
@ -0,0 +1,22 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -xe
|
||||
|
||||
REPO_ROOT=../../../
|
||||
ARTIFACTS_DIR=$(pwd)/artifacts
|
||||
|
||||
# Reset the ARTIFACTS_DIR.
|
||||
rm -rf $ARTIFACTS_DIR
|
||||
mkdir -p $ARTIFACTS_DIR
|
||||
|
||||
# Operate on the repository root to have the proper
|
||||
# docker context.
|
||||
pushd $REPO_ROOT
|
||||
|
||||
# Build the PyInstaller image
|
||||
docker build -t pyinstaller-httpie -f extras/packaging/linux/Dockerfile .
|
||||
|
||||
# Copy the artifacts to the designated directory.
|
||||
docker run --rm -i -v $ARTIFACTS_DIR:/artifacts pyinstaller-httpie:latest
|
||||
|
||||
popd
|
||||
14
extras/packaging/linux/scripts/hooks/hook-pip.py
Normal file
14
extras/packaging/linux/scripts/hooks/hook-pip.py
Normal file
@ -0,0 +1,14 @@
|
||||
from pathlib import Path
|
||||
from PyInstaller.utils.hooks import collect_all
|
||||
|
||||
def hook(hook_api):
|
||||
for pkg in [
|
||||
'pip',
|
||||
'setuptools',
|
||||
'distutils',
|
||||
'pkg_resources'
|
||||
]:
|
||||
datas, binaries, hiddenimports = collect_all(pkg)
|
||||
hook_api.add_datas(datas)
|
||||
hook_api.add_binaries(binaries)
|
||||
hook_api.add_imports(*hiddenimports)
|
||||
5
extras/packaging/linux/scripts/http_cli.py
Normal file
5
extras/packaging/linux/scripts/http_cli.py
Normal file
@ -0,0 +1,5 @@
|
||||
from httpie.__main__ import main
|
||||
|
||||
if __name__ == '__main__':
|
||||
import sys
|
||||
sys.exit(main())
|
||||
5
extras/packaging/linux/scripts/httpie_cli.py
Normal file
5
extras/packaging/linux/scripts/httpie_cli.py
Normal file
@ -0,0 +1,5 @@
|
||||
from httpie.manager.__main__ import main
|
||||
|
||||
if __name__ == '__main__':
|
||||
import sys
|
||||
sys.exit(main())
|
||||
@ -12,7 +12,10 @@ cookiejar.DefaultCookiePolicy = HTTPieCookiePolicy
|
||||
|
||||
|
||||
is_windows = 'win32' in str(sys.platform).lower()
|
||||
is_frozen = getattr(sys, 'frozen', False)
|
||||
|
||||
MIN_SUPPORTED_PY_VERSION = (3, 7)
|
||||
MAX_SUPPORTED_PY_VERSION = (3, 11)
|
||||
|
||||
try:
|
||||
from functools import cached_property
|
||||
|
||||
69
httpie/manager/compat.py
Normal file
69
httpie/manager/compat.py
Normal file
@ -0,0 +1,69 @@
|
||||
import sys
|
||||
import shutil
|
||||
import subprocess
|
||||
|
||||
from contextlib import suppress
|
||||
from typing import List, Optional
|
||||
from httpie.compat import is_frozen
|
||||
|
||||
|
||||
class PipError(Exception):
|
||||
"""An exception that occurs when pip exits with an error status code."""
|
||||
|
||||
def __init__(self, stdout, stderr):
|
||||
self.stdout = stdout
|
||||
self.stderr = stderr
|
||||
|
||||
|
||||
def _discover_system_pip() -> List[str]:
|
||||
# When we are running inside of a frozen binary, we need the system
|
||||
# pip to install plugins since there is no way for us to execute any
|
||||
# code outside of the HTTPie.
|
||||
#
|
||||
# We explicitly depend on system pip, so the SystemError should not
|
||||
# be executed (except for broken installations).
|
||||
def _check_pip_version(pip_location: Optional[str]) -> bool:
|
||||
if not pip_location:
|
||||
return False
|
||||
|
||||
with suppress(subprocess.CalledProcessError):
|
||||
stdout = subprocess.check_output([pip_location, "--version"], text=True)
|
||||
return "python 3" in stdout
|
||||
|
||||
targets = [
|
||||
"pip",
|
||||
"pip3"
|
||||
]
|
||||
for target in targets:
|
||||
pip_location = shutil.which(target)
|
||||
if _check_pip_version(pip_location):
|
||||
return pip_location
|
||||
|
||||
raise SystemError("Couldn't find 'pip' executable. Please ensure that pip in your system is available.")
|
||||
|
||||
|
||||
def _run_pip_subprocess(pip_executable: List[str], args: List[str]) -> bytes:
|
||||
import subprocess
|
||||
|
||||
cmd = [*pip_executable, *args]
|
||||
try:
|
||||
process = subprocess.run(
|
||||
cmd,
|
||||
check=True,
|
||||
shell=False,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE
|
||||
)
|
||||
except subprocess.CalledProcessError as error:
|
||||
raise PipError(error.stdout, error.stderr) from error
|
||||
else:
|
||||
return process.stdout
|
||||
|
||||
|
||||
def run_pip(args: List[str]) -> bytes:
|
||||
if is_frozen:
|
||||
pip_executable = [_discover_system_pip()]
|
||||
else:
|
||||
pip_executable = [sys.executable, '-m', 'pip']
|
||||
|
||||
return _run_pip_subprocess(pip_executable, args)
|
||||
@ -1,20 +1,19 @@
|
||||
import argparse
|
||||
import os
|
||||
import textwrap
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import textwrap
|
||||
from collections import defaultdict
|
||||
from contextlib import suppress
|
||||
from pathlib import Path
|
||||
from typing import List, Optional, Tuple
|
||||
|
||||
from httpie.manager.compat import PipError, run_pip
|
||||
from httpie.manager.cli import parser, missing_subcommand
|
||||
from httpie.compat import get_dist_name, importlib_metadata
|
||||
from httpie.context import Environment
|
||||
from httpie.manager.cli import missing_subcommand, parser
|
||||
from httpie.status import ExitStatus
|
||||
from httpie.utils import as_site
|
||||
from httpie.utils import get_site_paths
|
||||
|
||||
PEP_503 = re.compile(r"[-_.]+")
|
||||
|
||||
@ -58,46 +57,37 @@ class PluginInstaller:
|
||||
self.env.stderr.write(message + '\n')
|
||||
return ExitStatus.ERROR
|
||||
|
||||
def pip(self, *args, **kwargs) -> subprocess.CompletedProcess:
|
||||
options = {
|
||||
'check': True,
|
||||
'shell': False,
|
||||
'stdout': self.env.stdout,
|
||||
'stderr': subprocess.PIPE,
|
||||
}
|
||||
options.update(kwargs)
|
||||
|
||||
cmd = [sys.executable, '-m', 'pip', *args]
|
||||
return subprocess.run(
|
||||
cmd,
|
||||
**options
|
||||
)
|
||||
|
||||
def _install(self, targets: List[str], mode='install', **process_options) -> Tuple[
|
||||
Optional[bytes], ExitStatus
|
||||
def _install(self, targets: List[str], mode='install') -> Tuple[
|
||||
bytes, ExitStatus
|
||||
]:
|
||||
pip_args = [
|
||||
'install',
|
||||
'--prefer-binary',
|
||||
f'--prefix={self.dir}',
|
||||
'--no-warn-script-location',
|
||||
]
|
||||
if mode == 'upgrade':
|
||||
pip_args.append('--upgrade')
|
||||
pip_args.extend(targets)
|
||||
|
||||
try:
|
||||
process = self.pip(
|
||||
*pip_args,
|
||||
*targets,
|
||||
**process_options,
|
||||
)
|
||||
except subprocess.CalledProcessError as error:
|
||||
stdout = run_pip(pip_args)
|
||||
except PipError as pip_error:
|
||||
error = pip_error
|
||||
stdout = pip_error.stdout
|
||||
else:
|
||||
error = None
|
||||
|
||||
self.env.stdout.write(stdout.decode())
|
||||
|
||||
if error:
|
||||
reason = None
|
||||
if error.stderr:
|
||||
stderr = error.stderr.decode()
|
||||
|
||||
if self.debug:
|
||||
self.env.stderr.write('Command failed: ')
|
||||
self.env.stderr.write(' '.join(error.cmd) + '\n')
|
||||
self.env.stderr.write('pip ' + ' '.join(pip_args) + '\n')
|
||||
self.env.stderr.write(textwrap.indent(' ', stderr))
|
||||
|
||||
last_line = stderr.strip().splitlines()[-1]
|
||||
@ -108,7 +98,6 @@ class PluginInstaller:
|
||||
stdout = error.stdout
|
||||
exit_status = self.fail(mode, ', '.join(targets), reason)
|
||||
else:
|
||||
stdout = process.stdout
|
||||
exit_status = ExitStatus.SUCCESS
|
||||
|
||||
return stdout, exit_status
|
||||
@ -124,7 +113,8 @@ class PluginInstaller:
|
||||
# existing metadata for old versions manually.
|
||||
# [0]: https://github.com/pypa/pip/issues/10727
|
||||
result_deps = defaultdict(list)
|
||||
for child in as_site(self.dir).iterdir():
|
||||
for site_dir in get_site_paths(self.dir):
|
||||
for child in site_dir.iterdir():
|
||||
if child.suffix in {'.dist-info', '.egg-info'}:
|
||||
name, _, version = child.stem.rpartition('-')
|
||||
result_deps[name].append((version, child))
|
||||
@ -145,15 +135,12 @@ class PluginInstaller:
|
||||
|
||||
raw_stdout, exit_status = self._install(
|
||||
targets,
|
||||
mode='upgrade',
|
||||
stdout=subprocess.PIPE
|
||||
mode='upgrade'
|
||||
)
|
||||
if not raw_stdout:
|
||||
return exit_status
|
||||
|
||||
stdout = raw_stdout.decode()
|
||||
self.env.stdout.write(stdout)
|
||||
|
||||
installation_line = stdout.splitlines()[-1]
|
||||
if installation_line.startswith('Successfully installed'):
|
||||
self._clear_metadata(installation_line.split()[2:])
|
||||
|
||||
@ -4,13 +4,13 @@ import warnings
|
||||
|
||||
from itertools import groupby
|
||||
from operator import attrgetter
|
||||
from typing import Dict, List, Type, Iterator, Optional, ContextManager
|
||||
from typing import Dict, List, Type, Iterator, Iterable, Optional, ContextManager
|
||||
from pathlib import Path
|
||||
from contextlib import contextmanager, nullcontext
|
||||
|
||||
from ..compat import importlib_metadata, find_entry_points, get_dist_name
|
||||
|
||||
from ..utils import repr_dict, as_site
|
||||
from ..utils import repr_dict, get_site_paths
|
||||
from . import AuthPlugin, ConverterPlugin, FormatterPlugin, TransportPlugin
|
||||
from .base import BasePlugin
|
||||
|
||||
@ -25,20 +25,24 @@ ENTRY_POINT_NAMES = list(ENTRY_POINT_CLASSES.keys())
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _load_directory(plugins_dir: Path) -> Iterator[None]:
|
||||
plugins_path = os.fspath(plugins_dir)
|
||||
sys.path.insert(0, plugins_path)
|
||||
def _load_directories(site_dirs: Iterable[Path]) -> Iterator[None]:
|
||||
plugin_dirs = [
|
||||
os.fspath(site_dir)
|
||||
for site_dir in site_dirs
|
||||
]
|
||||
sys.path.extend(plugin_dirs)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
sys.path.remove(plugins_path)
|
||||
for plugin_dir in plugin_dirs:
|
||||
sys.path.remove(plugin_dir)
|
||||
|
||||
|
||||
def enable_plugins(plugins_dir: Optional[Path]) -> ContextManager[None]:
|
||||
if plugins_dir is None:
|
||||
return nullcontext()
|
||||
else:
|
||||
return _load_directory(as_site(plugins_dir))
|
||||
return _load_directories(get_site_paths(plugins_dir))
|
||||
|
||||
|
||||
class PluginManager(list):
|
||||
|
||||
@ -214,14 +214,33 @@ def parse_content_type_header(header):
|
||||
return content_type, params_dict
|
||||
|
||||
|
||||
def as_site(path: Path) -> Path:
|
||||
def as_site(path: Path, **extra_vars) -> Path:
|
||||
site_packages_path = sysconfig.get_path(
|
||||
'purelib',
|
||||
vars={'base': str(path)}
|
||||
vars={'base': str(path), **extra_vars}
|
||||
)
|
||||
return Path(site_packages_path)
|
||||
|
||||
|
||||
def get_site_paths(path: Path) -> Iterable[Path]:
|
||||
from httpie.compat import (
|
||||
MIN_SUPPORTED_PY_VERSION,
|
||||
MAX_SUPPORTED_PY_VERSION,
|
||||
is_frozen
|
||||
)
|
||||
|
||||
if is_frozen:
|
||||
[major, min_minor] = MIN_SUPPORTED_PY_VERSION
|
||||
[major, max_minor] = MAX_SUPPORTED_PY_VERSION
|
||||
for minor in range(min_minor, max_minor + 1):
|
||||
yield as_site(
|
||||
path,
|
||||
py_version_short=f'{major}.{minor}'
|
||||
)
|
||||
else:
|
||||
yield as_site(path)
|
||||
|
||||
|
||||
def split(iterable: Iterable[T], key: Callable[[T], bool]) -> Tuple[List[T], List[T]]:
|
||||
left, right = [], []
|
||||
for item in iterable:
|
||||
|
||||
Loading…
Reference in New Issue
Block a user