Merge branch 'feature/KASM-2301-upload-debug-packages' into 'master'

Resolve KASM-2301 "Feature/ upload debug packages"

Closes KASM-2301

See merge request kasm-technologies/internal/KasmVNC!26
This commit is contained in:
Matthew McClaskey 2022-02-11 14:57:25 +00:00
commit 8bf5ae9ef7
8 changed files with 129 additions and 15 deletions

View File

@ -6,6 +6,27 @@ is_kasmvnc() {
echo "$package" | grep -q 'kasmvncserver_'
}
detect_deb_package_arch() {
local deb_package="$1"
echo "$deb_package" | sed -e 's/.\+_\([^.]\+\)\.\(d\?\)deb/\1/'
}
find_deb_package() {
local dbgsym_package="$1"
echo "$dbgsym_package" | sed -e 's/-dbgsym//; s/ddeb/deb/'
}
fetch_xvnc_md5sum() {
local deb_package="$1"
deb_package=$(realpath "$deb_package")
local tmpdir=$(mktemp -d)
cd "$tmpdir"
dpkg-deb -e "$deb_package"
cat DEBIAN/md5sums | grep bin/Xkasmvnc | cut -d' ' -f 1
}
function prepare_upload_filename() {
local package="$1";
@ -34,19 +55,18 @@ function prepare_upload_filename() {
function upload_to_s3() {
local package="$1";
local upload_filename="$2";
local s3_bucket="$3";
# Transfer to S3
python3 amazon-s3-bitbucket-pipelines-python/s3_upload.py "${S3_BUCKET}" "$package" "${S3_BUILD_DIRECTORY}/${upload_filename}";
python3 amazon-s3-bitbucket-pipelines-python/s3_upload.py "${s3_bucket}" "$package" "${upload_filename}";
# Use the Gitlab API to tell Gitlab where the artifact was stored
export S3_URL="https://${S3_BUCKET}.s3.amazonaws.com/${S3_BUILD_DIRECTORY}/${upload_filename}";
export BUILD_STATUS="{\"key\":\"doc\", \"state\":\"SUCCESSFUL\", \"name\":\"${upload_filename}\", \"url\":\"${S3_URL}\"}";
curl --request POST --header "PRIVATE-TOKEN:${GITLAB_API_TOKEN}" "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/statuses/${CI_COMMIT_SHA}?state=success&name=build-url&target_url=${S3_URL}";
export S3_URL="https://${s3_bucket}.s3.amazonaws.com/${upload_filename}";
};
function prepare_to_run_scripts_and_s3_uploads() {
export DEBIAN_FRONTEND=noninteractive;
apt-get update;
apt-get install -y ruby2.7 git;
apt-get install -y ruby2.7 git wget;
apt-get install -y python3 python3-pip python3-boto3 curl pkg-config libxmlsec1-dev;
git clone https://bitbucket.org/awslabs/amazon-s3-bitbucket-pipelines-python.git;
};

View File

@ -3,9 +3,14 @@ services:
- docker:dind
variables:
KASMVNC_COMMIT_ID: $CI_COMMIT_SHA
GITLAB_SHARED_DIND_DIR: /builds/$CI_PROJECT_PATH/shared
GIT_SUBMODULE_STRATEGY: normal
GIT_FETCH_EXTRA_FLAGS: --tags
# E.g. BUILD_JOBS: build_debian_buster,build_ubuntu_bionic. This will include
# arm builds, because build_debian_buster_arm matches build_debian_buster.
# "BUILD_JOBS: none" won't build any build jobs, nor www.
BUILD_JOBS: all
stages:
- www
@ -37,6 +42,9 @@ build_www:
- mkdir -p output/www
- cd builder
- tar -zcvf ../output/www/kasm_www.tar.gz www
only:
variables:
- $BUILD_JOBS !~ /^none$/
artifacts:
paths:
- output/
@ -51,6 +59,9 @@ build_ubuntu_bionic:
- *prepare_artfacts
script:
- bash builder/build-package ubuntu bionic
only:
variables:
- $BUILD_JOBS == 'all' || $BUILD_JOBS =~ $CI_JOB_NAME
artifacts:
paths:
- output/
@ -67,6 +78,9 @@ build_ubuntu_bionic_arm:
- *prepare_artfacts
script:
- bash builder/build-package ubuntu bionic
only:
variables:
- $BUILD_JOBS == 'all' || $BUILD_JOBS =~ $CI_JOB_NAME
artifacts:
paths:
- output/
@ -81,6 +95,9 @@ build_ubuntu_bionic_libjpeg_turbo:
- *prepare_artfacts
script:
- bash builder/build-package ubuntu bionic +libjpeg-turbo_latest
only:
variables:
- $BUILD_JOBS == 'all' || $BUILD_JOBS =~ $CI_JOB_NAME
artifacts:
paths:
- output/
@ -95,6 +112,9 @@ build_ubuntu_focal:
- *prepare_artfacts
script:
- bash builder/build-package ubuntu focal;
only:
variables:
- $BUILD_JOBS == 'all' || $BUILD_JOBS =~ $CI_JOB_NAME
artifacts:
paths:
- output/
@ -111,6 +131,9 @@ build_ubuntu_focal_arm:
- *prepare_artfacts
script:
- bash builder/build-package ubuntu focal;
only:
variables:
- $BUILD_JOBS == 'all' || $BUILD_JOBS =~ $CI_JOB_NAME
artifacts:
paths:
- output/
@ -127,7 +150,7 @@ build_debian_buster:
- bash builder/build-package debian buster;
only:
variables:
- $CI_COMMIT_MESSAGE =~ /\[full [cC][Ii]\]/
- $CI_COMMIT_MESSAGE =~ /\[full [cC][Ii]\]/ || $BUILD_JOBS =~ $CI_JOB_NAME
artifacts:
paths:
- output/
@ -146,7 +169,7 @@ build_debian_buster_arm:
- bash builder/build-package debian buster;
only:
variables:
- $CI_COMMIT_MESSAGE =~ /\[full [cC][Ii]\]/
- $CI_COMMIT_MESSAGE =~ /\[full [cC][Ii]\]/ || $BUILD_JOBS =~ $CI_JOB_NAME
artifacts:
paths:
- output/
@ -163,7 +186,7 @@ build_debian_bullseye:
- bash builder/build-package debian bullseye;
only:
variables:
- $CI_COMMIT_MESSAGE =~ /\[full [cC][Ii]\]/
- $CI_COMMIT_MESSAGE =~ /\[full [cC][Ii]\]/ || $BUILD_JOBS =~ $CI_JOB_NAME
artifacts:
paths:
- output/
@ -182,7 +205,7 @@ build_debian_bullseye_arm:
- bash builder/build-package debian bullseye;
only:
variables:
- $CI_COMMIT_MESSAGE =~ /\[full [cC][Ii]\]/
- $CI_COMMIT_MESSAGE =~ /\[full [cC][Ii]\]/ || $BUILD_JOBS =~ $CI_JOB_NAME
artifacts:
paths:
- output/
@ -197,6 +220,9 @@ build_kali_rolling:
- *prepare_artfacts
script:
- bash builder/build-package kali kali-rolling;
only:
variables:
- $BUILD_JOBS == 'all' || $BUILD_JOBS =~ $CI_JOB_NAME
artifacts:
paths:
- output/
@ -213,6 +239,9 @@ build_kali_rolling_arm:
- *prepare_artfacts
script:
- bash builder/build-package kali kali-rolling;
only:
variables:
- $BUILD_JOBS == 'all' || $BUILD_JOBS =~ $CI_JOB_NAME
artifacts:
paths:
- output/
@ -227,6 +256,9 @@ build_centos7:
- *prepare_artfacts
script:
- bash builder/build-package centos core
only:
variables:
- $BUILD_JOBS == 'all' || $BUILD_JOBS =~ $CI_JOB_NAME
artifacts:
paths:
- output/
@ -237,14 +269,25 @@ upload:
before_script:
- . .ci/upload.sh
script:
- export S3_BUILD_DIRECTORY="kasmvnc/${CI_COMMIT_SHA}"
- prepare_to_run_scripts_and_s3_uploads
- S3_CRASHPAD_BUILD_DIRECTORY="kasmvnc/crashpad/${CI_COMMIT_SHA}"
- for dbgsym_package in `find output/ -type f -name '*dbgsym*deb'`; do
deb_package=$(find_deb_package "$dbgsym_package");
xvnc_md5sum=$(fetch_xvnc_md5sum "$deb_package");
upload_filename="${S3_CRASHPAD_BUILD_DIRECTORY}/${xvnc_md5sum}/kasmvncserver-dbgsym.deb";
echo;
echo "File to upload $upload_filename";
upload_to_s3 "$dbgsym_package" "$upload_filename" "$S3_BUCKET";
rm "$dbgsym_package";
done
- export S3_BUILD_DIRECTORY="kasmvnc/${CI_COMMIT_SHA}"
- export RELEASE_VERSION=$(.ci/next_release_version "$CI_COMMIT_REF_NAME")
- for package in `find output/ -type f -name '*.deb' -or -name '*.rpm'`; do
prepare_upload_filename "$package";
upload_filename="${S3_BUILD_DIRECTORY}/$upload_filename";
echo;
echo "File to upload $upload_filename";
upload_to_s3 "$package" "$upload_filename";
UPLOAD_NAME=$(echo $upload_filename | sed 's#kasmvncserver_##' | sed -r 's#_([0-9]{1,3}\.){2}[0-9]{1,2}_\S+?([a-f0-9]{6})##' | sed -r 's#\.(deb|rpm)##');
upload_to_s3 "$package" "$upload_filename" "$S3_BUCKET";
UPLOAD_NAME=$(basename $upload_filename | sed 's#kasmvncserver_##' | sed -r 's#_([0-9]{1,3}\.){2}[0-9]{1,2}_\S+?([a-f0-9]{6})##' | sed -r 's#\.(deb|rpm)##');
curl --request POST --header "PRIVATE-TOKEN:${GITLAB_API_TOKEN}" "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/statuses/${CI_COMMIT_SHA}?state=success&name=${UPLOAD_NAME}&target_url=${S3_URL}";
done

View File

@ -29,5 +29,6 @@ fi
dpkg-buildpackage -us -uc -b
mkdir -p "$os_dir"
cp ../*dbgsym*deb "$os_dir"
cp ../*.deb "$os_dir"
lintian ../*.deb || true

View File

@ -31,6 +31,7 @@ docker build -t "$builder_image" \
-f builder/dockerfile.${os}_${os_codename}${build_tag}.build .
mkdir -p builder/build
docker run -v $shared_with_docker_dir:/build -e BUILD_TAG="$build_tag" \
-e KASMVNC_COMMIT_ID="$KASMVNC_COMMIT_ID" \
--rm "$builder_image"
L_GID=$(id -g)

View File

@ -7,6 +7,12 @@ detect_quilt() {
fi
}
ensure_crashpad_can_fetch_line_number_by_address() {
if [ ! -f /etc/centos-release ]; then
export LDFLAGS="$LDFLAGS -no-pie"
fi
}
# For build-dep to work, the apt sources need to have the source server
#sudo apt-get build-dep xorg-server
@ -38,7 +44,7 @@ tar -C unix/xserver -xf /tmp/xorg-server-${XORG_VER}.tar.bz2 --strip-components=
cd unix/xserver
patch -Np1 -i ../xserver${XORG_PATCH}.patch
case "$XORG_VER" in
1.20.*)
1.20.*)
if [ -f ../xserver120.7.patch ]; then
patch -Np1 -i ../xserver120.7.patch
fi ;;
@ -49,12 +55,13 @@ autoreconf -i
# The distro paths start after prefix and end with the font path,
# everything after that is based on BUILDING.txt to remove unneeded
# components.
ensure_crashpad_can_fetch_line_number_by_address
./configure --prefix=/opt/kasmweb \
--with-xkb-path=/usr/share/X11/xkb \
--with-xkb-output=/var/lib/xkb \
--with-xkb-bin-directory=/usr/bin \
--with-default-font-path="/usr/share/fonts/X11/misc,/usr/share/fonts/X11/cyrillic,/usr/share/fonts/X11/100dpi/:unscaled,/usr/share/fonts/X11/75dpi/:unscaled,/usr/share/fonts/X11/Type1,/usr/share/fonts/X11/100dpi,/usr/share/fonts/X11/75dpi,built-ins" \
--with-pic --without-dtrace --disable-dri \
--without-dtrace --disable-dri \
--disable-static \
--disable-xinerama --disable-xvfb --disable-xnest --disable-xorg \
--disable-dmx --disable-xwin --disable-xephyr --disable-kdrive \

39
unix/kasm-upload-crash-dump.sh Executable file
View File

@ -0,0 +1,39 @@
#!/bin/sh
URL=https://to-be-filled.com/path
die() {
echo "$@"
exit
}
[ "$#" -ne 1 ] && die "Usage: $0 logfile"
grep -q Backtrace: $1 || die "No crash in log file"
CURL=`which curl`
WGET=`which wget`
[ -n "$CURL" -o -n "$WGET" ] || die "Curl or wget required"
BIN=`grep vnc $1 | tail -n1 | cut -d: -f2 | cut -d\( -f1`
[ -f $BIN ] || die "Can't locate binary"
#
# prep done, filter the log file
#
TMP=`mktemp`
LANG=C date >> $TMP
md5sum $BIN >> $TMP
$BIN -version 2>&1 | grep built >> $TMP
grep -A200 Backtrace: $1 >> $TMP
if [ -n "$CURL" ]; then
echo curl --data-binary @"$TMP" "$URL"
else
echo wget --post-file "$TMP" "$URL"
fi
rm $TMP

View File

@ -41,6 +41,7 @@ nodist_EXTRA_Xvnc_SOURCES = dummy.cxx
Xvnc_CPPFLAGS = $(XVNC_CPPFLAGS) -DKASMVNC -DNO_MODULE_EXTS \
-UHAVE_CONFIG_H \
-DXVNCEXTRAVERSION="\".$(KASMVNC_COMMIT_ID)\"" \
-DXFree86Server -DVENDOR_RELEASE="$(VENDOR_RELEASE)" \
-DVENDOR_STRING="\"$(VENDOR_STRING)\"" -I$(KASMVNC_SRCDIR)/common -I$(KASMVNC_SRCDIR)/unix/common \
-I$(top_srcdir)/include ${XSERVERLIBS_CFLAGS} -I$(includedir)

View File

@ -159,7 +159,9 @@ static int vncVerbose = DEFAULT_LOG_VERBOSITY;
static void
vncPrintBanner(void)
{
ErrorF("\nXvnc %s - built %s\n%s", XVNCVERSION, buildtime, XVNCCOPYRIGHT);
ErrorF("\nXvnc %s%s - built %s\n%s", XVNCVERSION,
sizeof(XVNCEXTRAVERSION) > 2 ? XVNCEXTRAVERSION : "",
buildtime, XVNCCOPYRIGHT);
ErrorF("Underlying X server release %d, %s\n\n", VENDOR_RELEASE,
VENDOR_STRING);
}