From 49890cf8da3d530228ca90d1167b25d78b56633c Mon Sep 17 00:00:00 2001 From: Dmitry Maksyoma Date: Fri, 11 Feb 2022 14:57:25 +0000 Subject: [PATCH] Resolve KASM-2301 "Feature/ upload debug packages" --- .ci/upload.sh | 30 ++++++++++++++--- .gitlab-ci.yml | 57 +++++++++++++++++++++++++++++---- builder/build-deb-inside-docker | 1 + builder/build-tarball | 1 + builder/build.sh | 11 +++++-- unix/kasm-upload-crash-dump.sh | 39 ++++++++++++++++++++++ unix/xserver/hw/vnc/Makefile.am | 1 + unix/xserver/hw/vnc/xvnc.c | 4 ++- 8 files changed, 129 insertions(+), 15 deletions(-) create mode 100755 unix/kasm-upload-crash-dump.sh diff --git a/.ci/upload.sh b/.ci/upload.sh index ce810cf..c68b990 100644 --- a/.ci/upload.sh +++ b/.ci/upload.sh @@ -6,6 +6,27 @@ is_kasmvnc() { echo "$package" | grep -q 'kasmvncserver_' } +detect_deb_package_arch() { + local deb_package="$1" + echo "$deb_package" | sed -e 's/.\+_\([^.]\+\)\.\(d\?\)deb/\1/' +} + +find_deb_package() { + local dbgsym_package="$1" + + echo "$dbgsym_package" | sed -e 's/-dbgsym//; s/ddeb/deb/' +} + +fetch_xvnc_md5sum() { + local deb_package="$1" + deb_package=$(realpath "$deb_package") + + local tmpdir=$(mktemp -d) + cd "$tmpdir" + dpkg-deb -e "$deb_package" + cat DEBIAN/md5sums | grep bin/Xkasmvnc | cut -d' ' -f 1 +} + function prepare_upload_filename() { local package="$1"; @@ -34,19 +55,18 @@ function prepare_upload_filename() { function upload_to_s3() { local package="$1"; local upload_filename="$2"; + local s3_bucket="$3"; # Transfer to S3 - python3 amazon-s3-bitbucket-pipelines-python/s3_upload.py "${S3_BUCKET}" "$package" "${S3_BUILD_DIRECTORY}/${upload_filename}"; + python3 amazon-s3-bitbucket-pipelines-python/s3_upload.py "${s3_bucket}" "$package" "${upload_filename}"; # Use the Gitlab API to tell Gitlab where the artifact was stored - export S3_URL="https://${S3_BUCKET}.s3.amazonaws.com/${S3_BUILD_DIRECTORY}/${upload_filename}"; - export BUILD_STATUS="{\"key\":\"doc\", \"state\":\"SUCCESSFUL\", \"name\":\"${upload_filename}\", \"url\":\"${S3_URL}\"}"; - curl --request POST --header "PRIVATE-TOKEN:${GITLAB_API_TOKEN}" "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/statuses/${CI_COMMIT_SHA}?state=success&name=build-url&target_url=${S3_URL}"; + export S3_URL="https://${s3_bucket}.s3.amazonaws.com/${upload_filename}"; }; function prepare_to_run_scripts_and_s3_uploads() { export DEBIAN_FRONTEND=noninteractive; apt-get update; - apt-get install -y ruby2.7 git; + apt-get install -y ruby2.7 git wget; apt-get install -y python3 python3-pip python3-boto3 curl pkg-config libxmlsec1-dev; git clone https://bitbucket.org/awslabs/amazon-s3-bitbucket-pipelines-python.git; }; diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6bb1b3c..f360ce6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,9 +3,14 @@ services: - docker:dind variables: + KASMVNC_COMMIT_ID: $CI_COMMIT_SHA GITLAB_SHARED_DIND_DIR: /builds/$CI_PROJECT_PATH/shared GIT_SUBMODULE_STRATEGY: normal GIT_FETCH_EXTRA_FLAGS: --tags + # E.g. BUILD_JOBS: build_debian_buster,build_ubuntu_bionic. This will include + # arm builds, because build_debian_buster_arm matches build_debian_buster. + # "BUILD_JOBS: none" won't build any build jobs, nor www. + BUILD_JOBS: all stages: - www @@ -37,6 +42,9 @@ build_www: - mkdir -p output/www - cd builder - tar -zcvf ../output/www/kasm_www.tar.gz www + only: + variables: + - $BUILD_JOBS !~ /^none$/ artifacts: paths: - output/ @@ -51,6 +59,9 @@ build_ubuntu_bionic: - *prepare_artfacts script: - bash builder/build-package ubuntu bionic + only: + variables: + - $BUILD_JOBS == 'all' || $BUILD_JOBS =~ $CI_JOB_NAME artifacts: paths: - output/ @@ -67,6 +78,9 @@ build_ubuntu_bionic_arm: - *prepare_artfacts script: - bash builder/build-package ubuntu bionic + only: + variables: + - $BUILD_JOBS == 'all' || $BUILD_JOBS =~ $CI_JOB_NAME artifacts: paths: - output/ @@ -81,6 +95,9 @@ build_ubuntu_bionic_libjpeg_turbo: - *prepare_artfacts script: - bash builder/build-package ubuntu bionic +libjpeg-turbo_latest + only: + variables: + - $BUILD_JOBS == 'all' || $BUILD_JOBS =~ $CI_JOB_NAME artifacts: paths: - output/ @@ -95,6 +112,9 @@ build_ubuntu_focal: - *prepare_artfacts script: - bash builder/build-package ubuntu focal; + only: + variables: + - $BUILD_JOBS == 'all' || $BUILD_JOBS =~ $CI_JOB_NAME artifacts: paths: - output/ @@ -111,6 +131,9 @@ build_ubuntu_focal_arm: - *prepare_artfacts script: - bash builder/build-package ubuntu focal; + only: + variables: + - $BUILD_JOBS == 'all' || $BUILD_JOBS =~ $CI_JOB_NAME artifacts: paths: - output/ @@ -127,7 +150,7 @@ build_debian_buster: - bash builder/build-package debian buster; only: variables: - - $CI_COMMIT_MESSAGE =~ /\[full [cC][Ii]\]/ + - $CI_COMMIT_MESSAGE =~ /\[full [cC][Ii]\]/ || $BUILD_JOBS =~ $CI_JOB_NAME artifacts: paths: - output/ @@ -146,7 +169,7 @@ build_debian_buster_arm: - bash builder/build-package debian buster; only: variables: - - $CI_COMMIT_MESSAGE =~ /\[full [cC][Ii]\]/ + - $CI_COMMIT_MESSAGE =~ /\[full [cC][Ii]\]/ || $BUILD_JOBS =~ $CI_JOB_NAME artifacts: paths: - output/ @@ -163,7 +186,7 @@ build_debian_bullseye: - bash builder/build-package debian bullseye; only: variables: - - $CI_COMMIT_MESSAGE =~ /\[full [cC][Ii]\]/ + - $CI_COMMIT_MESSAGE =~ /\[full [cC][Ii]\]/ || $BUILD_JOBS =~ $CI_JOB_NAME artifacts: paths: - output/ @@ -182,7 +205,7 @@ build_debian_bullseye_arm: - bash builder/build-package debian bullseye; only: variables: - - $CI_COMMIT_MESSAGE =~ /\[full [cC][Ii]\]/ + - $CI_COMMIT_MESSAGE =~ /\[full [cC][Ii]\]/ || $BUILD_JOBS =~ $CI_JOB_NAME artifacts: paths: - output/ @@ -197,6 +220,9 @@ build_kali_rolling: - *prepare_artfacts script: - bash builder/build-package kali kali-rolling; + only: + variables: + - $BUILD_JOBS == 'all' || $BUILD_JOBS =~ $CI_JOB_NAME artifacts: paths: - output/ @@ -213,6 +239,9 @@ build_kali_rolling_arm: - *prepare_artfacts script: - bash builder/build-package kali kali-rolling; + only: + variables: + - $BUILD_JOBS == 'all' || $BUILD_JOBS =~ $CI_JOB_NAME artifacts: paths: - output/ @@ -227,6 +256,9 @@ build_centos7: - *prepare_artfacts script: - bash builder/build-package centos core + only: + variables: + - $BUILD_JOBS == 'all' || $BUILD_JOBS =~ $CI_JOB_NAME artifacts: paths: - output/ @@ -237,14 +269,25 @@ upload: before_script: - . .ci/upload.sh script: - - export S3_BUILD_DIRECTORY="kasmvnc/${CI_COMMIT_SHA}" - prepare_to_run_scripts_and_s3_uploads + - S3_CRASHPAD_BUILD_DIRECTORY="kasmvnc/crashpad/${CI_COMMIT_SHA}" + - for dbgsym_package in `find output/ -type f -name '*dbgsym*deb'`; do + deb_package=$(find_deb_package "$dbgsym_package"); + xvnc_md5sum=$(fetch_xvnc_md5sum "$deb_package"); + upload_filename="${S3_CRASHPAD_BUILD_DIRECTORY}/${xvnc_md5sum}/kasmvncserver-dbgsym.deb"; + echo; + echo "File to upload $upload_filename"; + upload_to_s3 "$dbgsym_package" "$upload_filename" "$S3_BUCKET"; + rm "$dbgsym_package"; + done + - export S3_BUILD_DIRECTORY="kasmvnc/${CI_COMMIT_SHA}" - export RELEASE_VERSION=$(.ci/next_release_version "$CI_COMMIT_REF_NAME") - for package in `find output/ -type f -name '*.deb' -or -name '*.rpm'`; do prepare_upload_filename "$package"; + upload_filename="${S3_BUILD_DIRECTORY}/$upload_filename"; echo; echo "File to upload $upload_filename"; - upload_to_s3 "$package" "$upload_filename"; - UPLOAD_NAME=$(echo $upload_filename | sed 's#kasmvncserver_##' | sed -r 's#_([0-9]{1,3}\.){2}[0-9]{1,2}_\S+?([a-f0-9]{6})##' | sed -r 's#\.(deb|rpm)##'); + upload_to_s3 "$package" "$upload_filename" "$S3_BUCKET"; + UPLOAD_NAME=$(basename $upload_filename | sed 's#kasmvncserver_##' | sed -r 's#_([0-9]{1,3}\.){2}[0-9]{1,2}_\S+?([a-f0-9]{6})##' | sed -r 's#\.(deb|rpm)##'); curl --request POST --header "PRIVATE-TOKEN:${GITLAB_API_TOKEN}" "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/statuses/${CI_COMMIT_SHA}?state=success&name=${UPLOAD_NAME}&target_url=${S3_URL}"; done diff --git a/builder/build-deb-inside-docker b/builder/build-deb-inside-docker index 734b846..778f8aa 100755 --- a/builder/build-deb-inside-docker +++ b/builder/build-deb-inside-docker @@ -29,5 +29,6 @@ fi dpkg-buildpackage -us -uc -b mkdir -p "$os_dir" +cp ../*dbgsym*deb "$os_dir" cp ../*.deb "$os_dir" lintian ../*.deb || true diff --git a/builder/build-tarball b/builder/build-tarball index 3a85a8e..dec43e9 100755 --- a/builder/build-tarball +++ b/builder/build-tarball @@ -31,6 +31,7 @@ docker build -t "$builder_image" \ -f builder/dockerfile.${os}_${os_codename}${build_tag}.build . mkdir -p builder/build docker run -v $shared_with_docker_dir:/build -e BUILD_TAG="$build_tag" \ + -e KASMVNC_COMMIT_ID="$KASMVNC_COMMIT_ID" \ --rm "$builder_image" L_GID=$(id -g) diff --git a/builder/build.sh b/builder/build.sh index 9db927c..8ace18f 100755 --- a/builder/build.sh +++ b/builder/build.sh @@ -7,6 +7,12 @@ detect_quilt() { fi } +ensure_crashpad_can_fetch_line_number_by_address() { + if [ ! -f /etc/centos-release ]; then + export LDFLAGS="$LDFLAGS -no-pie" + fi +} + # For build-dep to work, the apt sources need to have the source server #sudo apt-get build-dep xorg-server @@ -38,7 +44,7 @@ tar -C unix/xserver -xf /tmp/xorg-server-${XORG_VER}.tar.bz2 --strip-components= cd unix/xserver patch -Np1 -i ../xserver${XORG_PATCH}.patch case "$XORG_VER" in - 1.20.*) + 1.20.*) if [ -f ../xserver120.7.patch ]; then patch -Np1 -i ../xserver120.7.patch fi ;; @@ -49,12 +55,13 @@ autoreconf -i # The distro paths start after prefix and end with the font path, # everything after that is based on BUILDING.txt to remove unneeded # components. +ensure_crashpad_can_fetch_line_number_by_address ./configure --prefix=/opt/kasmweb \ --with-xkb-path=/usr/share/X11/xkb \ --with-xkb-output=/var/lib/xkb \ --with-xkb-bin-directory=/usr/bin \ --with-default-font-path="/usr/share/fonts/X11/misc,/usr/share/fonts/X11/cyrillic,/usr/share/fonts/X11/100dpi/:unscaled,/usr/share/fonts/X11/75dpi/:unscaled,/usr/share/fonts/X11/Type1,/usr/share/fonts/X11/100dpi,/usr/share/fonts/X11/75dpi,built-ins" \ - --with-pic --without-dtrace --disable-dri \ + --without-dtrace --disable-dri \ --disable-static \ --disable-xinerama --disable-xvfb --disable-xnest --disable-xorg \ --disable-dmx --disable-xwin --disable-xephyr --disable-kdrive \ diff --git a/unix/kasm-upload-crash-dump.sh b/unix/kasm-upload-crash-dump.sh new file mode 100755 index 0000000..a06fba6 --- /dev/null +++ b/unix/kasm-upload-crash-dump.sh @@ -0,0 +1,39 @@ +#!/bin/sh + +URL=https://to-be-filled.com/path + +die() { + echo "$@" + exit +} + +[ "$#" -ne 1 ] && die "Usage: $0 logfile" + +grep -q Backtrace: $1 || die "No crash in log file" + +CURL=`which curl` +WGET=`which wget` + +[ -n "$CURL" -o -n "$WGET" ] || die "Curl or wget required" + +BIN=`grep vnc $1 | tail -n1 | cut -d: -f2 | cut -d\( -f1` +[ -f $BIN ] || die "Can't locate binary" + +# +# prep done, filter the log file +# + +TMP=`mktemp` + +LANG=C date >> $TMP +md5sum $BIN >> $TMP +$BIN -version 2>&1 | grep built >> $TMP +grep -A200 Backtrace: $1 >> $TMP + +if [ -n "$CURL" ]; then + echo curl --data-binary @"$TMP" "$URL" +else + echo wget --post-file "$TMP" "$URL" +fi + +rm $TMP diff --git a/unix/xserver/hw/vnc/Makefile.am b/unix/xserver/hw/vnc/Makefile.am index 3692555..4a4958e 100644 --- a/unix/xserver/hw/vnc/Makefile.am +++ b/unix/xserver/hw/vnc/Makefile.am @@ -41,6 +41,7 @@ nodist_EXTRA_Xvnc_SOURCES = dummy.cxx Xvnc_CPPFLAGS = $(XVNC_CPPFLAGS) -DKASMVNC -DNO_MODULE_EXTS \ -UHAVE_CONFIG_H \ + -DXVNCEXTRAVERSION="\".$(KASMVNC_COMMIT_ID)\"" \ -DXFree86Server -DVENDOR_RELEASE="$(VENDOR_RELEASE)" \ -DVENDOR_STRING="\"$(VENDOR_STRING)\"" -I$(KASMVNC_SRCDIR)/common -I$(KASMVNC_SRCDIR)/unix/common \ -I$(top_srcdir)/include ${XSERVERLIBS_CFLAGS} -I$(includedir) diff --git a/unix/xserver/hw/vnc/xvnc.c b/unix/xserver/hw/vnc/xvnc.c index 2971290..6cc834d 100644 --- a/unix/xserver/hw/vnc/xvnc.c +++ b/unix/xserver/hw/vnc/xvnc.c @@ -159,7 +159,9 @@ static int vncVerbose = DEFAULT_LOG_VERBOSITY; static void vncPrintBanner(void) { - ErrorF("\nXvnc %s - built %s\n%s", XVNCVERSION, buildtime, XVNCCOPYRIGHT); + ErrorF("\nXvnc %s%s - built %s\n%s", XVNCVERSION, + sizeof(XVNCEXTRAVERSION) > 2 ? XVNCEXTRAVERSION : "", + buildtime, XVNCCOPYRIGHT); ErrorF("Underlying X server release %d, %s\n\n", VENDOR_RELEASE, VENDOR_STRING); }