From 68c789ed93624620348b32ec92490c9bfc537a25 Mon Sep 17 00:00:00 2001 From: Thanos Doukoudakis Date: Sat, 19 Jan 2019 15:19:08 +0000 Subject: Create Debian packages using Docker MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This patch will setup Docker containers to build the source and Debian packages. The script will use the private key under gpg-keys folder to sign the Debian packages. If the folder is not there or doesn’t exist then the build will fail. Keys are expected to be in .asc format, unencrypted and not password protected. Added .dockerignore file to avoid copying the cache files to the container. The output is stored in out/ folder and the folder structure is ready for update_debian_repo.sh script. SWIFT_PACKAGE_PLATFORMS defines the platforms that it will be build for. Default values are: ubuntu:bionic ubuntu:cosmic debian:stretch debian:sid. DOCKERRUNEXTRAFLAGS variable can set additional flags for Docker builders. Currently is set to --privileged to avoid an issue during build with Qt5.11.1 https://bugreports.qt.io/browse/QTBUG-64490. Updated the Debian repo scripts to new distributions and Docker changes. Test-Information: Run the script and created source & binaries for all the distributions. Verified that changelog, compact, control, copyright, rules and source/format exist under the Debian folder when building the Debian package. Created tags and generated dev builds, rc builds, and release builds to test the output of the different release channels. Created a container, tested the Swift and Swiften packages, and the installations: Tested the following debs with piuparts in a docker container of the system they are build for. No errors raised and a few warnings related with chroot (which fails within a docker container) *libswiften-dev-dbgsym*.deb, *libswiften-dev_*.deb *libswiften5-dbgsym_*.deb *libswiften5_*.deb *swift-im-dbgsym_*.deb *swift-im_*.deb Installed the above packages and verified the install with swift-im –version, headers exist in /usr/include/Swiften/ and libSwiften.so under /usr/lib/. Verified the gpg signatures on the .changes and .buildinfo files. Tested the output with the update_debian_repo.sh and the import was successful. Verified the .changes and .buildinfo files with lintion which reported the following errors and warnings (there are differences between different channels release, beta, development): E: swift-im changes: bad-distribution-in-changes-file development E: swift-im changes: multiple-distributions-in-changes-file beta development W: libswiften5: new-package-should-close-itp-bug W: libswiften-dev: new-package-should-close-itp-bug W: swift-im: new-package-should-close-itp-bug Tested that the man page exists after installing in a docker container (ubuntu xenial, debian stetch, debian sid). Change-Id: Ic28d2731bbca34716645e76e7e7e7e8183dea4f8 diff --git a/Swift/Packaging/Debian/.dockerignore b/Swift/Packaging/Debian/.dockerignore new file mode 100644 index 0000000..5d238e7 --- /dev/null +++ b/Swift/Packaging/Debian/.dockerignore @@ -0,0 +1 @@ +**/out diff --git a/Swift/Packaging/Debian/.gitignore b/Swift/Packaging/Debian/.gitignore index 5401ae0..9c8f275 100644 --- a/Swift/Packaging/Debian/.gitignore +++ b/Swift/Packaging/Debian/.gitignore @@ -1,3 +1,7 @@ /swift-* /swift_* /libswiften* +/out/ +/swift-src/ +swift-im*.orig.tar.gz +*.tmp.package diff --git a/Swift/Packaging/Debian/Dockerfile.package.in b/Swift/Packaging/Debian/Dockerfile.package.in new file mode 100644 index 0000000..071e475 --- /dev/null +++ b/Swift/Packaging/Debian/Dockerfile.package.in @@ -0,0 +1,23 @@ +FROM __DISTRO__ +MAINTAINER packages@swift.im + +ENV DEBIAN_VERSION=__DEBIAN_VERSION__ + +RUN \ + apt-get -qq update && \ + apt-get install -y lintian devscripts scons build-essential dh-make pkg-config libssl-dev qt5-default libqt5x11extras5-dev libqt5webkit5-dev qtmultimedia5-dev qttools5-dev-tools qt5-image-formats-plugins libqt5svg5-dev libminiupnpc-dev libnatpmp-dev libhunspell-dev libxml2-dev libxss-dev libboost-dev libboost-filesystem-dev libboost-program-options-dev libboost-regex-dev libboost-signals-dev libboost-system-dev libboost-thread-dev libboost-date-time-dev libidn11-dev docbook-xsl docbook-xml xsltproc libxml2-utils libsqlite3-dev zlib1g-dev help2man && \ + apt-get clean + +RUN mkdir /swift-packages +COPY * /swift-packages/ + +CMD \ + gpg --allow-secret-key-import --import /gpg-keys/*.asc && \ + cd /swift-packages && \ + tar xf swift-im_${DEBIAN_VERSION}*.orig.tar.gz && \ + tar xf swift-im_${DEBIAN_VERSION}*.debian.tar.xz -C /swift-packages/swift-im-${DEBIAN_VERSION}/ && \ + cd swift-im-${DEBIAN_VERSION} && \ + debuild -i -I -b --lintian-opts --profile debian --pedantic && \ + cd /swift-packages/ && \ + rm -rf /swift-packages/swift-im-${DEBIAN_VERSION}/ && \ + mv * /out/ diff --git a/Swift/Packaging/Debian/Dockerfile.source b/Swift/Packaging/Debian/Dockerfile.source new file mode 100644 index 0000000..3c23b2c --- /dev/null +++ b/Swift/Packaging/Debian/Dockerfile.source @@ -0,0 +1,17 @@ +FROM debian:stretch +VOLUME ["/gpg-keys"] +MAINTAINER packages@swift.im + +RUN \ + apt-get -qq update && \ + apt-get install -y lintian devscripts scons build-essential pkg-config libssl-dev qt5-default libqt5x11extras5-dev libqt5webkit5-dev qtmultimedia5-dev qttools5-dev-tools qt5-image-formats-plugins libqt5svg5-dev libminiupnpc-dev libnatpmp-dev libhunspell-dev libxml2-dev libxss-dev libboost-dev libboost-filesystem-dev libboost-program-options-dev libboost-regex-dev libboost-signals-dev libboost-system-dev libboost-thread-dev libboost-date-time-dev libidn11-dev docbook-xsl docbook-xml xsltproc libxml2-utils libsqlite3-dev zlib1g-dev help2man && \ + apt-get clean + +COPY swift-src /swift-src + +CMD \ + gpg --allow-secret-key-import --import /gpg-keys/*.asc && \ + cd /swift-src && \ + mv swift-im_*.orig.tar.gz ../ && \ + debuild -i -I -S --lintian-opts --pedantic && \ + mv /swift-im* /out/ diff --git a/Swift/Packaging/Debian/package_docker.sh b/Swift/Packaging/Debian/package_docker.sh new file mode 100755 index 0000000..7fd1a3e --- /dev/null +++ b/Swift/Packaging/Debian/package_docker.sh @@ -0,0 +1,70 @@ +#!/bin/bash + +set -e + +cd $(dirname $(readlink -f $0 || realpath $0)) +export PYTHONPATH=../../../BuildTools/SCons +VERSION=${VERSION:=`../../../BuildTools/GetBuildVersion.py swift`} +DEBIAN_VERSION=`../../../BuildTools/DebianizeVersion.py $VERSION` +SWIFTEN_SOVERSION=`../../../BuildTools/GetBuildVersion.py swift --major` +SWIFT_SOURCE_DIR=swift-im-$DEBIAN_VERSION +TARBALLBARE="swift-im_$DEBIAN_VERSION.orig.tar.gz" +TARBALL="${SHARED_DIR}${TARBALLBARE}" +# DEVATTN there is an issue with some versions of Qt(5.11.1) and docker see https://bugreports.qt.io/browse/QTBUG-64490. Workaround is to add privileged flag +if [ -z ${DOCKERRUNEXTRAFLAGS+x} ]; +then + DOCKERRUNEXTRAFLAGS="--privileged" +fi + +rm -rf $SWIFT_SOURCE_DIR +git clone ../../../.git $SWIFT_SOURCE_DIR + +rm -rf $SWIFT_SOURCE_DIR/.git +find $SWIFT_SOURCE_DIR -name .gitignore | xargs rm -f +find $SWIFT_SOURCE_DIR/3rdParty -type f | grep -v uuid | grep -v SConscript | grep -v miniupnp | grep -v natpmp || xargs rm -f +find $SWIFT_SOURCE_DIR/3rdParty -depth -empty -type d -exec rmdir {} \; +rm -rf $SWIFT_SOURCE_DIR/3rdParty/SCons +rm -rf $SWIFT_SOURCE_DIR/Swift/Packaging/Debian +rm -rf $SWIFT_SOURCE_DIR/BuildTools/DocBook/Fonts +rm -rf $SWIFT_SOURCE_DIR/BuildTools/Git/Hooks/commit-msg +echo $VERSION > $SWIFT_SOURCE_DIR/VERSION.swift + +tar czf $TARBALLBARE $SWIFT_SOURCE_DIR +cp $TARBALLBARE $SWIFT_SOURCE_DIR + +cp -r debian $SWIFT_SOURCE_DIR/debian +../../../BuildTools/UpdateDebianChangelog.py $SWIFT_SOURCE_DIR/debian/changelog $DEBIAN_VERSION + +cat $SWIFT_SOURCE_DIR/debian/control.in | sed -e "s/%SWIFTEN_SOVERSION%/$SWIFTEN_SOVERSION/g" | sed -e "s/%WEBKIT_DEPENDENCY%/$WEBKIT_DEPENDENCY/g" > $SWIFT_SOURCE_DIR/debian/control +rm $SWIFT_SOURCE_DIR/debian/control.in +mv $SWIFT_SOURCE_DIR/debian/libswiften.install $SWIFT_SOURCE_DIR/debian/libswiften$SWIFTEN_SOVERSION.install +cat ../../../COPYING.thirdparty | tail -n +3 >> $SWIFT_SOURCE_DIR/debian/copyright + +rm -rf swift-src +mv $SWIFT_SOURCE_DIR swift-src +docker build -t swift/swiftsourcebuilder -f Dockerfile.source . +mkdir -p out/source +docker run --rm -v`pwd`/gpg-keys:/gpg-keys -v`pwd`/out/source:/out swift/swiftsourcebuilder + +TMP_DOCKERFILE=Dockerfile.tmp.package + +if [ -z ${SWIFT_PACKAGE_PLATFORMS+x} ]; +then + platformsarray=( ubuntu:bionic ubuntu:cosmic debian:stretch debian:sid ) +else + platformsarray=( $SWIFT_PACKAGE_PLATFORMS ) +fi + +for distro in "${platformsarray[@]}"; +do + if [ -f $TMP_DOCKERFILE ]; + then + rm $TMP_DOCKERFILE + fi + cat Dockerfile.package.in | sed -e "s/__DISTRO__/$distro/" -e "s/__DEBIAN_VERSION__/$DEBIAN_VERSION/" > $TMP_DOCKERFILE + docker build -t swift/packager -f $TMP_DOCKERFILE out/source + OUT_DIR="out/${distro//\://}" + echo $OUT_DIR + mkdir -p $OUT_DIR + docker run --rm $DOCKERRUNEXTRAFLAGS -v`pwd`/gpg-keys:/gpg-keys -v`pwd`/$OUT_DIR:/out swift/packager +done diff --git a/Swift/Packaging/Debian/update_debian_repo.sh b/Swift/Packaging/Debian/update_debian_repo.sh index b057103..fc736ed 100644 --- a/Swift/Packaging/Debian/update_debian_repo.sh +++ b/Swift/Packaging/Debian/update_debian_repo.sh @@ -74,7 +74,7 @@ command -v reprepro >/dev/null 2>&1 || { echo >&2 "This script requires aptly bu mkdir -p $APT_REPO_ROOT if [ -z ${SWIFT_PACKAGE_PLATFORMS+x} ]; then - platformsarray=( xenial artful jessie stretch sid ) + platformsarray=( bionic cosmic stretch sid ) else platformsarray=( $SWIFT_PACKAGE_PLATFORMS ) fi @@ -85,13 +85,13 @@ for full_distribution_path in $INCOMING_FOLDER/{debian,ubuntu}/*; do distro_name=$(basename `dirname $full_distribution_path`) distro_reprepro_root=${APT_REPO_ROOT}/${distro_name}/${distro_version} - if ! [[ $SWIFT_PACKAGE_PLATFORMS == *"$distro_version"* ]]; then + if ! [[ ${platformsarray[@]} == *"$distro_version"* ]]; then echo "$distro_version was not found in SWIFT_PACKAGE_PLATFORMS. Skipping..." continue fi - # ensure reprepro diretctory for this distribution version is present - if [ ! -d "$distro_preprepro_root" ]; then + # ensure reprepro directory for this distribution version is present + if [ ! -d "$distro_reprepro_root" ]; then echo "Create distribution repository for $distro_name/$distro_version" mkdir -p $distro_reprepro_root mkdir -p ${distro_reprepro_root}/conf -- cgit v0.10.2-6-g49f6