Adding the generic_package.sh script file

build-gluster-org/jobs/debian-ubuntu-package.yml

@@ -0,0 +1,58 @@
+- job:
+    name: debian-package-builder
+    node: master
+    description: glusterfs package script to build both debian and ubuntu
+    project-type: freestyle
+
+    scm:
+    - glusterfs
+
+    parameters:
+    - string:
+        default: v7.0
+        description: Refspec to "git checkout" as the release code snapshot. Typically
+          a git annotated tag like "v6.0". Specifying a branch name will use the
+          current HEAD of that branch.
+        name: GERRIT_REFSPEC
+    - string:
+        default: 7
+        description: Series number for the package to be built against.
+        name: SERIES
+    - string:
+        default: 0
+        description: Version number for the package to be built against.
+        name: VERSION
+    - string:
+        default:
+        description: Release number for the package to be built against.
+          Leave it empty if you are building above series 5.
+        name: RELEASE
+    - string:
+        default: all
+        description: Use the default "all" or leave it empty to build
+          it for both Debian and Ubuntu. If a specific distribution is needed,
+          please name it as "debian" or "ubuntu".
+        name: OS
+    - string:
+        default:
+        description: Leave it empty to build it for both Debian and Ubuntu.
+          If a specific distribution is selected above, then mention the
+          specific flavor here. is needed. Flavors can be named as "xenial" or
+          numbered as "16.04" for ubuntu. For debian it can named as "bullseye"
+          or it can be numbered as "11".
+        name: FLAVOR
+    - string:
+        default:
+        description: The latest series with respect to all the existing series.
+        name: LATEST_SERIES
+    - string:
+        default:
+        Description: The latest version of latest series.
+        name: LATEST_VERSION
+    - string:
+        default: [email protected],[email protected]
+        description: Mail addresses to send the announcement of the packages too.
+        name: ANNOUNCE_EMAIL
+
+    builders:
+    - shell: /opt/qa/debian-ubuntu-package.sh

build-gluster-org/scripts/debian-ubuntu-package.sh

@@ -0,0 +1,63 @@
+#!/bin/bash
+
+#for both the debian and ubuntu packaging of various versions we have to
+#ssh to a particular ubuntu machine and package it using pbuilder.
+
+set -xe
+
+deb_flavors=(stretch buster bullseye)
+ub_flavors=(bionic groovy xenial focal hirsute)
+
+while [ $# -eq 5 ]
+do
+        echo "building everything"
+        echo "packing debian distribution"
+        for i in ${!deb_flavors[@]}; do
+        ~/build-gluster-org/scripts/generic_package.sh debian ${deb_flavors[$i]} $SERIES $VERSION $RELEASE $LATEST_SERIES $LATEST_VERSION
+        done
+
+        echo "packing ubuntu distribution"
+        for i in ${!ub_flavors[@]}; do
+        ~/build-gluster-org/scripts/generic_package.sh debian ${ub_flavors[$i]} $SERIES $VERSION $RELEASE $LATEST_SERIES $LATEST_VERSION
+        done
+done
+
+while [ $# -gt 5 ]
+do
+    if [ "$OS" == "all" ]; then
+        echo "packing all distribution"
+        echo "packing debian distribution"
+        flavors=(stretch buster bullseye)
+        for i in ${!deb_flavors[@]}; do
+        ~/build-gluster-org/scripts/generic_package.sh debian ${deb_flavors[$i]} $SERIES $VERSION $RELEASE $LATEST_SERIES $LATEST_VERSION
+        done
+
+        echo "packing ubuntu distribution"
+        flavors=(bionic groovy xenial focal hirsute)
+        for i in ${!ub_flavors[@]}; do
+        ~/build-gluster-org/scripts/generic_package.sh debian ${ub_flavors[$i]} $SERIES $VERSION $RELEASE $LATEST_SERIES $LATEST_VERSION
+        done
+
+    elif [ "$OS" == "debian" ]; then
+        echo "packing debian alone"
+        case $FLAVOR in
+        "stretch" | "9" | "buster" | "10" | "bullseye" | "11")
+        echo "packing debian ${FLAVOR} alone"
+        ../scripts/generic_package.sh $OS $FLAVOR $SERIES $VERSION $RELEASE $LATEST_SERIES $LATEST_VERSION
+        ;;
+        esac
+
+    elif [ "$OS" == "ubuntu" ]; then
+        echo "packing ubuntu alone"
+        case $FLAVOR in
+        "xenial" | "16.04" | "bionic" | "18.04" | "eoan" | "19.10" | "focal" | "Focal Fossa" | "20.04" | "hirsute" | "21.04" | "hirsute hippo")
+        echo "packing ubuntu ${FLAVOR} alone"
+        ../scripts/generic_package.sh $OS $FLAVOR $SERIES $VERSION $RELEASE $LATEST_SERIES $LATEST_VERSION
+        ;;
+        esac
+    fi
+done
+
+RET=$?
+
+exit $RET

build-gluster-org/scripts/generic_package.sh

@@ -0,0 +1,169 @@
+#!/bin/bash
+
+#immediately exit if any command has a non-zero exit status
+set -euxo pipefail
+
+#enable debugging
+set -x
+
+#cleanup
+trap finish EXIT
+trap finish SIGQUIT
+function finish {
+  # cleanup code
+  echo "removing the chroot"
+  sudo rm -rf /var/cache/pbuilder/base.tgz
+
+  #removing folders created while packaging
+  rm -rf ~/${os}-${flavor}-Glusterfs-${version}
+
+  #clean pool/* dists/* db/*
+  dpath='/var/www/repos/apt/debian/'
+  rm -rf ${dpath}/pool/* ${dpath}/dists/* ${dpath}/db/*
+
+}
+
+################################################################################################
+#OS (e.g. Ubuntu/Debian)
+#Series (e.g. 4.1)
+#Version (e.g. 4.1.0)
+#Release (e.g. 1)
+#Flavor(e.g. Ubuntu - xenial/bionic/eoan/focal/groovy, Debian - buster/stretch/bullseye)
+
+# to run use: 'bash -x generic_package.sh debian stretch 6 4.1.0 1'
+
+#Key generation: https://help.ubuntu.com/community/GnuPrivacyGuardHowto#Generating_an_OpenPGP_Key
+#Addditional info: sending of pubkey to a keyserv is been done using the MIT keyserv
+#################################################################################################
+os=$1
+flavor=$2
+series=$3
+version=$4
+release=$5
+latest_series=$6
+latest_version=$7
+
+#Keys required in debian builds
+declare -a debuild_keys
+debuild_keys="F9C958A3AEE0D2184FAD1CBD43607F0DC2F8238C"
+
+declare -a pbuild_keys
+pbuild_keys="BF11C87C"
+
+# Check for OS(Ubuntu or Debian)
+if [ "$os" == "ubuntu" ]; then
+        mirror="http://ubuntu.osuosl.org/ubuntu/"
+	debuild_key=4F5B5CA5
+elif [ "$os" == "debian" ]; then
+        mirror="http://ftp.us.debian.org/debian/"
+        if [ "$flavor" == "stretch" ]; then
+             debuild_key=${pbuild_keys}
+        else
+             debuild_key=${debuild_keys}
+        fi
+        pbuild_key=${pbuild_keys}
+else
+	echo "Exiting: OS should be debian or ubuntu. Please provide the right one"
+	exit
+fi
+
+mkdir ${os}-${flavor}-Glusterfs-${version}
+
+cd ${os}-${flavor}-Glusterfs-${version}
+
+mkdir build packages
+
+echo "Building glusterfs-${version}-${release} for ${flavor}"
+
+cd build
+
+TGZS=(`ls ~/glusterfs-${version}-?-*/build/glusterfs-${version}.tar.gz`)
+echo ${TGZS[0]}
+
+if [ -z ${TGZS[0]} ]; then
+        echo "wget https://download.gluster.org/pub/gluster/glusterfs/${series}/${version}/glusterfs-${version}.tar.gz"
+        wget https://download.gluster.org/pub/gluster/glusterfs/${series}/${version}/glusterfs-${version}.tar.gz
+else
+        echo "found ${TGZS[0]}, using it..."
+        cp ${TGZS[0]} .
+fi
+
+echo "Creating link file.."
+ln -s glusterfs-${version}.tar.gz glusterfs_${version}.orig.tar.gz
+
+echo "Untaring.."
+tar xpf glusterfs-${version}.tar.gz
+
+# Changelogs needed for building are maintained in a separate repo.
+# the repo has to be clone and updated properly so we can copy the changelogs so far.
+
+echo "Cloning the glusterfs-debian repo"
+git clone ssh://github.com/gluster/glusterfs-debian.git
+
+cd glusterfs-debian/
+
+git checkout -b ${flavor}-${series}-local origin/${flavor}-glusterfs-${series}
+
+if [ "$os" == "ubuntu" ]; then
+        dch --distribution ${flavor} -u medium -v ${version}-${os}1~${flavor}1 "GlusterFS ${version} GA"
+elif [ "$os" == "debian" ]; then
+        dch --distribution ${flavor} -u low -v (${version}-1 ${flavor}) "GlusterFS ${version} GA"
+fi
+
+git commit -a -m "Glusterfs ${version} G.A (${flavor})"
+
+echo "Copying Changelog to source"
+cp -a debian ../glusterfs-${version}/
+
+echo "Building source package.."
+cd ../glusterfs-${version}
+debuild -S -sa -k${debuild_key}
+
+echo "Uploading the packages.."
+if [ "$os" == "ubuntu" ]; then
+        cd ..
+        dput ppa:gluster/glusterfs-${series} glusterfs_${version}-${os}1~${flavor}1_source.changes
+
+    echo "Done"
+    exit
+fi
+
+# we are using the same builder machine to build so we are running the "pbuilder
+# create" everytime to create the chroot according to the os and flavor we want to build.
+echo "creating chroot for ${os} ${flavor}"
+sudo pbuilder create --distribution ${flavor} --mirror ${mirror} --debootstrapopts --keyring=/usr/share/keyrings/${os}-archive-keyring.gpg
+
+echo "Building glusterfs-${version} for ${os} ${flavor} using the chroot and .dsc we created"
+
+# have to use the .dsc file inside the ${os}${flavor} folder
+sudo pbuilder build ~/${os}-${flavor}-Glusterfs-${version}/build/glusterfs_${version}-${release}.dsc | tee build.log
+
+#move the packages to packages directory.
+mv /var/cache/pbuilder/result/glusterfs*${version}-${release}*.deb ~/${os}-${flavor}-Glusterfs-${version}/packages/
+
+if [ "$flavor" != "stretch" ]; then
+     mv /var/cache/pbuilder/result/libg*${version}-${release}*.deb ~/${os}-${flavor}-Glusterfs-${version}/packages/
+fi
+/usr/share/debdelta/dpkg-sig -v -k ${pbuild_key} --sign builder ~/${os}-${flavor}-Glusterfs-${version}/packages/glusterfs-*${version}-${release}*.deb
+
+cp ~/conf.distributions/${series} conf/distributions
+
+# distribute Debian packages using apt
+for i in ~/${os}-${flavor}-Glusterfs-${version}/packages/glusterfs-*${version}-${release}*; do reprepro includedeb $flavor $i; done
+if [ "$flavor" != "stretch" ]; then
+     for i in ~/${os}-${flavor}-Glusterfs-${version}/packages/libg*${version}-${release}*.deb; do reprepro includedeb $flavor $i; done
+fi
+reprepro includedsc ${flavor} ~/${os}-${flavor}-Glusterfs-${version}/build/glusterfs_${version}-${release}.dsc
+
+tar czf ~/${os}-${flavor}-Glusterfs-${version}/${flavor}-apt-amd64-${version}.tgz pool/ dists/
+
+echo "Pushing Changelog changes.."
+git push origin ${flavor}-${series}-local:${flavor}-glusterfs-${series}
+
+cd ~/${os}-${flavor}-Glusterfs-${version}
+
+#copy the tar.gz file produced by the build to download.rht.gluster.org:/var/www/scratch
+scp $flavor-apt-amd64-$version.tgz [email protected]:/var/www/scratch
+
+ssh [email protected] /var/www/html/pub/gluster/unpacking-script.sh $series $version $os $flavor $latest_version $latest_series
+echo "Done."