From 05d86902d57fb1dfb1b764b140148a03908ebe4b Mon Sep 17 00:00:00 2001 From: Sergey Kulanov Date: Fri, 16 Oct 2015 15:14:00 +0300 Subject: [PATCH] [perestroika] Inject perestroika into fuel-mirror While we have migration of perestroika to Python implementation we need to have friendly and open way to share and fix issues in current perestroika code Change-Id: I7fced52ddadc81d4d8a52659358936f6e68cc3e4 Closes-bug: #1489470 --- perestroika/.gitignore | 2 + perestroika/build-deb.sh | 135 +++++++++ perestroika/build-fuel-deb.sh | 112 ++++++++ perestroika/build-fuel-rpm.sh | 116 ++++++++ perestroika/build-functions.sh | 262 ++++++++++++++++++ perestroika/build-rpm.sh | 145 ++++++++++ .../docker-builder/build-deb-package.sh | 42 +++ .../docker-builder/build-rpm-package.sh | 43 +++ perestroika/docker-builder/config | 1 + .../docker-builder/create-deb-chroot.sh | 36 +++ .../docker-builder/create-rpm-chroot.sh | 32 +++ .../docker-builder/mockbuild/Dockerfile | 14 + .../mockbuild/mock_configure.sh | 24 ++ perestroika/docker-builder/sbuild/04tmpfs | 11 + perestroika/docker-builder/sbuild/Dockerfile | 41 +++ .../sbuild/precise-amd64-sbuild | 8 + .../docker-builder/sbuild/sbuild-key.pub | Bin 0 -> 427 bytes .../docker-builder/sbuild/sbuild-key.sec | Bin 0 -> 759 bytes .../docker-builder/sbuild/trusty-amd64-sbuild | 8 + .../docker-builder/update-deb-chroot.sh | 21 ++ .../docker-builder/update-rpm-chroot.sh | 23 ++ perestroika/publisher.v5/functions/locking.sh | 33 +++ .../functions/publish-functions.sh | 102 +++++++ .../publisher.v5/functions/rsync_functions.sh | 193 +++++++++++++ .../publisher.v5/publish-deb-binaries.sh | 208 ++++++++++++++ .../publisher.v5/publish-rpm-binaries.sh | 233 ++++++++++++++++ perestroika/publisher.v5/repoquerysrpm.py | 66 +++++ 27 files changed, 1911 insertions(+) create mode 100644 perestroika/.gitignore create mode 100755 perestroika/build-deb.sh create mode 100644 perestroika/build-fuel-deb.sh create mode 100644 perestroika/build-fuel-rpm.sh create mode 100644 perestroika/build-functions.sh create mode 100755 perestroika/build-rpm.sh create mode 100755 perestroika/docker-builder/build-deb-package.sh create mode 100755 perestroika/docker-builder/build-rpm-package.sh create mode 100644 perestroika/docker-builder/config create mode 100755 perestroika/docker-builder/create-deb-chroot.sh create mode 100755 perestroika/docker-builder/create-rpm-chroot.sh create mode 100644 perestroika/docker-builder/mockbuild/Dockerfile create mode 100755 perestroika/docker-builder/mockbuild/mock_configure.sh create mode 100644 perestroika/docker-builder/sbuild/04tmpfs create mode 100644 perestroika/docker-builder/sbuild/Dockerfile create mode 100644 perestroika/docker-builder/sbuild/precise-amd64-sbuild create mode 100644 perestroika/docker-builder/sbuild/sbuild-key.pub create mode 100644 perestroika/docker-builder/sbuild/sbuild-key.sec create mode 100644 perestroika/docker-builder/sbuild/trusty-amd64-sbuild create mode 100755 perestroika/docker-builder/update-deb-chroot.sh create mode 100755 perestroika/docker-builder/update-rpm-chroot.sh create mode 100644 perestroika/publisher.v5/functions/locking.sh create mode 100644 perestroika/publisher.v5/functions/publish-functions.sh create mode 100644 perestroika/publisher.v5/functions/rsync_functions.sh create mode 100755 perestroika/publisher.v5/publish-deb-binaries.sh create mode 100755 perestroika/publisher.v5/publish-rpm-binaries.sh create mode 100755 perestroika/publisher.v5/repoquerysrpm.py diff --git a/perestroika/.gitignore b/perestroika/.gitignore new file mode 100644 index 0000000..4993445 --- /dev/null +++ b/perestroika/.gitignore @@ -0,0 +1,2 @@ +.package-defaults +.publisher-defaults diff --git a/perestroika/build-deb.sh b/perestroika/build-deb.sh new file mode 100755 index 0000000..a0338e4 --- /dev/null +++ b/perestroika/build-deb.sh @@ -0,0 +1,135 @@ +#!/bin/bash + +set -o xtrace +set -o errexit + +[ -f ".packages-defaults" ] && source .packages-defaults +source build-functions.sh + +main () { + set_default_params + # Get package tree from gerrit + fetch_upstream + local _srcpath="${MYOUTDIR}/${PACKAGENAME}-src" + local _specpath=$_srcpath + local _testspath=$_srcpath + [ "$IS_OPENSTACK" == "true" ] && _specpath="${MYOUTDIR}/${PACKAGENAME}-spec${SPEC_PREFIX_PATH}" && _testspath="${MYOUTDIR}/${PACKAGENAME}-spec" + local _debianpath=$_specpath + + if [ -d "${_debianpath}/debian" ] ; then + # Unpacked sources and specs + local srcpackagename=`head -1 ${_debianpath}/debian/changelog | cut -d' ' -f1` + local version=`head -1 ${_debianpath}/debian/changelog | sed 's|^.*(||;s|).*$||' | awk -F "-" '{print $1}'` + # Get version number from the latest git tag for openstack packages + [ "$IS_OPENSTACK" == "true" ] && version=`git -C $_srcpath describe --abbrev=0` + # TODO: Deal with openstack RC tags like 2015.1.0rc1 + # It breaks debian version comparison. Need to replace 'rc' with '~rc' + local binpackagenames="`cat ${_debianpath}/debian/control | grep ^Package | cut -d' ' -f 2 | tr '\n' ' '`" + local epochnumber=`head -1 ${_debianpath}/debian/changelog | grep -o "(.:" | sed 's|(||'` + local distro=`head -1 ${_debianpath}/debian/changelog | awk -F'[ ;]' '{print $3}'` + + # Get last commit info + # $message $author $email $cdate $commitsha $lastgitlog + get_last_commit_info ${_srcpath} + + TAR_NAME="${srcpackagename}_${version#*:}.orig.tar.gz" + if [ "$IS_OPENSTACK" == "true" ] ; then + # Get revision number as commit count for src+spec projects + local _src_commit_count=`git -C $_srcpath rev-list --no-merges origin/${SOURCE_BRANCH} | wc -l` + local _spec_commit_count=`git -C $_specpath rev-list --no-merges origin/${SPEC_BRANCH} | wc -l` + local _rev=$(( $_src_commit_count + $_spec_commit_count )) + [ "$GERRIT_CHANGE_STATUS" == "NEW" ] && _rev=$(( $_rev + 1 )) + local release="1~u14.04+mos${_rev}" + [ "$GERRIT_CHANGE_STATUS" == "NEW" ] && release="${release}+git.${gitshasrc}.${gitshaspec}" + local fullver=${epochnumber}${version}-${release} + # Update version and changelog + local firstline=1 + local _dchopts="-c ${_debianpath}/debian/changelog" + echo "$lastgitlog" | while read LINE; do + [ $firstline == 1 ] && local cmd="dch $_dchopts -D $distro -b --force-distribution -v $fullver" || local cmd="dch $_dchopts -a" + firstline=0 + local commitid=`echo "$LINE" | cut -d'|' -f1` + local email=`echo "$LINE" | cut -d'|' -f2` + local author=`echo "$LINE" | cut -d'|' -f3` + local subject=`echo "$LINE" | cut -d'|' -f4` + DEBFULLNAME="$author" DEBEMAIL="$email" $cmd "$commitid $subject" + done + # Prepare source tarball + pushd $_srcpath &>/dev/null + if [ "$PACKAGENAME" == "murano-apps" -o "$PACKAGENAME" == "rally" ]; then + # Do not perform `setup.py sdist` for murano-apps and rally packages + tar -czf ${BUILDDIR}/$TAR_NAME $EXCLUDES . + else + python setup.py --version # this will download pbr if it's not available + PBR_VERSION=$version python setup.py sdist -d ${BUILDDIR}/ + mv ${BUILDDIR}/*.gz ${BUILDDIR}/$TAR_NAME + fi + popd &>/dev/null + else + # Update changelog + DEBFULLNAME=$author DEBEMAIL=$email dch -c ${_debianpath}/debian/changelog -a "$commitsha $message" + # Prepare source tarball + # Exclude debian and tests dir + mv ${_srcpath}/debian ${_srcpath}/renameforexcludedebian + [ -d "${_srcpath}/tests" ] && mv ${_srcpath}/tests ${_srcpath}/renameforexcludetests + pushd ${_srcpath} &>/dev/null + tar -czf "${BUILDDIR}/${TAR_NAME}" $EXCLUDES --exclude=renameforexcludedebian --exclude=renameforexcludetests * + popd &>/dev/null + mv ${_srcpath}/renameforexcludedebian ${_srcpath}/debian + [ -d "${_srcpath}/renameforexcludetests" ] && mv ${_srcpath}/renameforexcludetests ${_srcpath}/tests + fi + mkdir -p ${BUILDDIR}/$srcpackagename + cp -R ${_debianpath}/debian ${BUILDDIR}/${srcpackagename}/ + else + # Packed sources (.dsc + .gz ) + cp ${_srcpath}/* $BUILDDIR + fi + # Prepare tests folder to provide as parameter + rm -f ${WRKDIR}/tests.envfile + [ -d "${_testspath}/tests" ] && echo "TESTS_CONTENT='`tar -cz -C ${_testspath} tests | base64 -w0`'" > ${WRKDIR}/tests.envfile + + # Build stage + local REQUEST=$REQUEST_NUM + [ -n "$LP_BUG" ] && REQUEST=$LP_BUG + + COMPONENTS="main restricted" + EXTRAREPO="http://${REMOTE_REPO_HOST}/${DEB_REPO_PATH} ${DEB_DIST_NAME} ${COMPONENTS}" + [ "$IS_UPDATES" == 'true' ] \ + && EXTRAREPO="${EXTRAREPO}|http://${REMOTE_REPO_HOST}/${DEB_REPO_PATH} ${DEB_PROPOSED_DIST_NAME} ${COMPONENTS}" + [ "$GERRIT_CHANGE_STATUS" == "NEW" ] && [ "$IS_UPDATES" == "false" ] \ + && EXTRAREPO="${EXTRAREPO}|http://${REMOTE_REPO_HOST}/${REPO_REQUEST_PATH_PREFIX}/${REQUEST}/${DEB_REPO_PATH} ${DEB_DIST_NAME} ${COMPONENTS}" + [ "$GERRIT_CHANGE_STATUS" == "NEW" ] && [ "$IS_UPDATES" == "true" ] \ + && EXTRAREPO="${EXTRAREPO}|http://${REMOTE_REPO_HOST}/${REPO_REQUEST_PATH_PREFIX}/${REQUEST}/${DEB_REPO_PATH} ${DEB_PROPOSED_DIST_NAME} ${COMPONENTS}" + export EXTRAREPO + + pushd $BUILDDIR &>/dev/null + echo "BUILD_SUCCEEDED=false" > ${WRKDIR}/buildresult.params + bash -ex ${WRKDIR}/docker-builder/build-deb-package.sh + local exitstatus=`cat buildresult/exitstatus.sbuild || echo 1` + rm -f buildresult/exitstatus.sbuild + [ -f "buildresult/buildlog.sbuild" ] && mv buildresult/buildlog.sbuild ${WRKDIR}/buildlog.txt + fill_buildresult $exitstatus 0 $PACKAGENAME DEB + if [ "$exitstatus" == "0" ] ; then + tmpdir=`mktemp -d ${PKG_DIR}/build-XXXXXXXX` + rm -f ${WRKDIR}/buildresult.params + cat >${WRKDIR}/buildresult.params<<-EOL + BUILD_HOST=`hostname -f` + PKG_PATH=$tmpdir + GERRIT_CHANGE_STATUS=$GERRIT_CHANGE_STATUS + REQUEST_NUM=$REQUEST_NUM + LP_BUG=$LP_BUG + IS_SECURITY=$IS_SECURITY + EXTRAREPO="$EXTRAREPO" + REPO_TYPE=deb + DIST=$DIST + EOL + mv buildresult/* $tmpdir/ + fi + popd &>/dev/null + + exit $exitstatus +} + +main "$@" + +exit 0 diff --git a/perestroika/build-fuel-deb.sh b/perestroika/build-fuel-deb.sh new file mode 100644 index 0000000..27317d4 --- /dev/null +++ b/perestroika/build-fuel-deb.sh @@ -0,0 +1,112 @@ +#!/bin/bash + +set -o xtrace +set -o errexit + +[ -f .fuel-default ] && source .fuel-default +source $(dirname `readlink -e $0`)/build-functions.sh + +main () { + set_default_params + [ -n "$GERRIT_BRANCH" ] && SOURCE_BRANCH=$GERRIT_BRANCH && SOURCE_REFSPEC=$GERRIT_REFSPEC + [ -n "$GERRIT_PROJECT" ] && SRC_PROJECT=$GERRIT_PROJECT + PACKAGENAME=${SRC_PROJECT##*/} + local DEBSPECFILES="${PACKAGENAME}-src/debian" + fetch_upstream + + local _srcpath="${MYOUTDIR}/${PACKAGENAME}-src" + local _specpath=$_srcpath + local _debianpath=$_specpath + + if [ -d "${_debianpath}/debian" ] ; then + # Unpacked sources and specs + local srcpackagename=`head -1 ${_debianpath}/debian/changelog | cut -d' ' -f1` + local version=`head -1 ${_debianpath}/debian/changelog | sed 's|^.*(||;s|).*$||' | awk -F "-" '{print $1}'` + local binpackagenames="`cat ${_debianpath}/debian/control | grep ^Package | cut -d' ' -f 2 | tr '\n' ' '`" + local epochnumber=`head -1 ${_debianpath}/debian/changelog | grep -o "(.:" | sed 's|(||'` + local distro=`head -1 ${_debianpath}/debian/changelog | awk -F'[ ;]' '{print $3}'` + + # Get last commit info + # $message $author $email $cdate $commitsha $lastgitlog + get_last_commit_info ${_srcpath} + + # Get revision number as commit count for src+spec projects + local _rev=`git -C $_srcpath rev-list --no-merges origin/${SOURCE_BRANCH} | wc -l` + [ "$GERRIT_CHANGE_STATUS" == "NEW" ] && _rev=$(( $_rev + 1 )) + local release="1~u14.04+mos${_rev}" + [ "$GERRIT_CHANGE_STATUS" == "NEW" ] && release="${release}+git.${gitshasrc}" + local fullver=${epochnumber}${version}-${release} + # Update version and changelog + local firstline=1 + local _dchopts="-c ${_debianpath}/debian/changelog" + echo "$lastgitlog" | while read LINE; do + [ $firstline == 1 ] && local cmd="dch $_dchopts -D $distro -b --force-distribution -v $fullver" || local cmd="dch $_dchopts -a" + firstline=0 + local commitid=`echo "$LINE" | cut -d'|' -f1` + local email=`echo "$LINE" | cut -d'|' -f2` + local author=`echo "$LINE" | cut -d'|' -f3` + local subject=`echo "$LINE" | cut -d'|' -f4` + DEBFULLNAME="$author" DEBEMAIL="$email" $cmd "$commitid $subject" + done + + TAR_NAME="${srcpackagename}_${version#*:}.orig.tar.gz" + # Update changelog + DEBFULLNAME=$author DEBEMAIL=$email dch -c ${_debianpath}/debian/changelog -a "$commitsha $message" + # Prepare source tarball + # Exclude debian dir + mv ${_srcpath}/debian ${_srcpath}/renameforexcludedebian + pushd ${_srcpath} &>/dev/null + tar -czf "${BUILDDIR}/${TAR_NAME}" $EXCLUDES --exclude=renameforexcludedebian * + popd &>/dev/null + mv ${_srcpath}/renameforexcludedebian ${_srcpath}/debian + + mkdir -p ${BUILDDIR}/$srcpackagename + cp -R ${_debianpath}/debian ${BUILDDIR}/${srcpackagename}/ + fi + + # Build stage + local REQUEST=$REQUEST_NUM + [ -n "$LP_BUG" ] && REQUEST=$LP_BUG + + COMPONENTS="main restricted" + EXTRAREPO="http://${REMOTE_REPO_HOST}/${DEB_REPO_PATH} ${DEB_DIST_NAME} ${COMPONENTS}" + [ "$IS_UPDATES" == 'true' ] \ + && EXTRAREPO="${EXTRAREPO}|http://${REMOTE_REPO_HOST}/${DEB_REPO_PATH} ${DEB_PROPOSED_DIST_NAME} ${COMPONENTS}" + [ "$GERRIT_CHANGE_STATUS" == "NEW" ] && [ "$IS_UPDATES" == "false" ] \ + && EXTRAREPO="${EXTRAREPO}|http://${REMOTE_REPO_HOST}/${REPO_REQUEST_PATH_PREFIX}/${REQUEST}/${DEB_REPO_PATH} ${DEB_DIST_NAME} ${COMPONENTS}" + [ "$GERRIT_CHANGE_STATUS" == "NEW" ] && [ "$IS_UPDATES" == "true" ] \ + && EXTRAREPO="${EXTRAREPO}|http://${REMOTE_REPO_HOST}/${REPO_REQUEST_PATH_PREFIX}/${REQUEST}/${DEB_REPO_PATH} ${DEB_PROPOSED_DIST_NAME} ${COMPONENTS}" + export EXTRAREPO + + pushd $BUILDDIR &>/dev/null + echo "BUILD_SUCCEEDED=false" > ${WRKDIR}/buildresult.params + bash -ex ${WRKDIR}/docker-builder/build-deb-package.sh + local exitstatus=`cat buildresult/exitstatus.sbuild || echo 1` + rm -f buildresult/exitstatus.sbuild + [ -f "buildresult/buildlog.sbuild" ] && mv buildresult/buildlog.sbuild ${WRKDIR}/buildlog.txt + fill_buildresult $exitstatus 0 $PACKAGENAME DEB + if [ "$exitstatus" == "0" ] ; then + tmpdir=`mktemp -d ${PKG_DIR}/build-XXXXXXXX` + rm -f ${WRKDIR}/buildresult.params + cat >${WRKDIR}/buildresult.params<<-EOL + BUILD_HOST=`hostname -f` + PKG_PATH=$tmpdir + GERRIT_CHANGE_STATUS=$GERRIT_CHANGE_STATUS + REQUEST_NUM=$REQUEST_NUM + LP_BUG=$LP_BUG + IS_SECURITY=$IS_SECURITY + EXTRAREPO="$EXTRAREPO" + REPO_TYPE=deb + DIST=$DIST + EOL + mv buildresult/* $tmpdir/ + fi + popd &>/dev/null + echo "Packages: $PACKAGENAME" + + exit $exitstatus +} + +main $@ + +exit 0 diff --git a/perestroika/build-fuel-rpm.sh b/perestroika/build-fuel-rpm.sh new file mode 100644 index 0000000..2ac2f03 --- /dev/null +++ b/perestroika/build-fuel-rpm.sh @@ -0,0 +1,116 @@ +#!/bin/bash + +set -o xtrace +set -o errexit + +[ -f .fuel-default ] && source .fuel-default +source $(dirname `readlink -e $0`)/build-functions.sh + +main () { + set_default_params + [ -n "$GERRIT_BRANCH" ] && SOURCE_BRANCH=$GERRIT_BRANCH && SOURCE_REFSPEC=$GERRIT_REFSPEC + [ -n "$GERRIT_PROJECT" ] && SRC_PROJECT=$GERRIT_PROJECT + PACKAGENAME=${SRC_PROJECT##*/} + # Get package tree from gerrit + fetch_upstream + local _srcpath="${MYOUTDIR}/${PACKAGENAME}-src" + local _specpath="${_srcpath}/specs" + + # Get last commit info + # $message $author $email $cdate $commitsha $lastgitlog + get_last_commit_info ${_srcpath} + + # Update specs + local specfile=`find $_specpath -name *.spec` + local version=`rpm -q --specfile $specfile --queryformat '%{VERSION}\n' | head -1` + local release=`rpm -q --specfile $specfile --queryformat '%{RELEASE}\n' | head -1` + ## Add changelog section if it doesn't exist + [ `cat ${specfile} | grep -c '^%changelog'` -eq 0 ] && echo "%changelog" >> ${specfile} + local _rev=`git -C $_srcpath rev-list --no-merges origin/${SOURCE_BRANCH} | wc -l` + [ "$GERRIT_CHANGE_STATUS" == "NEW" ] && _rev=$(( $_rev + 1 )) + local release="1.mos${_rev}" + [ "$GERRIT_CHANGE_STATUS" == "NEW" ] && release="${release}.git.${gitshasrc}" + local TAR_NAME=${PACKAGENAME}-${version}.tar.gz + # Update version and changelog + sed -i "s|Version:.*$|Version: ${version}|" $specfile + sed -i "s|Release:.*$|Release: ${release}|" $specfile + sed -i "s|Source0:.*$|Source0: ${TAR_NAME}|" $specfile + ## Update changelog + local firstline=1 + if [ ! -z "$lastgitlog" ]; then + sed -i "/%changelog/i%newchangelog" ${specfile} + echo "$lastgitlog" | while read LINE; do + local commitid=`echo "$LINE" | cut -d'|' -f1` + local email=`echo "$LINE" | cut -d'|' -f2` + local author=`echo "$LINE" | cut -d'|' -f3` + # Get current date to avoid wrong chronological order in %changelog section + local date=`LC_TIME=C date +"%a %b %d %Y"` + local subject=`echo "$LINE" | cut -d'|' -f4` + [ $firstline == 1 ] && sed -i "/%changelog/i\* $date $author \<${email}\> \- ${version}-${release}" ${specfile} + sed -i "/%changelog/i\- $commitid $subject" ${specfile} + firstline=0 + done + fi + sed -i '/%changelog/i\\' ${specfile} + sed -i '/^%changelog/d' ${specfile} + sed -i 's|^%newchangelog|%changelog|' ${specfile} + cp ${specfile} ${BUILDDIR}/ + + # Prepare source tarball + pushd $_srcpath &>/dev/null + git archive --format tar --worktree-attributes HEAD > ${BUILDDIR}/${PACKAGENAME}.tar + git rev-parse HEAD > ${BUILDDIR}/version.txt + pushd $BUILDDIR &>/dev/null + tar -rf ${PACKAGENAME}.tar version.txt + gzip -9 ${PACKAGENAME}.tar + mv ${PACKAGENAME}.tar.gz ${PACKAGENAME}-${version}.tar.gz + [ -f version.txt ] && rm -f version.txt + popd &>/dev/null + popd &>/dev/null + + # Build stage + local REQUEST=$REQUEST_NUM + [ -n "$LP_BUG" ] && REQUEST=$LP_BUG + + EXTRAREPO="repo1,http://${REMOTE_REPO_HOST}/${RPM_OS_REPO_PATH}/x86_64" + [ "$IS_UPDATES" == 'true' ] && \ + EXTRAREPO="${EXTRAREPO}|repo2,http://${REMOTE_REPO_HOST}/${RPM_PROPOSED_REPO_PATH}/x86_64" + [ "$GERRIT_CHANGE_STATUS" == "NEW" ] && [ "$IS_UPDATES" == "false" ] && \ + EXTRAREPO="${EXTRAREPO}|repo3,http://${REMOTE_REPO_HOST}/${REPO_REQUEST_PATH_PREFIX}/${REQUEST}/${RPM_OS_REPO_PATH}/x86_64" + [ "$GERRIT_STATUS" == "NEW" ] && [ "$IS_UPDATES" == "true" ] && \ + EXTRAREPO="${EXTRAREPO}|repo3,http://${REMOTE_REPO_HOST}/${REPO_REQUEST_PATH_PREFIX}/${REQUEST}/${RPM_PROPOSED_REPO_PATH}/x86_64" + export EXTRAREPO + + pushd $BUILDDIR &>/dev/null + echo "BUILD_SUCCEEDED=false" > ${WRKDIR}/buildresult.params + bash -x ${WRKDIR}/docker-builder/build-rpm-package.sh + local exitstatus=`cat build/exitstatus.mock || echo 1` + rm -f build/exitstatus.mock build/state.log + [ -f "build/build.log" ] && mv build/build.log ${WRKDIR}/buildlog.txt + [ -f "build/root.log" ] && mv build/root.log ${WRKDIR}/rootlog.txt + fill_buildresult $exitstatus 0 $PACKAGENAME RPM + if [ "$exitstatus" == "0" ] ; then + tmpdir=`mktemp -d ${PKG_DIR}/build-XXXXXXXX` + rm -f ${WRKDIR}/buildresult.params + cat >${WRKDIR}/buildresult.params<<-EOL + BUILD_HOST=`hostname -f` + PKG_PATH=$tmpdir + GERRIT_CHANGE_STATUS=$GERRIT_CHANGE_STATUS + REQUEST_NUM=$REQUEST_NUM + LP_BUG=$LP_BUG + IS_SECURITY=$IS_SECURITY + EXTRAREPO="$EXTRAREPO" + REPO_TYPE=rpm + DIST=$DIST + EOL + mv build/* $tmpdir/ + fi + popd &>/dev/null + echo "Packages: $PACKAGENAME" + + exit $exitstatus +} + +main $@ + +exit 0 diff --git a/perestroika/build-functions.sh b/perestroika/build-functions.sh new file mode 100644 index 0000000..12106f9 --- /dev/null +++ b/perestroika/build-functions.sh @@ -0,0 +1,262 @@ +#!/bin/bash +[ -z "$GERRIT_USER" ] && GERRIT_USER='openstack-ci-jenkins' +[ -z "$GERRIT_HOST" ] && GERRIT_HOST=$gerrit_host +[ -z "$GERRIT_PORT" ] && GERRIT_PORT=$gerrit_port +[ -z "$GERRIT_PORT" ] && GERRIT_PORT=29418 +[ -z "$GERRIT_SCHEME" ] && GERRIT_SCHEME="ssh" +URL="${GERRIT_SCHEME}://${GERRIT_USER}@${GERRIT_HOST}:${GERRIT_PORT}" +GITDATA=${HOME}/gitdata/$GERRIT_HOST +METADATA=${HOME}/repometadata +PKG_DIR=${HOME}/built_packages +EXCLUDES='--exclude-vcs' +WRKDIR=`pwd` +MYOUTDIR=${WRKDIR}/wrk-build +BUILDDIR=${MYOUTDIR}/src-to-build +rm -rf $BUILDDIR +mkdir -p $BUILDDIR +[ ! -d "$PKG_DIR" ] && mkdir -p $PKG_DIR +[ -f "${WRKDIR}/buildlog.txt" ] && rm -f ${WRKDIR}/buildlog.txt + +error () { + echo + echo -e "ERROR: $*" + echo + exit 1 +} + +info () { + echo + echo -e "INFO: $*" + echo +} + +job_lock() { + local LOCKFILE=$1 + local TIMEOUT=600 + shift + fd=15 + eval "exec $fd>$LOCKFILE" + if [ "$1" = "set" ]; then + flock --timeout $TIMEOUT -x $fd + elif [ "$1" = "unset" ]; then + flock -u $fd + fi +} + +request_is_merged () { + local REF=$1 + local CHANGENUMBER=`echo $REF | cut -d '/' -f4` + local result=1 + local status=`ssh ${GERRIT_USER}@${GERRIT_HOST} -p $GERRIT_PORT gerrit query --format=TEXT $CHANGENUMBER | egrep -o " +status:.*" | awk -F': ' '{print $2}'` + [ "$status" == "MERGED" ] && local result=0 + return $result +} + +set_default_params () { + [ -z "$PROJECT_NAME" ] && error "Project name is not defined! Exiting!" + [ -z "$PROJECT_VERSION" ] && error "Project version is not defined! Exiting!" + [ -z "$SECUPDATETAG" ] && local SECUPDATETAG="^Security-update" + [ -z "$IS_SECURITY" ] && IS_SECURITY='false' + if [ -n "$GERRIT_PROJECT" ]; then + GERRIT_CHANGE_STATUS="NEW" + if [ -n "$GERRIT_REFSPEC" ]; then + request_is_merged $GERRIT_REFSPEC && GERRIT_CHANGE_STATUS="MERGED" + else + # Support ref-updated gerrit event + GERRIT_CHANGE_STATUS="REF_UPDATED" + GERRIT_BRANCH=$GERRIT_REFNAME + fi + if [ -n "$GERRIT_CHANGE_COMMIT_MESSAGE" ] ; then + local GERRIT_MEGGASE="`echo $GERRIT_CHANGE_COMMIT_MESSAGE | base64 -d || :`" + fi + if [ "$GERRIT_CHANGE_STATUS" == "NEW" ] ; then + REQUEST_NUM="CR-$GERRIT_CHANGE_NUMBER" + local _LP_BUG=`echo "$GERRIT_TOPIC" | egrep -o "bug/[0-9]+" | cut -d'/' -f2` + [ -z "$_LP_BUG" ] && _LP_BUG=`echo "$GERRIT_MEGGASE" | egrep -i -o "(closes|partial|related)-bug: ?#?[0-9]+" | sort -u | head -1 | awk -F'[: #]' '{print $NF}'` + [ -n "$_LP_BUG" ] && LP_BUG="LP-$_LP_BUG" + else + if [ -n "$GERRIT_MESSAGE" ] ; then + if [ `echo $GERRIT_MESSAGE | grep -c \"$SECUPDATETAG\"` -gt 0 ] ; then + IS_SECURITY='true' + fi + fi + fi + # Detect packagename + PACKAGENAME=${GERRIT_PROJECT##*/} + [ "${PACKAGENAME##*-}" == "build" ] && PACKAGENAME=${PACKAGENAME%-*} + SRC_PROJECT=${SRC_PROJECT_PATH}/$PACKAGENAME + [ "$IS_OPENSTACK" == "true" ] && SPEC_PROJECT=${SPEC_PROJECT_PATH}/${PACKAGENAME}${SPEC_PROJECT_SUFFIX} + case $GERRIT_PROJECT in + "$SRC_PROJECT" ) SOURCE_REFSPEC=$GERRIT_REFSPEC ;; + "$SPEC_PROJECT" ) SPEC_REFSPEC=$GERRIT_REFSPEC ;; + esac + SOURCE_BRANCH=$GERRIT_BRANCH + [ "$IS_OPENSTACK" == "true" ] && SPEC_BRANCH=$GERRIT_BRANCH + fi + [ -z "$PACKAGENAME" ] && error "Package name is not defined! Exiting!" + [ -z "$SOURCE_BRANCH" ] && error "Source branch is not defined! Exiting!" + [ "$IS_OPENSTACK" == "true" ] && [ -z "$SPEC_BRANCH" ] && SPEC_BRANCH=$SOURCE_BRANCH + [ "$IS_OPENSTACK" == "true" ] && SPEC_PROJECT=${SPEC_PROJECT_PATH}/${PACKAGENAME}${SPEC_PROJECT_SUFFIX} + SRC_PROJECT=${SRC_PROJECT_PATH}/$PACKAGENAME +} + +fetch_upstream () { + # find corresponding requests + if [ -n "$SPEC_PROJECT" ] ; then + local CORR_GERRIT_PROJECT=$SRC_PROJECT + [ "$GERRIT_PROJECT" == "$SRC_PROJECT" ] && CORR_GERRIT_PROJECT=$SPEC_PROJECT + local search_string="topic:${GERRIT_TOPIC} branch:${GERRIT_BRANCH} project:${CORR_GERRIT_PROJECT} -status:abandoned" + local CORR_CHANGE=`ssh -p $GERRIT_PORT ${GERRIT_USER}@$GERRIT_HOST gerrit query --current-patch-set \'${search_string}\'` + local CORR_CHANGE_REFSPEC="`echo \"${CORR_CHANGE}\" | grep 'ref:' | awk '{print $NF}'`" + local CORR_CHANGE_NUMBER=`echo $CORR_CHANGE_REFSPEC | cut -d'/' -f4` + local CORR_PATCHSET_NUMBER=`echo $CORR_CHANGE_REFSPEC | cut -d'/' -f5` + local CORR_CHANGE_URL=`echo "${CORR_CHANGE}" | grep 'url:' | awk '{print $NF}'` + local CORR_CHANGE_STATUS=`echo "${CORR_CHANGE}" | grep 'status:' | awk '{print $NF}'` + + local corr_ref_count=`echo "$CORR_CHANGE_REFSPEC" | wc -l` + [ $corr_ref_count -gt 1 ] && error "ERROR: Multiple corresponding changes found!" + if [ -n "$CORR_CHANGE_NUMBER" ] ; then + # Provide corresponding change to vote script + cat > ${WRKDIR}/corr.setenvfile <<-EOL + CORR_CHANGE_NUMBER=$CORR_CHANGE_NUMBER + CORR_PATCHSET_NUMBER=$CORR_PATCHSET_NUMBER + CORR_CHANGE_URL=$CORR_CHANGE_URL + CORR_CHANGE_REFSPEC=$CORR_CHANGE_REFSPEC + EOL + fi + # Do not perform build stage if corresponding CR is not merged + if [ -n "${CORR_CHANGE_STATUS}" ] && [ "$GERRIT_CHANGE_STATUS" == "MERGED" ] && [ "$CORR_CHANGE_STATUS" != "MERGED" ] ; then + echo "SKIPPED=1" >> ${WRKDIR}/corr.setenvfile + error "Skipping build due to unmerged status of corresponding change ${CORR_CHANGE_URL}" + fi + fi + + # Do not clone projects every time. It makes gerrit sad. Cache it! + for prj in $SRC_PROJECT $SPEC_PROJECT; do + # Update code base cache + [ -d ${GITDATA} ] || mkdir -p ${GITDATA} + if [ ! -d ${GITDATA}/$prj ]; then + info "Cache for $prj doesn't exist. Cloning to ${HOME}/gitdata/$prj" + mkdir -p ${GITDATA}/$prj + # Lock cache directory + job_lock ${GITDATA}/${prj}.lock set + pushd ${GITDATA} &>/dev/null + info "Cloning sources from $URL/$prj.git ..." + git clone "$URL/$prj.git" "$prj" + popd &>/dev/null + else + # Lock cache directory + job_lock ${GITDATA}/${prj}.lock set + info "Updating cache for $prj" + pushd ${GITDATA}/$prj &>/dev/null + info "Fetching sources from $URL/$prj.git ..." + # Replace git remote user + local remote=`git remote -v | head -1 | awk '{print $2}' | sed "s|//.*@|//${GERRIT_USER}@|"` + git remote rm origin + git remote add origin $remote + # Update gitdata + git fetch --all + popd &>/dev/null + fi + if [ "$prj" == "$SRC_PROJECT" ]; then + local _DIRSUFFIX=src + local _BRANCH=$SOURCE_BRANCH + [ -n "$SOURCE_REFSPEC" ] && local _REFSPEC=$SOURCE_REFSPEC + fi + if [ "$prj" == "$SPEC_PROJECT" ]; then + local _DIRSUFFIX=spec + local _BRANCH=$SPEC_BRANCH + [ -n "$SPEC_REFSPEC" ] && local _REFSPEC=$SPEC_REFSPEC + fi + [ -e "${MYOUTDIR}/${PACKAGENAME}-${_DIRSUFFIX}" ] && rm -rf "${MYOUTDIR}/${PACKAGENAME}-${_DIRSUFFIX}" + info "Getting $_DIRSUFFIX from $URL/$prj.git ..." + cp -R ${GITDATA}/${prj} ${MYOUTDIR}/${PACKAGENAME}-${_DIRSUFFIX} + # Unlock cache directory + job_lock ${GITDATA}/${prj}.lock unset + pushd ${MYOUTDIR}/${PACKAGENAME}-${_DIRSUFFIX} &>/dev/null + switch_to_revision $_BRANCH + # Get code from HEAD if change is merged + [ "$GERRIT_CHANGE_STATUS" == "MERGED" ] && unset _REFSPEC + # If _REFSPEC specified switch to it + if [ -n "$_REFSPEC" ] ; then + switch_to_changeset $prj $_REFSPEC + else + [ "$prj" == "${CORR_GERRIT_PROJECT}" ] && [ -n "${CORR_CHANGE_REFSPEC}" ] && switch_to_changeset $prj $CORR_CHANGE_REFSPEC + fi + popd &>/dev/null + case $_DIRSUFFIX in + src) gitshasrc=$gitsha + ;; + spec) gitshaspec=$gitsha + ;; + *) error "Unknown project type" + ;; + esac + unset _DIRSUFFIX + unset _BRANCH + unset _REFSPEC + done +} + +switch_to_revision () { + info "Switching to branch $*" + if ! git checkout $*; then + error "$* not accessible by default clone/fetch" + else + git reset --hard origin/$* + gitsha=`git log -1 --pretty="%h"` + fi +} + +switch_to_changeset () { + info "Switching to changeset $2" + git fetch "$URL/$1.git" $2 + git checkout FETCH_HEAD + gitsha=`git log -1 --pretty="%h"` +} + +get_last_commit_info () { + if [ -n "$1" ] ; then + pushd $1 &>/dev/null + message="$(git log -n 1 --pretty=format:%B)" + author=$(git log -n 1 --pretty=format:%an) + email=$(git log -n 1 --pretty=format:%ae) + cdate=$(git log -n 1 --pretty=format:%ad | cut -d' ' -f1-3,5) + commitsha=$(git log -n 1 --pretty=format:%h) + lastgitlog=$(git log --pretty="%h|%ae|%an|%s" -n 10) + popd &>/dev/null + fi +} + +fill_buildresult () { + #$status $time $PACKAGENAME $pkgtype + local status=$1 + local time=$2 + local packagename=$3 + local pkgtype=$4 + local xmlfilename=${WRKDIR}/buildresult.xml + local failcnt=0 + local buildstat="Succeeded" + [ "$status" != "0" ] && failcnt=1 && buildstat="Failed" + echo "" > $xmlfilename + echo -n "> $xmlfilename + if [ "$failcnt" == "0" ] ; then + echo "/>" >> $xmlfilename + else + echo ">" >> $xmlfilename + echo "" >> $xmlfilename + if [ -f "${WRKDIR}/buildlog.txt" ] ; then + cat ${WRKDIR}/buildlog.txt | sed -n '/^dpkg: error/,/^Package installation failed/p' | egrep -v '^Get|Selecting|Unpacking|Preparing' >> $xmlfilename || : + cat ${WRKDIR}/buildlog.txt | sed -n '/^The following information may help to resolve the situation/,/^Package installation failed/p' >> $xmlfilename || : + cat ${WRKDIR}/buildlog.txt | grep -B 20 '^dpkg-buildpackage: error' >> $xmlfilename || : + cat ${WRKDIR}/buildlog.txt | grep -B 20 '^EXCEPTION:' >> $xmlfilename || : + fi + if [ -f "${WRKDIR}/rootlog.txt" ] ; then + cat ${WRKDIR}/rootlog.txt | sed -n '/No Package found/,/Exception/p' >> $xmlfilename || : + cat ${WRKDIR}/rootlog.txt | sed -n '/Error: /,/You could try using --skip-broken to work around the problem/p' >> $xmlfilename || : + fi + echo "" >> $xmlfilename + echo "" >> $xmlfilename + fi + echo "" >> $xmlfilename +} diff --git a/perestroika/build-rpm.sh b/perestroika/build-rpm.sh new file mode 100755 index 0000000..67db7bd --- /dev/null +++ b/perestroika/build-rpm.sh @@ -0,0 +1,145 @@ +#!/bin/bash + +set -o xtrace +set -o errexit + +[ -f ".packages-defaults" ] && source .packages-defaults +source build-functions.sh + +main () { + set_default_params + # Get package tree from gerrit + fetch_upstream + local _srcpath="${MYOUTDIR}/${PACKAGENAME}-src" + local _specpath=$_srcpath + local _testspath=$_srcpath + [ "$IS_OPENSTACK" == "true" ] && _specpath="${MYOUTDIR}/${PACKAGENAME}-spec${SPEC_PREFIX_PATH}" && _testspath="${MYOUTDIR}/${PACKAGENAME}-spec" + + # Get last commit info + # $message $author $email $cdate $commitsha $lastgitlog + get_last_commit_info ${_srcpath} + + # Update specs + local specfile=`find $_specpath -name *.spec` + #local binpackagename=`rpm -q $RPMQUERYPARAMS --specfile $specfile --queryformat %{NAME}"\n" | head -1` + local define_macros=( + -D 'kernel_module_package_buildreqs kernel-devel' + -D 'kernel_module_package(n:v:r:s:f:xp:) \ +%package -n kmod-%{-n*} \ +Summary: %{-n*} kernel module(s) \ +Version: %{version} \ +Release: %{release} \ +%description -n kmod-%{-n*} \ +This package provides the %{-n*} kernel modules +' ) + local version=`rpm -q "${define_macros[@]}" --specfile $specfile --queryformat %{VERSION}"\n" | head -1` + local release=`rpm -q "${define_macros[@]}" --specfile $specfile --queryformat %{RELEASE}"\n" | head -1` + ## Add changelog section if it doesn't exist + [ "`cat ${specfile} | grep -c '^%changelog'`" -eq 0 ] && echo "%changelog" >> ${specfile} + if [ "$IS_OPENSTACK" == "true" ] ; then + # Get version number from the latest git tag for openstack packages + local version=`git -C $_srcpath describe --abbrev=0` + # TODO: Deal with openstack RC tags like 2015.1.0rc1 + # It breaks rpm version comparison. + # Get revision number as commit count for src+spec projects + local _src_commit_count=`git -C $_srcpath rev-list --no-merges origin/${SOURCE_BRANCH} | wc -l` + local _spec_commit_count=`git -C $_specpath rev-list --no-merges origin/${SPEC_BRANCH} | wc -l` + local _rev=$(( $_src_commit_count + $_spec_commit_count )) + [ "$GERRIT_CHANGE_STATUS" == "NEW" ] && _rev=$(( $_rev + 1 )) + local release="mos8.0.${_rev}" + [ "$GERRIT_CHANGE_STATUS" == "NEW" ] && release="${release}.git.${gitshasrc}.${gitshaspec}" + local TAR_NAME=${PACKAGENAME}-${version}.tar.gz + # Update version and changelog + sed -i "s|Version:.*$|Version: ${version}|" $specfile + sed -i "/Release/s|%{?dist}.*$|%{?dist}~${release}|" $specfile + sed -i "s|Source0:.*$|Source0: ${TAR_NAME}|" $specfile + # Prepare source tarball + pushd $_srcpath &>/dev/null + if [ "$PACKAGENAME" == "murano-apps" ]; then + # Do not perform `setup.py sdist` for murano-apps package + tar -czf ${BUILDDIR}/$TAR_NAME $EXCLUDES . + else + python setup.py --version # this will download pbr if it's not available + PBR_VERSION=$version python setup.py sdist -d ${BUILDDIR}/ + mv ${BUILDDIR}/*.gz ${BUILDDIR}/$TAR_NAME || : + fi + cp $_specpath/rpm/SOURCES/* ${BUILDDIR}/ &>/dev/null || : + else + # TODO: Support unpacked source tree + # Packed sources (.spec + .gz + stuff) + # Exclude tests folder + cp -R ${_srcpath}/* $BUILDDIR + [ -d "${BUILDDIR}/tests" ] && rm -rf ${BUILDDIR}/tests + fi + ## Update changelog + firstline=1 + if [ ! -z "$lastgitlog" ]; then + sed -i "/^%changelog/i%newchangelog" ${specfile} + echo "$lastgitlog" | while read LINE; do + commitid=`echo "$LINE" | cut -d'|' -f1` + email=`echo "$LINE" | cut -d'|' -f2` + author=`echo "$LINE" | cut -d'|' -f3` + # Get current date to avoid wrong chronological order in %changelog section + date=`LC_TIME=C date +"%a %b %d %Y"` + subject=`echo "$LINE" | cut -d'|' -f4` + + [ $firstline == 1 ] && sed -i "/^%changelog/i\* $date $author \<${email}\> \- ${version}-${release}" ${specfile} + sed -i "/^%changelog/i\- $commitid $subject" ${specfile} + firstline=0 + done + sed -i '/^%changelog/i\\' ${specfile} + sed -i '/^%changelog/d' ${specfile} + sed -i 's|^%newchangelog|%changelog|' ${specfile} + fi + echo "Resulting spec-file:" + cat ${specfile} + cp ${specfile} ${BUILDDIR}/ + # Prepare tests folder to provide as parameter + rm -f ${WRKDIR}/tests.envfile + [ -d "${_testspath}/tests" ] && echo "TESTS_CONTENT='`tar -cz -C ${_testspath} tests | base64 -w0`'" > ${WRKDIR}/tests.envfile + + # Build stage + local REQUEST=$REQUEST_NUM + [ -n "$LP_BUG" ] && REQUEST=$LP_BUG + EXTRAREPO="repo1,http://${REMOTE_REPO_HOST}/${RPM_OS_REPO_PATH}/x86_64" + + [ "$IS_UPDATES" == 'true' ] && \ + EXTRAREPO="${EXTRAREPO}|repo2,http://${REMOTE_REPO_HOST}/${RPM_PROPOSED_REPO_PATH}/x86_64" + [ "$GERRIT_CHANGE_STATUS" == "NEW" ] && [ "$IS_UPDATES" == "false" ] && \ + EXTRAREPO="${EXTRAREPO}|repo3,http://${REMOTE_REPO_HOST}/${REPO_REQUEST_PATH_PREFIX}/${REQUEST}/${RPM_OS_REPO_PATH}/x86_64" + [ "$GERRIT_STATUS" == "NEW" ] && [ "$IS_UPDATES" == "true" ] && \ + EXTRAREPO="${EXTRAREPO}|repo3,http://${REMOTE_REPO_HOST}/${REPO_REQUEST_PATH_PREFIX}/${REQUEST}/${RPM_PROPOSED_REPO_PATH}/x86_64" + export EXTRAREPO + + pushd $BUILDDIR &>/dev/null + echo "BUILD_SUCCEEDED=false" > ${WRKDIR}/buildresult.params + bash -x ${WRKDIR}/docker-builder/build-rpm-package.sh + local exitstatus=`cat build/exitstatus.mock || echo 1` + rm -f build/exitstatus.mock build/state.log + [ -f "build/build.log" ] && mv build/build.log ${WRKDIR}/buildlog.txt + [ -f "build/root.log" ] && mv build/root.log ${WRKDIR}/rootlog.txt + fill_buildresult $exitstatus 0 $PACKAGENAME RPM + if [ "$exitstatus" == "0" ] ; then + tmpdir=`mktemp -d ${PKG_DIR}/build-XXXXXXXX` + rm -f ${WRKDIR}/buildresult.params + cat >${WRKDIR}/buildresult.params<<-EOL + BUILD_HOST=`hostname -f` + PKG_PATH=$tmpdir + GERRIT_CHANGE_STATUS=$GERRIT_CHANGE_STATUS + REQUEST_NUM=$REQUEST_NUM + LP_BUG=$LP_BUG + IS_SECURITY=$IS_SECURITY + EXTRAREPO="$EXTRAREPO" + REPO_TYPE=rpm + DIST=$DIST + EOL + mv build/* $tmpdir/ + fi + popd &>/dev/null + + exit $exitstatus +} + +main "$@" + +exit 0 diff --git a/perestroika/docker-builder/build-deb-package.sh b/perestroika/docker-builder/build-deb-package.sh new file mode 100755 index 0000000..66af367 --- /dev/null +++ b/perestroika/docker-builder/build-deb-package.sh @@ -0,0 +1,42 @@ +#!/bin/bash -ex +. $(dirname $(readlink -f $0))/config +CONTAINERNAME=sbuild:latest +CACHEPATH=/var/cache/docker-builder/sbuild +[ -z "$DIST" ] && DIST=trusty + +if [ -n "$EXTRAREPO" ] ; then + EXTRACMD="" + OLDIFS="$IFS" + IFS='|' + for repo in $EXTRAREPO; do + IFS="$OLDIFS" + EXTRACMD="${EXTRACMD} --chroot-setup-commands=\"apt-add-repo deb $repo\" " + IFS='|' + done + IFS="$OLDIFS" +fi +dscfile=$(find . -maxdepth 1 -name \*.dsc | head -1) +debianfolder=$(find . -wholename "*debian/changelog*" | head -1 | sed 's|^./||; s|debian/changelog||') + +if [ -n "$dscfile" ]; then + SOURCEDEST=$dscfile + SOURCEDEST=`basename $SOURCEDEST` +elif [ -n "$debianfolder" ] ; then + SOURCEDEST=$debianfolder +fi + +docker run ${DNSPARAM} --privileged --rm -v ${CACHEPATH}:/srv/images:ro \ + -v $(pwd):/srv/source ${CONTAINERNAME} \ + bash -c "( sed -i '/debian\/rules/d' /usr/bin/sbuild + + DEB_BUILD_OPTIONS=nocheck /usr/bin/sbuild -d ${DIST} --nolog \ + --source --force-orig-source \ + $EXTRACMD \ + --chroot-setup-commands=\"apt-get update\" \ + --chroot-setup-commands=\"apt-get upgrade -f -y --force-yes\" \ + /srv/source/${SOURCEDEST} 2>&1 + echo \$? > /srv/build/exitstatus.sbuild ) \ + | tee /srv/build/buildlog.sbuild + rm -rf /srv/source/buildresult + mv /srv/build /srv/source/buildresult + chown -R `id -u`:`id -g` /srv/source" diff --git a/perestroika/docker-builder/build-rpm-package.sh b/perestroika/docker-builder/build-rpm-package.sh new file mode 100755 index 0000000..63d2525 --- /dev/null +++ b/perestroika/docker-builder/build-rpm-package.sh @@ -0,0 +1,43 @@ +#!/bin/bash -ex +. $(dirname $(readlink -f $0))/config +CONTAINERNAME=mockbuild:latest +CACHEPATH=/var/cache/docker-builder/mock +DIST_VERSION=`echo $DIST | sed 's|centos||'` +[ -z "${DIST_VERSION}" ] && DIST_VERSION=7 + +EXTRACMD=":" +if [ -n "$EXTRAREPO" ] ; then + EXTRACMD="sed -i" + OLDIFS="$IFS" + IFS='|' + for repo in $EXTRAREPO ; do + IFS="$OLDIFS" + reponame=${repo%%,*} + repourl=${repo##*,} + EXTRACMD="$EXTRACMD -e \"$ i[${reponame}]\nname=${reponame}\nbaseurl=${repourl}\ngpgcheck=0\nenabled=1\nskip_if_unavailable=1\"" + IFS='|' + done + IFS="$OLDIFS" + EXTRACMD="$EXTRACMD /etc/mock/centos-${DIST_VERSION}-x86_64.cfg" +fi + +docker run ${DNSPARAM} --privileged --rm -v ${CACHEPATH}:/srv/mock:ro \ + -v $(pwd):/home/abuild/rpmbuild ${CONTAINERNAME} \ + bash -x -c "mkdir -p /srv/tmpfs/cache + mount -t tmpfs overlay /srv/tmpfs/cache + mount -t aufs -o br=/srv/tmpfs/cache/:/srv/mock/cache none /var/cache/mock/ + $EXTRACMD + su - abuild -c 'mock -r centos-${DIST_VERSION}-x86_64 --verbose --update' + chown -R abuild.mock /home/abuild + [[ \$(ls /home/abuild/rpmbuild/*.src.rpm | wc -l) -eq 0 ]] \ + && su - abuild -c 'mock -r centos-${DIST_VERSION}-x86_64 --no-clean --no-cleanup-after --buildsrpm --verbose \ + --sources=/home/abuild/rpmbuild --resultdir=/home/abuild/rpmbuild --buildsrpm \ + --spec=\$(ls /home/abuild/rpmbuild/*.spec)' + rm -rf /home/abuild/rpmbuild/build + su - abuild -c 'mock -r centos-${DIST_VERSION}-x86_64 --no-clean --no-cleanup-after --verbose \ + --resultdir=/home/abuild/rpmbuild/build \$(ls /home/abuild/rpmbuild/*.src.rpm)' + echo \$? > /home/abuild/rpmbuild/build/exitstatus.mock + umount -f /var/cache/mock /srv/tmpfs/cache + rm -rf /srv/tmpfs + rm -f /home/abuild/rpmbuild/\*.src.rpm /home/abuild/rpmbuild/{build,root,state}.log + chown -R `id -u`:`id -g` /home/abuild" diff --git a/perestroika/docker-builder/config b/perestroika/docker-builder/config new file mode 100644 index 0000000..ea62f2f --- /dev/null +++ b/perestroika/docker-builder/config @@ -0,0 +1 @@ +DNSPARAM="--dns 172.18.80.136" diff --git a/perestroika/docker-builder/create-deb-chroot.sh b/perestroika/docker-builder/create-deb-chroot.sh new file mode 100755 index 0000000..61304f8 --- /dev/null +++ b/perestroika/docker-builder/create-deb-chroot.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# +# Prepare chroot (must exist before starting any builds) environment +# with `sbuild-createchroot` which prepares everything for building DEBs +# +# Usage: DIST=trusty ./create-deb-chroot.sh # for Trusty +# DIST=precise ./create-deb-chroot.sh # for Precise +# UPSTREAM_MIRROR=http://ua.archive.ubuntu.com/ubuntu/ ./create-deb-chroot.sh + +set -ex + +BIN="${0%/*}" + +source "${BIN}/config" + +CONTAINERNAME=sbuild:latest +CACHEPATH=/var/cache/docker-builder/sbuild +# define upstream Ubuntu mirror +MIRROR=${UPSTREAM_MIRROR:-http://mirror.yandex.ru/ubuntu} +# Use trusty distro by default +[ -z "${DIST}" ] && DIST=trusty + +if [ "${DIST}" != "precise" ] && [ "${DIST}" != "trusty" ]; then + echo "Unknown dist version: ${DIST}" + exit 1 +fi + +docker run ${DNSPARAM} --privileged --rm -v ${CACHEPATH}:/srv/images ${CONTAINERNAME} \ + bash -c "rm -f /etc/schroot/chroot.d/* + sbuild-createchroot ${DIST} /srv/images/${DIST}-amd64 ${MIRROR} + echo deb ${MIRROR} ${DIST} main universe multiverse restricted > /srv/images/${DIST}-amd64/etc/apt/sources.list + echo deb ${MIRROR} ${DIST}-updates main universe multiverse restricted >> /srv/images/${DIST}-amd64/etc/apt/sources.list + sbuild-update -udcar ${DIST} + echo '#!/bin/bash' > /srv/images/${DIST}-amd64/usr/bin/apt-add-repo + echo 'echo \$* >> /etc/apt/sources.list' >> /srv/images/${DIST}-amd64/usr/bin/apt-add-repo + chmod +x /srv/images/${DIST}-amd64/usr/bin/apt-add-repo" diff --git a/perestroika/docker-builder/create-rpm-chroot.sh b/perestroika/docker-builder/create-rpm-chroot.sh new file mode 100755 index 0000000..40e29b8 --- /dev/null +++ b/perestroika/docker-builder/create-rpm-chroot.sh @@ -0,0 +1,32 @@ +#!/bin/bash +# +# Prepare chroot (must exist before starting any builds) environment +# with `mock --init` which installs all packages (@buildsys-build) +# required for building RPMs +# +# Usage: DIST=6 ./create-rpm-chroot.sh # for CentOS 6 +# DIST=7 ./create-rpm-chroot.sh # for CentOS 7 + +set -ex + +BIN="${0%/*}" + +source "${BIN}/config" + +CONTAINERNAME=mockbuild:latest +CACHEPATH=/var/cache/docker-builder/mock + +# check DIST=centos6 which can be passed from upstream job or defined in env +DIST_VERSION=${DIST/centos/} +# by default we init env for CentOS 7 +[ -z "${DIST_VERSION}" ] && DIST_VERSION=7 + +if [ "${DIST_VERSION}" != 6 ] && [ "${DIST_VERSION}" != 7 ]; then + echo "Unknown dist version: ${DIST_VERSION}" + exit 1 +fi + +docker run ${DNSPARAM} --privileged --rm -v ${CACHEPATH}/cache:/var/cache/mock ${CONTAINERNAME} \ + bash -c "chown -R abuild:mock /var/cache/mock + chmod g+s /var/cache/mock + su - abuild -c 'mock -r centos-${DIST_VERSION}-x86_64 -v --init'" diff --git a/perestroika/docker-builder/mockbuild/Dockerfile b/perestroika/docker-builder/mockbuild/Dockerfile new file mode 100644 index 0000000..fea5fab --- /dev/null +++ b/perestroika/docker-builder/mockbuild/Dockerfile @@ -0,0 +1,14 @@ +FROM centos:centos7 +# Authors: Dmitry Burmistrov +# Igor Gnatenko +MAINTAINER Igor Gnatenko + + +RUN yum -y --disableplugin=fastestmirror install http://dl.fedoraproject.org/pub/epel/7/x86_64/e/epel-release-7-5.noarch.rpm && \ + yum -y --disableplugin=fastestmirror install --enablerepo=epel-testing mock && \ + yum clean --enablerepo=epel-testing all && \ + useradd abuild -g mock + +COPY mock_configure.sh / +RUN /mock_configure.sh; \ + rm -f /mock_configure.sh diff --git a/perestroika/docker-builder/mockbuild/mock_configure.sh b/perestroika/docker-builder/mockbuild/mock_configure.sh new file mode 100755 index 0000000..7560aa3 --- /dev/null +++ b/perestroika/docker-builder/mockbuild/mock_configure.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# Generate mock configuration files: +# /etc/mock/centos-7-x86_64.cfg +# /etc/mock/centos-6-x86_64.cfg +# both for el6, el7, +# Add configuration param: +# config_opts['macros']['%dist'] = '.${DIST}${DISTSUFFIX}' + +set -e + +for cfg in /etc/mock/epel-{6,7}-x86_64.cfg; do + DIST=$(awk -F"'" "/config_opts\['dist'\]/ {print \$4}" "${cfg}") + sed -e "/config_opts\['dist'\]/s/$/\nconfig_opts['macros']['%dist'] = '.${DIST}${DISTSUFFIX}'/" $cfg \ + >${cfg/epel/centos} +done +# Enable tmpfs mock plugin +cat > /etc/mock/site-defaults.cfg <> /etc/resolv.conf && \ + echo "deb $MIRROR $DIST main universe multiverse restricted" > /etc/apt/sources.list && \ + echo "deb $MIRROR $DIST-updates main universe multiverse restricted" >> /etc/apt/sources.list && \ + apt-get update && apt-get -y install sbuild debhelper && \ + apt-get clean && \ + mkdir -p /srv/build && \ + sed -i '/^1/d' /etc/sbuild/sbuild.conf && \ + echo "\$build_arch_all = 1;" >> /etc/sbuild/sbuild.conf && \ + echo "\$log_colour = 0;" >> /etc/sbuild/sbuild.conf && \ + echo "\$apt_allow_unauthenticated = 1;" >> /etc/sbuild/sbuild.conf && \ + echo "\$apt_update = 0;" >> /etc/sbuild/sbuild.conf && \ + echo "\$apt_clean = 0;" >> /etc/sbuild/sbuild.conf && \ + echo "\$build_source = 1;" >> /etc/sbuild/sbuild.conf && \ + echo "\$build_dir = '/srv/build';" >> /etc/sbuild/sbuild.conf && \ + echo "\$log_dir = '/srv/build';" >> /etc/sbuild/sbuild.conf && \ + echo "\$stats_dir = '/srv/build';" >> /etc/sbuild/sbuild.conf && \ + echo "\$verbose = 100;" >> /etc/sbuild/sbuild.conf && \ + echo "\$mailprog = '/bin/true';" >> /etc/sbuild/sbuild.conf && \ + echo "\$purge_build_deps = 'never';" >> /etc/sbuild/sbuild.conf && \ + echo "1;" >> /etc/sbuild/sbuild.conf + +COPY ./04tmpfs /etc/schroot/setup.d/04tmpfs +RUN chmod +x /etc/schroot/setup.d/04tmpfs + +COPY ./precise-amd64-sbuild /etc/schroot/chroot.d/precise-amd64-sbuild +COPY ./trusty-amd64-sbuild /etc/schroot/chroot.d/trusty-amd64-sbuild diff --git a/perestroika/docker-builder/sbuild/precise-amd64-sbuild b/perestroika/docker-builder/sbuild/precise-amd64-sbuild new file mode 100644 index 0000000..c067ebc --- /dev/null +++ b/perestroika/docker-builder/sbuild/precise-amd64-sbuild @@ -0,0 +1,8 @@ +[precise-amd64-sbuild] +type=directory +description=Ubuntu precise/amd64 build environment +directory=/srv/images/precise-amd64 +groups=root,sbuild +root-groups=root,sbuild +profile=sbuild +union-type=aufs \ No newline at end of file diff --git a/perestroika/docker-builder/sbuild/sbuild-key.pub b/perestroika/docker-builder/sbuild/sbuild-key.pub new file mode 100644 index 0000000000000000000000000000000000000000..4adcf0cfbc0b6b026653069e9ced57a2ee54637b GIT binary patch literal 427 zcmV;c0aX5&jRaI#NMZp50H{hc{?^v!qD-#aef==L1*cpI;>bxRu-;9Hw#*-~*iJ=C zaT+JJ=+wFY#Jmcpxjm^Muf@}p1+oB%p(@=aUvGb2wM6CmzkpT?0MW&4o3E{J5Gi+Kf)eTsjj*a7Bx!W?*I`200FdMQ(|># zY-Au)X=iR_av&&EVs&Y3WFSIyX>4R5L}hSgZe(R{V|gG!a${&|c4Z(-WqBzeJYsce zY-D6DbZ>8Lb1h_Lc4cfpY-w|Jb1q?QX>W9BE@Wk5X<=?IZ*pfoh`0n30RjLb1p-uA zNMZsTF9r(<2nPcK1{DYb2?`4Y76JnS0v-VZ7k~f?2@p{`S~%+#8fd5|1OIZmfymiF zu^`}VY>z}v8qPXa9sX$z{KC(i=tAK!F`d|qm*6K!v$v=MyvP?FZKgI%1@Te2$xCQc zF)ih22MG`_qL|B^-1^PbsQ!us_|$uy6AUA$T2V8ddT_wdAwGr4_M VY+feXeeie~PTa5_Qi{%ce2bJnvpxU- literal 0 HcmV?d00001 diff --git a/perestroika/docker-builder/sbuild/sbuild-key.sec b/perestroika/docker-builder/sbuild/sbuild-key.sec new file mode 100644 index 0000000000000000000000000000000000000000..8b4b2ba0be64c597249336ad012bd08a8442ac01 GIT binary patch literal 759 zcmV1MLC; zyR%)jwJo?gk@6#}=vbsL5+f-Dm;fvZIw+9bhooOH)4igdoVkxW@+MI@u?QYd*VrP5 zzbPk_(ehkSf-J6zpe(dtQ(|>#Y-Au)X=iR_av&&EVs&Y3WFSIyX>4R5L}hSgZe(R{ zV|gG!a${&|c4Z(-WqBzeJYsceY-D6DbZ>8Lb1h_Lc4cfpY-w|Jb1q?QX>W9BE@Wk5 zX<=?IZ*pfoh`0n30RjLb1p-uANMZsTF9r(<2nPcK1{DYb2?`4Y76JnS0v-VZ7k~f? z2@p{`S~%+#8fd5|1OIZmfymiFu^`}VY>z}v8qPXa9sX$z{KC(i=tAK!F`d|qm*6K! zv$v=MyvP?FZKgI%1@Te2$xCQcF)ih22MG`_qL|B^-1^PbsQ!us_|$uy6AUA$T2V8ddT_wdAwGr4_MY+feXeeie~PTa5_Qi{%ce2X3pRE_`u literal 0 HcmV?d00001 diff --git a/perestroika/docker-builder/sbuild/trusty-amd64-sbuild b/perestroika/docker-builder/sbuild/trusty-amd64-sbuild new file mode 100644 index 0000000..f1ee10e --- /dev/null +++ b/perestroika/docker-builder/sbuild/trusty-amd64-sbuild @@ -0,0 +1,8 @@ +[trusty-amd64-sbuild] +type=directory +description=Ubuntu trusty/amd64 build environment +directory=/srv/images/trusty-amd64 +groups=root,sbuild +root-groups=root,sbuild +profile=sbuild +union-type=aufs \ No newline at end of file diff --git a/perestroika/docker-builder/update-deb-chroot.sh b/perestroika/docker-builder/update-deb-chroot.sh new file mode 100755 index 0000000..6c95f4d --- /dev/null +++ b/perestroika/docker-builder/update-deb-chroot.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +set -ex + +BIN="${0%/*}" + +source "${BIN}/config" + +CONTAINERNAME=sbuild:latest +CACHEPATH=/var/cache/docker-builder/sbuild + +# Use trusty distro by default +[ -z "${DIST}" ] && DIST=trusty + +if [ "${DIST}" != "precise" ] && [ "${DIST}" != "trusty" ]; then + echo "Unknown dist version: ${DIST}" + exit 1 +fi + +docker run ${DNSPARAM} --privileged --rm -v ${CACHEPATH}:/srv/images ${CONTAINERNAME} \ + bash -c "sbuild-update -udcar ${DIST}" diff --git a/perestroika/docker-builder/update-rpm-chroot.sh b/perestroika/docker-builder/update-rpm-chroot.sh new file mode 100755 index 0000000..71b56a0 --- /dev/null +++ b/perestroika/docker-builder/update-rpm-chroot.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +set -ex + +BIN="${0%/*}" + +source "${BIN}/config" + +CONTAINERNAME=mockbuild:latest +CACHEPATH=/var/cache/docker-builder/mock + +# check DIST=centos6 which can be passed from upstream job or defined in env +DIST_VERSION=${DIST/centos/} +# by default we init env for CentOS 7 +[ -z "${DIST_VERSION}" ] && DIST_VERSION=7 + +if [ "${DIST_VERSION}" != 6 ] && [ "${DIST_VERSION}" != 7 ]; then + echo "Unknown dist version: ${DIST_VERSION}" + exit 1 +fi + +docker run ${DNSPARAM} --privileged --rm -v ${CACHEPATH}/cache:/var/cache/mock ${CONTAINERNAME} \ + bash -c "su - abuild -c 'mock -r centos-${DIST_VERSION}-x86_64 -v --update'" diff --git a/perestroika/publisher.v5/functions/locking.sh b/perestroika/publisher.v5/functions/locking.sh new file mode 100644 index 0000000..8b1f701 --- /dev/null +++ b/perestroika/publisher.v5/functions/locking.sh @@ -0,0 +1,33 @@ +#!/bin/bash -xe + +export LANG=C + +function exit_with_error() { + echo "$@" + exit 1 +} + +function job_lock() { + [ -z "$1" ] && exit_with_error "Lock file is not specified" + local LOCKFILE=$1 + shift + local fd=1000 + eval "exec $fd>>$LOCKFILE" + case $1 in + "set") + flock -x -n $fd \ + || exit_with_error "Process already running. Lockfile: $LOCKFILE" + ;; + "unset") + flock -u $fd + rm -f $LOCKFILE + ;; + "wait") + local TIMEOUT=${2:-3600} + echo "Waiting of concurrent process (lockfile: $LOCKFILE, timeout = $TIMEOUT seconds) ..." + flock -x -w $TIMEOUT $fd \ + && echo DONE \ + || exit_with_error "Timeout error (lockfile: $LOCKFILE)" + ;; + esac +} diff --git a/perestroika/publisher.v5/functions/publish-functions.sh b/perestroika/publisher.v5/functions/publish-functions.sh new file mode 100644 index 0000000..df4dc37 --- /dev/null +++ b/perestroika/publisher.v5/functions/publish-functions.sh @@ -0,0 +1,102 @@ +#!/bin/bash + +#[ -z "$RESYNCONLY" ] && RESYNCONLY=false +[ -z "$REPO_BASE_PATH" ] && REPO_BASE_PATH=${HOME}/pubrepos +[ -z "$PKG_PATH" ] && echo "ERROR: Remote path to built packages is not defined" && exit 1 +WRK_DIR=`pwd` +TMP_DIR=${WRK_DIR}/.tmpdir + +error () { + echo + echo -e "ERROR: $*" + echo + exit 1 +} + +info () { + echo + echo -e "INFO: $*" + echo +} + +check-gpg() { + local RESULT=0 + [ -z "$SIGKEYID" ] && echo "WARNING: No secret keys given" && RESULT=1 + # Test secret keys + [ $RESULT -eq 0 ] && [ `gpg --list-secret-keys | grep ^sec | grep -c "$SIGKEYID"` -eq 0 ] && error "No secret keys found" + # Check for password + if [ $RESULT -eq 0 ] ; then + timeout 5s bash -c "echo test | gpg -q --no-tty --batch --no-verbose --local-user $SIGKEYID -so - &>/dev/null" \ + || error "Unable to sign with $SIGKEYID key. Passphrase needed!" + fi + [ $RESULT -ne 0 ] && echo "WARNING: Fall back to unsigned mode" + return $RESULT +} + +sync-repo() { + local LOCAL_DIR=$1 + local REMOTE_DIR=$2 + local REQUEST_PATH_PREFIX=$3 + [ -n "$4" ] && local REQUEST_NUM=$4 + [ -n "$5" ] && local LP_BUG=$5 + + RSYNC_USER=${RSYNC_USER:-"mirror-sync"} + [ -z "$REMOTE_REPO_HOST" ] && error "Remote host to sync is not defined." + [ ! -d "${LOCAL_DIR}" ] && error "Repository ${LOCAL_DIR} doesn't exist!" + ## SYNC + source $(dirname `readlink -e $0`)/functions/rsync_functions.sh + mirrors_fail="" + for host in $REMOTE_REPO_HOST; do + # sync files to remote host + # $1 - remote host + # $2 - rsync user + # $3 - local dir + # $4 - remote dir + if [ "$GERRIT_CHANGE_STATUS" == "NEW" ] ; then + rsync_create_dir $host $RSYNC_USER ${REQUEST_PATH_PREFIX} + if [ -n "$LP_BUG" ] && [ -n "$REQUEST_NUM" ] ; then + # Remove existing REQUEST_NUM repository and set it as symlink to LP_BUG one + if [ $(rsync_list_links $host $RSYNC_USER ${REQUEST_PATH_PREFIX} | grep -c "^${REQUEST_NUM} ") -eq 0 ] ; then + rsync_delete_dir $host $RSYNC_USER ${REQUEST_PATH_PREFIX}${REQUEST_NUM} + else + rsync_delete_file $host $RSYNC_USER ${REQUEST_PATH_PREFIX}${REQUEST_NUM} + fi + rsync_create_symlink $host $RSYNC_USER ${REQUEST_PATH_PREFIX}${REQUEST_NUM} ${LP_BUG} + REMOTE_DIR=${REQUEST_PATH_PREFIX}${LP_BUG}/${REMOTE_DIR} + else + # Symlinked REQUEST_NUM repository should be removed in order to not affect LP_BUG one + [ $(rsync_list_links $host $RSYNC_USER ${REQUEST_PATH_PREFIX} | grep -c "^${REQUEST_NUM} ") -gt 0 ] \ + && rsync_delete_file $host $RSYNC_USER ${REQUEST_PATH_PREFIX}${REQUEST_NUM} + REMOTE_DIR=${REQUEST_PATH_PREFIX}${REQUEST_NUM}/${REMOTE_DIR} + fi + elif [ -n "$REQUEST_PATH_PREFIX" ] ; then + # Remove unused request repos + if [ -n "$REQUEST_NUM" ] ; then + if [ $(rsync_list_links $host $RSYNC_USER ${REQUEST_PATH_PREFIX} | grep -c "^${REQUEST_NUM} ") -eq 0 ] ; then + rsync_delete_dir $host $RSYNC_USER ${REQUEST_PATH_PREFIX}${REQUEST_NUM} + else + rsync_delete_file $host $RSYNC_USER ${REQUEST_PATH_PREFIX}${REQUEST_NUM} + fi + [ $(rsync_list_files $host $RSYNC_USER ${REQUEST_PATH_PREFIX} | grep -cF $REQUEST_NUM) -gt 0 ] \ + && rsync_delete_file $host $RSYNC_USER ${REQUEST_PATH_PREFIX}${REQUEST_NUM}.target.txt + fi + # Do not remove LP_BUG repo until all linked repos removed + [ -n "$LP_BUG" ] \ + && [ $(rsync_list_links $host $RSYNC_USER ${REQUEST_PATH_PREFIX} | grep -cF $LP_BUG) -eq 0 ] \ + && rsync_delete_dir $host $RSYNC_USER ${REQUEST_PATH_PREFIX}/$LP_BUG + fi + rsync_transfer $host $RSYNC_USER $LOCAL_DIR $REMOTE_DIR || mirrors_fail+=" ${host}" + done + #if [[ -n "$mirrors_fail" ]]; then + # echo Some mirrors failed to update: $mirrors_fail + # exit 1 + #else + # export MIRROR_VERSION="${TGTDIR}" + # export MIRROR_BASE="http://$RSYNCHOST_MSK/fwm/files/${MIRROR_VERSION}" + # echo "MIRROR = ${mirror}" > ${WORKSPACE:-"."}/mirror_staging.txt + # echo "MIRROR_VERSION = ${MIRROR_VERSION}" >> ${WORKSPACE:-"."}/mirror_staging.txt + # echo "MIRROR_BASE = $MIRROR_BASE" >> ${WORKSPACE:-"."}/mirror_staging.txt + # echo "FUEL_MAIN_BRANCH = ${FUEL_MAIN_BRANCH}" >> ${WORKSPACE:-"."}/mirror_staging.txt + # echo "Updated: ${MIRROR_VERSION}
ext msk srt kha" + #fi +} diff --git a/perestroika/publisher.v5/functions/rsync_functions.sh b/perestroika/publisher.v5/functions/rsync_functions.sh new file mode 100644 index 0000000..fcf1c0e --- /dev/null +++ b/perestroika/publisher.v5/functions/rsync_functions.sh @@ -0,0 +1,193 @@ +#!/bin/bash -xe + +export LANG=C + +# define this vars before use +SNAPSHOT_FOLDER=${SNAPSHOT_FOLDER:-"snapshots"} +LATESTSUFFIX=${LATESTSUFFIX:-"-latest"} + +export DATE=$(date "+%Y-%m-%d-%H%M%S") +export SAVE_LAST_DAYS=${SAVE_LAST_DAYS:-61} +export WARN_DATE=$(date "+%Y%m%d" -d "$SAVE_LAST_DAYS days ago") + +function get_empty_dir() { + echo $(mktemp -d) +} + +function get_symlink() { + local LINKDEST=$1 + local LINKNAME=$(mktemp -u) + ln -s --force $LINKDEST $LINKNAME && echo $LINKNAME +} + +function rsync_delete_file() { + local RSYNCHOST=$1 + local RSYNCUSER=$2 + local FILENAME=$(basename $3) + local FILEPATH=$(dirname $3) + local EMPTYDIR=$(get_empty_dir) + rsync -rv --delete --include=$FILENAME '--exclude=*' \ + $EMPTYDIR/ $RSYNCHOST::$RSYNCUSER/$FILEPATH/ + [ ! -z "$EMPTYDIR" ] && rm -rf $EMPTYDIR +} + +function rsync_delete_dir() { + local RSYNCHOST=$1 + local RSYNCUSER=$2 + local DIR=$3 + local EMPTYDIR=$(get_empty_dir) + rsync --delete -a $EMPTYDIR/ $RSYNCHOST::$RSYNCUSER/$DIR/ \ + && rsync_delete_file $RSYNCHOST $RSYNCUSER $DIR + [ ! -z "$EMPTYDIR" ] && rm -rf $EMPTYDIR +} + +function rsync_create_dir() { + local RSYNCHOST=$1 + local RSYNCUSER=$2 + local DIR=$3 + local EMPTYDIR=$(get_empty_dir) + local OIFS="$IFS" + IFS='/' + local dir='' + local _dir='' + for _dir in $DIR ; do + IFS="$OIFS" + dir="${dir}/${_dir}" + rsync -a $EMPTYDIR/ $RSYNCHOST::$RSYNCUSER/$dir/ + IFS='/' + done + IFS="$OIFS" + [ ! -z "$EMPTYDIR" ] && rm -rf $EMPTYDIR +} + +function rsync_create_symlink() { + # Create symlink $3 -> $4 + # E.g. "create_symlink repos/6.1 files/6.1-stable" + # wll create symlink repos/6.1 -> repos/files/6.1-stable + local RSYNCHOST=$1 + local RSYNCUSER=$2 + local LINKNAME=$3 + local LINKDEST=$4 + local SYMLINK_FILE=$(get_symlink "$LINKDEST") + rsync -vl $SYMLINK_FILE $RSYNCHOST::$RSYNCUSER/$LINKNAME + rm $SYMLINK_FILE + + # Make text file for dereference symlinks + local TARGET_TXT_FILE=$(mktemp) + echo "$LINKDEST" > $TARGET_TXT_FILE + rsync -vl $TARGET_TXT_FILE $RSYNCHOST::$RSYNCUSER/${LINKNAME}.target.txt + rm $TARGET_TXT_FILE +} + +function rsync_list() { + local RSYNCHOST=$1 + local RSYNCUSER=$2 + local DIR=$3 + local TEMPFILE=$(mktemp) + set +e + rsync -l $RSYNCHOST::$RSYNCUSER/$DIR/ 2>/dev/null > $TEMPFILE + local RESULT=$? + [ "$RESULT" == "0" ] && cat $TEMPFILE | grep -v '\.$' + rm $TEMPFILE + set -e + return $RESULT +} + +function rsync_list_links() { + local RSYNCHOST=$1 + local RSYNCUSER=$2 + local DIR=$3 + local TEMPFILE=$(mktemp) + set +e + rsync_list $RSYNCHOST $RSYNCUSER $DIR > $TEMPFILE + local RESULT=$? + [ "$RESULT" == "0" ] && cat $TEMPFILE | grep '^l' | awk '{print $(NF-2)" "$NF}' + rm $TEMPFILE + set -e + return $RESULT +} + +function rsync_list_dirs() { + local RSYNCHOST=$1 + local RSYNCUSER=$2 + local DIR=$3 + local TEMPFILE=$(mktemp) + set +e + rsync_list $RSYNCHOST $RSYNCUSER $DIR > $TEMPFILE + local RESULT=$? + [ "$RESULT" == "0" ] && cat $TEMPFILE | grep '^d' | awk '{print $NF}' + rm $TEMPFILE + set -e + return $RESULT +} + +function rsync_list_files() { + local RSYNCHOST=$1 + local RSYNCUSER=$2 + local DIR=$3 + local TEMPFILE=$(mktemp) + set +e + rsync_list $RSYNCHOST $RSYNCUSER ${DIR} > $TEMPFILE + local RESULT=$? + [ "$RESULT" == "0" ] && cat $TEMPFILE | grep -vE '^d|^l' | awk '{print $NF}' + rm $TEMPFILE + set -e + return $RESULT +} + +###################################################### +function rsync_remove_old_versions() { + # Remove mirrors older then $SAVE_LAST_DAYS and w/o symlinks on it + local RSYNCHOST=$1 + local RSYNCUSER=$2 + local REMOTEPATH=$3 + local FOLDERNAME=$4 + DIRS=$(rsync_list_dirs $RSYNCHOST $RSYNCUSER $REMOTEPATH | grep "^$FOLDERNAME\-" ) + for dir in $DIRS; do + ddate=$(echo $dir | awk -F '[-]' '{print $(NF-3)$(NF-2)$(NF-1)}') + [ "$ddate" -gt "$WARN_DATE" ] && continue + LINKS=$(rsync_list_links $RSYNCHOST $RSYNCUSER $REMOTEPATH | grep -F $dir ; rsync_list_links $RSYNCHOST $RSYNCUSER $(dirname $REMOTEPATH) | grep -F "$(basename $REMOTEPATH)/$dir") + if [ "$LINKS" = "" ]; then + rsync_delete_dir $RSYNCHOST $RSYNCUSER $REMOTEPATH/$dir + continue + fi + echo "Skip because symlinks $LINKS points to $dir" + done +} + +###################################################### +function rsync_transfer() { + # sync files to remote host + # $1 - remote host + # $2 - rsync module + # $3 - source dir 1/ + # $4 - remote dir 1/2/3/4/5 + # snapshots dir 1/2/3/4/snapshots + local RSYNC_HOST=$1 + local RSYNC_USER=$2 + local SOURCE_DIR=$3 + local REMOTE_DIR=$4 + + local SNAPSHOT_DIR=$(echo $REMOTE_DIR | sed "s|$(basename ${REMOTE_DIR})$|${SNAPSHOT_FOLDER}|") + + local SNAPSHOT_FOLDER=$(basename $SNAPSHOT_DIR) # snapshots + local SNAPSHOT_PATH=$(dirname $SNAPSHOT_DIR) # 1/2 + local REMOTE_ROOT=$(echo $REMOTE_DIR | sed "s|^$SNAPSHOT_PATH/||") + local REMOTE_ROOT=${REMOTE_ROOT%%/*} # 3 + rsync_list_dirs $RSYNC_HOST $RSYNC_USER $SNAPSHOT_DIR/${REMOTE_ROOT}-${DATE} \ + || rsync_create_dir $RSYNC_HOST $RSYNC_USER $SNAPSHOT_DIR/${REMOTE_ROOT}-${DATE} + + OPTIONS="--archive --verbose --force --ignore-errors --delete-excluded --no-owner --no-group \ + --delete --link-dest=/${SNAPSHOT_DIR}/${REMOTE_ROOT}${LATESTSUFFIX}" + + rsync ${OPTIONS} ${SOURCE_DIR}/ ${RSYNC_HOST}::${RSYNC_USER}/${SNAPSHOT_DIR}/${REMOTE_ROOT}-${DATE}/ \ + && rsync_delete_file $RSYNC_HOST $RSYNC_USER ${SNAPSHOT_DIR}/${REMOTE_ROOT}${LATESTSUFFIX} \ + && rsync_create_symlink $RSYNC_HOST $RSYNC_USER ${SNAPSHOT_DIR}/${REMOTE_ROOT}${LATESTSUFFIX} ${REMOTE_ROOT}-${DATE} \ + && rsync_delete_file $RSYNC_HOST $RSYNC_USER ${SNAPSHOT_PATH}/${REMOTE_ROOT} \ + && rsync_create_symlink $RSYNC_HOST $RSYNC_USER ${SNAPSHOT_PATH}/${REMOTE_ROOT} ${SNAPSHOT_FOLDER}/${REMOTE_ROOT}-${DATE} \ + && rsync_remove_old_versions $RSYNC_HOST $RSYNC_USER ${SNAPSHOT_DIR} ${REMOTE_ROOT} + RESULT=$? + [ $RESULT -ne 0 ] && rsync_delete_dir $RSYNC_HOST $RSYNC_USER ${SNAPSHOT_DIR}/${REMOTE_ROOT}-${DATE} + + return $RESULT +} diff --git a/perestroika/publisher.v5/publish-deb-binaries.sh b/perestroika/publisher.v5/publish-deb-binaries.sh new file mode 100755 index 0000000..7e9eebf --- /dev/null +++ b/perestroika/publisher.v5/publish-deb-binaries.sh @@ -0,0 +1,208 @@ +#!/bin/bash -ex + +[ -f ".publisher-defaults-deb" ] && source .publisher-defaults-deb +source $(dirname $(readlink -e $0))/functions/publish-functions.sh +source $(dirname $(readlink -e $0))/functions/locking.sh + +main() { + local SIGN_STRING="" + check-gpg && SIGN_STRING="true" + + ## Download sources from worker + [ -d $TMP_DIR ] && rm -rf $TMP_DIR + mkdir -p $TMP_DIR + rsync -avPzt \ + -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ${SSH_OPTS}" \ + ${SSH_USER}${BUILD_HOST}:${PKG_PATH}/ ${TMP_DIR}/ || error "Can't download packages" + + ## Resign source package + ## FixMe: disabled for discussion: does it really need to sign + #[ -n "${SIGN_STRING}" ] && \ + # for _dscfile in $(find ${TMP_DIR} -name "*.dsc") ; do + # debsign -pgpg --re-sign -k${SIGKEYID} ${_dscfile} + # done + + # Create all repositories + + # Paths + local URL_PREFIX="" + if [ "${GERRIT_CHANGE_STATUS}" = "NEW" ] ; then + REPO_BASE_PATH=${REPO_BASE_PATH}/${REPO_REQUEST_PATH_PREFIX} + URL_PREFIX=${REPO_REQUEST_PATH_PREFIX} + if [ -n "${LP_BUG}" ] ; then + REPO_BASE_PATH=${REPO_BASE_PATH}${LP_BUG} + URL_PREFIX=${URL_PREFIX}${LP_BUG}/ + else + REPO_BASE_PATH=${REPO_BASE_PATH}${REQUEST_NUM} + URL_PREFIX=${URL_PREFIX}${REQUEST_NUM}/ + fi + fi + + # Repos + for repo_path in ${DEB_REPO_PATH} ; do + local LOCAL_REPO_PATH=${REPO_BASE_PATH}/${repo_path} + local DBDIR="+b/db" + local CONFIGDIR="${LOCAL_REPO_PATH}/conf" + local DISTDIR="${LOCAL_REPO_PATH}/public/dists/" + local OUTDIR="+b/public/" + if [ ! -d "${CONFIGDIR}" ] ; then + mkdir -p ${CONFIGDIR} + job_lock ${CONFIGDIR}.lock wait 3600 + for dist_name in ${DEB_DIST_NAME} ${DEB_PROPOSED_DIST_NAME} ${DEB_UPDATES_DIST_NAME} \ + ${DEB_SECURITY_DIST_NAME} ${DEB_HOLDBACK_DIST_NAME} ; do + cat >> ${CONFIGDIR}/distributions <<- EOF + Origin: ${ORIGIN} + Label: ${DEB_DIST_NAME} + Suite: ${dist_name} + Codename: ${dist_name} + Version: ${PRODUCT_VERSION} + Architectures: amd64 i386 source + Components: main restricted + UDebComponents: main restricted + Contents: . .gz .bz2 + + EOF + + reprepro --basedir ${LOCAL_REPO_PATH} --dbdir ${DBDIR} \ + --outdir ${OUTDIR} --distdir ${DISTDIR} --confdir ${CONFIGDIR} \ + export ${dist_name} + # Fix Codename field + local release_file="${DISTDIR}/${dist_name}/Release" + sed "s|^Codename:.*$|Codename: ${DEB_DIST_NAME}|" \ + -i ${release_file} + rm -f ${release_file}.gpg + # ReSign Release file + [ -n "${SIGN_STRING}" ] \ + && gpg --sign --local-user ${SIGKEYID} -ba \ + -o ${release_file}.gpg ${release_file} + done + job_lock ${CONFIGDIR}.lock unset + fi + done + + DEB_BASE_DIST_NAME=${DEB_DIST_NAME} + + [ -z "${DEB_UPDATES_DIST_NAME}" ] && DEB_UPDATES_DIST_NAME=${DEB_DIST_NAME} + [ -z "${DEB_PROPOSED_DIST_NAME}" ] && DEB_PROPOSED_DIST_NAME=${DEB_DIST_NAME} + [ -z "${DEB_SECURITY_DIST_NAME}" ] && DEB_SECURITY_DIST_NAME=${DEB_DIST_NAME} + [ -z "${DEB_HOLDBACK_DIST_NAME}" ] && DEB_HOLDBACK_DIST_NAME=${DEB_DIST_NAME} + [ -z "${DEB_UPDATES_COMPONENT}" ] && DEB_UPDATES_COMPONENT=${DEB_COMPONENT} + [ -z "${DEB_PROPOSED_COMPONENT}" ] && DEB_PROPOSED_COMPONENT=${DEB_COMPONENT} + [ -z "${DEB_SECURITY_COMPONENT}" ] && DEB_SECURITY_COMPONENT=${DEB_COMPONENT} + [ -z "${DEB_HOLDBACK_COMPONENT}" ] && DEB_HOLDBACK_COMPONENT=${DEB_COMPONENT} + + if [ "${IS_UPDATES}" = 'true' ] ; then + DEB_DIST_NAME=${DEB_PROPOSED_DIST_NAME} + DEB_COMPONENT=${DEB_PROPOSED_COMPONENT} + fi + if [ "${IS_HOLDBACK}" = 'true' ] ; then + DEB_DIST_NAME=${DEB_HOLDBACK_DIST_NAME} + DEB_COMPONENT=${DEB_HOLDBACK_COMPONENT} + fi + if [ "${IS_SECURITY}" = 'true' ] ; then + DEB_DIST_NAME=${DEB_SECURITY_DIST_NAME} + DEB_COMPONENT=${DEB_SECURITY_COMPONENT} + fi + + [ -z "${DEB_COMPONENT}" ] && local DEB_COMPONENT=main + [ "${IS_RESTRICTED}" = 'true' ] && DEB_COMPONENT=restricted + + local LOCAL_REPO_PATH=${REPO_BASE_PATH}/${DEB_REPO_PATH} + local CONFIGDIR="${LOCAL_REPO_PATH}/conf" + local DBDIR="+b/db" + local DISTDIR="${LOCAL_REPO_PATH}/public/dists/" + local OUTDIR="${LOCAL_REPO_PATH}/public/" + local REPREPRO_OPTS="--verbose --basedir ${LOCAL_REPO_PATH} --dbdir ${DBDIR} \ + --outdir ${OUTDIR} --distdir ${DISTDIR} --confdir ${CONFIGDIR}" + local REPREPRO_COMP_OPTS="${REPREPRO_OPTS} --component ${DEB_COMPONENT}" + + # Parse incoming files + local BINDEBLIST="" + local BINDEBNAMES="" + local BINUDEBLIST="" + local BINSRCLIST="" + for binary in ${TMP_DIR}/* ; do + case ${binary##*.} in + deb) BINDEBLIST="${BINDEBLIST} ${binary}" + BINDEBNAMES="${BINDEBNAMES} ${binary##*/}" + ;; + udeb) BINUDEBLIST="${BINUDEBLIST} ${binary}" ;; + dsc) BINSRCLIST="${binary}" ;; + esac + done + + job_lock ${CONFIGDIR}.lock wait 3600 + + local SRC_NAME=$(awk '/^Source:/ {print $2}' ${BINSRCLIST}) + local NEW_VERSION=$(awk '/^Version:/ {print $2}' ${BINSRCLIST} | head -n 1) + local OLD_VERSION=$(reprepro ${REPREPRO_OPTS} --list-format '${version}\n' \ + listfilter ${DEB_DIST_NAME} "Package (==${SRC_NAME})" | sort -u | head -n 1) + [ "${OLD_VERSION}" == "" ] && OLD_VERSION=none + + # Remove existing packages for requests-on-review + # TODO: Get rid of removing. Just increase version properly + if [ "${GERRIT_CHANGE_STATUS}" = "NEW" ] ; then + reprepro ${REPREPRO_OPTS} removesrc ${DEB_DIST_NAME} ${SRC_NAME} ${OLD_VERSION} || : + fi + # Add .deb binaries + if [ "${BINDEBLIST}" != "" ]; then + reprepro ${REPREPRO_COMP_OPTS} includedeb ${DEB_DIST_NAME} ${BINDEBLIST} \ + || error "Can't include packages" + fi + # Add .udeb binaries + if [ "${BINUDEBLIST}" != "" ]; then + reprepro ${REPREPRO_COMP_OPTS} includeudeb ${DEB_DIST_NAME} ${BINUDEBLIST} \ + || error "Can't include packages" + fi + + # Replace sources + # TODO: Get rid of replacing. Just increase version properly + if [ "${BINSRCLIST}" != "" ]; then + reprepro ${REPREPRO_COMP_OPTS} --architecture source \ + remove ${DEB_DIST_NAME} ${SRC_NAME} || : + reprepro ${REPREPRO_COMP_OPTS} includedsc ${DEB_DIST_NAME} ${BINSRCLIST} \ + || error "Can't include packages" + fi + # Cleanup files from previous version + [ "${OLD_VERSION}" != "${NEW_VERSION}" ] \ + && reprepro ${REPREPRO_OPTS} removesrc ${DEB_DIST_NAME} ${SRC_NAME} ${OLD_VERSION} + + # Fix Codename field + local release_file="${DISTDIR}/${DEB_DIST_NAME}/Release" + sed "s|^Codename:.*$|Codename: ${DEB_BASE_DIST_NAME}|" -i ${release_file} + + # Resign Release file + rm -f ${release_file}.gpg + local pub_key_file="${LOCAL_REPO_PATH}/public/archive-${PROJECT_NAME}${PROJECT_VERSION}.key" + if [ -n "${SIGN_STRING}" ] ; then + gpg --sign --local-user ${SIGKEYID} -ba -o ${release_file}.gpg ${release_file} + [ ! -f "${pub_key_file}" ] && touch ${pub_key_file} + gpg -o ${pub_key_file}.tmp --armor --export ${SIGKEYID} + if diff -q ${pub_key_file} ${pub_key_file}.tmp &>/dev/null ; then + rm ${pub_key_file}.tmp + else + mv ${pub_key_file}.tmp ${pub_key_file} + fi + else + rm -f ${pub_key_file} + fi + + sync-repo ${OUTDIR} ${DEB_REPO_PATH} ${REPO_REQUEST_PATH_PREFIX} ${REQUEST_NUM} ${LP_BUG} + job_lock ${CONFIGDIR}.lock unset + + rm -f ${WRK_DIR}/deb.publish.setenvfile + cat > ${WRK_DIR}/deb.publish.setenvfile<<-EOF + DEB_PUBLISH_SUCCEEDED=true + DEB_DISTRO=${DIST} + DEB_REPO_URL="http://${REMOTE_REPO_HOST}/${URL_PREFIX}${DEB_REPO_PATH} ${DEB_DIST_NAME} ${DEB_COMPONENT}" + DEB_PACKAGENAME=${SRC_NAME} + DEB_VERSION=${NEW_VERSION} + DEB_BINARIES=$(cat ${BINSRCLIST} | grep ^Binary | sed 's|^Binary:||; s| ||g') + DEB_CHANGE_REVISION=${GERRIT_PATCHSET_REVISION} + LP_BUG=${LP_BUG} + EOF +} + +main "$@" + +exit 0 diff --git a/perestroika/publisher.v5/publish-rpm-binaries.sh b/perestroika/publisher.v5/publish-rpm-binaries.sh new file mode 100755 index 0000000..6017a80 --- /dev/null +++ b/perestroika/publisher.v5/publish-rpm-binaries.sh @@ -0,0 +1,233 @@ +#!/bin/bash -ex + +[ -f ".publisher-defaults-rpm" ] && source .publisher-defaults-rpm +source $(dirname $(readlink -e $0))/functions/publish-functions.sh +source $(dirname $(readlink -e $0))/functions/locking.sh + +[ -z "${DEFAULTCOMPSXML}" ] && DEFAULTCOMPSXML=http://mirror.fuel-infra.org/fwm/6.0/centos/os/x86_64/comps.xml + +main() { + if [ -n "${SIGKEYID}" ] ; then + check-gpg || : + gpg --export -a ${SIGKEYID} > RPM-GPG-KEY + if [ $(rpm -qa | grep gpg-pubkey | grep -ci ${SIGKEYID}) -eq 0 ]; then + rpm --import RPM-GPG-KEY + fi + fi + + # Get built binaries + [ -d ${TMP_DIR} ] && rm -rf ${TMP_DIR} + mkdir -p ${TMP_DIR} + rsync -avPzt -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ${SSH_OPTS}" \ + ${SSH_USER}${BUILD_HOST}:${PKG_PATH}/ ${TMP_DIR}/ || error "Can't download packages" + [ $(ls -1 ${TMP_DIR}/ | wc -l) -eq 0 ] && error "Can't download packages" + + ## Prepare repository + local URL_PREFIX='' + if [ "${GERRIT_CHANGE_STATUS}" == "NEW" ] ; then + REPO_BASE_PATH=${REPO_BASE_PATH}/${REPO_REQUEST_PATH_PREFIX} + URL_PREFIX=${REPO_REQUEST_PATH_PREFIX} + if [ -n "${LP_BUG}" ] ; then + REPO_BASE_PATH=${REPO_BASE_PATH}${LP_BUG} + URL_PREFIX=${URL_PREFIX}${LP_BUG}/ + else + REPO_BASE_PATH=${REPO_BASE_PATH}${REQUEST_NUM} + URL_PREFIX=${URL_PREFIX}${REQUEST_NUM}/ + fi + fi + + # Create all repositories + for repo_path in ${RPM_OS_REPO_PATH} ${RPM_PROPOSED_REPO_PATH} ${RPM_UPDATES_REPO_PATH} ${RPM_SECURITY_REPO_PATH} ${RPM_HOLDBACK_REPO_PATH} ; do + local LOCAL_REPO_PATH=${REPO_BASE_PATH}/${repo_path} + if [ ! -d "${LOCAL_REPO_PATH}" ] ; then + mkdir -p ${LOCAL_REPO_PATH}/{x86_64/Packages,Source/SPackages,x86_64/repodata} + job_lock ${LOCAL_REPO_PATH}.lock wait 3600 + createrepo --pretty --database --update -o ${LOCAL_REPO_PATH}/x86_64/ ${LOCAL_REPO_PATH}/x86_64 + createrepo --pretty --database --update -o ${LOCAL_REPO_PATH}/Source/ ${LOCAL_REPO_PATH}/Source + job_lock ${LOCAL_REPO_PATH}.lock unset + fi + done + + [ -z "${RPM_UPDATES_REPO_PATH}" ] && RPM_UPDATES_REPO_PATH=${RPM_OS_REPO_PATH} + [ -z "${RPM_PROPOSED_REPO_PATH}" ] && RPM_PROPOSED_REPO_PATH=${RPM_OS_REPO_PATH} + [ -z "${RPM_SECURITY_REPO_PATH}" ] && RPM_SECURITY_REPO_PATH=${RPM_OS_REPO_PATH} + [ -z "${RPM_HOLDBACK_REPO_PATH}" ] && RPM_HOLDBACK_REPO_PATH=${RPM_OS_REPO_PATH} + + RPM_REPO_PATH=${RPM_OS_REPO_PATH} + [ "${IS_UPDATES}" == 'true' ] && RPM_REPO_PATH=${RPM_PROPOSED_REPO_PATH} + [ "${IS_HOLDBACK}" == 'true' ] && RPM_REPO_PATH=${RPM_HOLDBACK_REPO_PATH} + [ "${IS_SECURITY}" == 'true' ] && RPM_REPO_PATH=${RPM_SECURITY_REPO_PATH} + + local LOCAL_REPO_PATH=${REPO_BASE_PATH}/${RPM_REPO_PATH} + + # Parse binary list + local BINRPMLIST="" + local BINSRCLIST="" + local BINSRCNAMES="" + local BINRPMNAMES="" + for binary in ${TMP_DIR}/* ; do + if [ "${binary:(-7)}" == "src.rpm" ] ; then + BINSRCLIST="${binary}" + BINSRCNAMES="${binary##*/}" + elif [ "${binary##*.}" == "rpm" ]; then + BINRPMLIST="${BINRPMLIST} ${binary}" + BINRPMNAMES="${BINRPMNAMES} ${binary##*/}" + fi + done + BINNAMES="${BINSRCNAMES} ${BINRPMNAMES}" + local PACKAGENAMES="" + + # Get existing srpm filename + local SRPM_NAME=$(rpm -qp --queryformat "%{NAME}" ${BINSRCLIST}) + local _repoid_source=$(mktemp -u XXXXXXXX) + local repoquery_opts="--repofrompath=${_repoid_source},file://${LOCAL_REPO_PATH}/Source/ --repoid=${_repoid_source}" + local EXIST_SRPM_FILE=$(repoquery ${repoquery_opts} --archlist=src --location ${SRPM_NAME}) + local EXIST_SRPM_FILE=${EXIST_SRPM_FILE##*/} + # Get existing rpm files + local repoquerysrpm_py="$(dirname $(readlink -e $0))/repoquerysrpm.py" + local EXIST_RPM_FILES=$(python ${repoquerysrpm_py} --srpm=${EXIST_SRPM_FILE} --path=${LOCAL_REPO_PATH}/x86_64/ | awk -F'/' '{print $NF}') + # Cleanup `repoquery` data + find /var/tmp/yum-${USER}-* -type d -name $_repoid_source -exec rm -rf {} \; 2>/dev/null || : + + job_lock ${LOCAL_REPO_PATH}.lock wait 3600 + # Sign and publish binaries + for binary in ${BINRPMLIST} ${BINSRCLIST} ; do + local PACKAGEFOLDER=x86_64/Packages + [ "${binary:(-7)}" == "src.rpm" ] && PACKAGEFOLDER=Source/SPackages + + # Get package info + local NEWBINDATA=$(rpm -qp --queryformat "%{EPOCH} %{NAME} %{VERSION} %{RELEASE} %{SHA1HEADER}\n" ${binary} 2>/dev/null) + local NEWBINEPOCH=$(echo ${NEWBINDATA} | cut -d' ' -f1) + [ "${NEWBINEPOCH}" == "(none)" ] && NEWBINEPOCH='0' + local BINNAME=$(echo ${NEWBINDATA} | cut -d' ' -f2) + [ "${binary:(-7)}" != "src.rpm" ] && local PACKAGENAMES="${PACKAGENAMES} ${BINNAME}" + local NEWBINVERSION=$(echo ${NEWBINDATA} | cut -d' ' -f3) + local NEWBINRELEASE=$(echo ${NEWBINDATA} | cut -d' ' -f4) + local NEWBINSHA=$(echo ${NEWBINDATA} | cut -d' ' -f5) + # EXISTBINDATA format pkg-name-epoch:version-release.arch (NEVRA) + local _repoid_os=$(mktemp -u XXXXXXXX) + local _repoid_updates=$(mktemp -u XXXXXXXX) + local _repoid_proposed=$(mktemp -u XXXXXXXX) + local _repoid_holdback=$(mktemp -u XXXXXXXX) + local _repoid_security=$(mktemp -u XXXXXXXX) + local repoquery_cmd="repoquery --repofrompath=${_repoid_os},file://${REPO_BASE_PATH}/${RPM_OS_REPO_PATH}/${PACKAGEFOLDER%/*} --repoid=${_repoid_os}" + local repoquery_cmd="${repoquery_cmd} --repofrompath=${_repoid_updates},file://${REPO_BASE_PATH}/${RPM_UPDATES_REPO_PATH}/${PACKAGEFOLDER%/*} --repoid=${_repoid_updates}" + local repoquery_cmd="${repoquery_cmd} --repofrompath=${_repoid_proposed},file://${REPO_BASE_PATH}/${RPM_PROPOSED_REPO_PATH}/${PACKAGEFOLDER%/*} --repoid=${_repoid_proposed}" + local repoquery_cmd="${repoquery_cmd} --repofrompath=${_repoid_holdback},file://${REPO_BASE_PATH}/${RPM_HOLDBACK_REPO_PATH}/${PACKAGEFOLDER%/*} --repoid=${_repoid_holdback}" + local repoquery_cmd="${repoquery_cmd} --repofrompath=${_repoid_security},file://${REPO_BASE_PATH}/${RPM_SECURITY_REPO_PATH}/${PACKAGEFOLDER%/*} --repoid=${_repoid_security}" + [ "${binary:(-7)}" == "src.rpm" ] && repoquery_cmd="${repoquery_cmd} --archlist=src" + local EXISTBINDATA=$(${repoquery_cmd} ${BINNAME} 2>/dev/null) + + # Cleanup `repoquery` data + for _repoid in $_repoid_os $_repoid_updates $_repoid_proposed $_repoid_holdback $_repoid_security ; do + find /var/tmp/yum-${USER}-* -type d -name $_repoid -exec rm -rf {} \; 2>/dev/null || : + done + # Get arch + local EXISTBINARCH=${EXISTBINDATA##*.} + # Skip arch + local EXISTBINDATA=${EXISTBINDATA%.*} + # Get epoch + local EXISTBINEPOCH=$(echo ${EXISTBINDATA} | cut -d':' -f1 | awk -F'-' '{print $NF}') + # Skip "pkg-name-epoch:" + local EXISTBINDATA=${EXISTBINDATA#*:} + # Get version + local EXISTBINVERSION=${EXISTBINDATA%%-*} + # Get release + local EXISTBINRELEASE=${EXISTBINDATA#*-} + ## FixMe: Improve packages removing + # Remove existing packages from repo (for new change requests only) + [ "${GERRIT_CHANGE_STATUS}" == "NEW" ] && \ + find ${LOCAL_REPO_PATH} -name "${BINNAME}-${EXISTBINVERSION}-${EXISTBINRELEASE}.${EXISTBINARCH}*" -exec rm -f {} \; + # Compare versions of new and existring packages + local SKIPPACKAGE=0 + if [ ! -z "${EXISTBINVERSION}" ] && [ "${GERRIT_CHANGE_STATUS}" != "NEW" ]; then + ############################################################ + ## Comparing versions before including package to the repo + ## + CMPVER=$(python <(cat <<-HERE + from rpmUtils import miscutils + print miscutils.compareEVR(("${EXISTBINEPOCH}", "${EXISTBINVERSION}", "${EXISTBINRELEASE}"), + ("${NEWBINEPOCH}", "${NEWBINVERSION}", "${NEWBINRELEASE}")) + HERE + )) + # Results: + # 1 - EXISTBIN is newer than NEWBIN + # 0 - EXISTBIN and NEWBIN have the same version + # -1 - EXISTBIN is older than NEWBIN + case ${CMPVER} in + 1) error "Can't publish ${binary#*/}. Existing ${BINNAME}-${EXISTBINEPOCH}:${EXISTBINVERSION}-${EXISTBINRELEASE} has newer version" ;; + 0) # Check sha for identical package names + EXISTRPMFILE=$(${repoquery_cmd} --location ${BINNAME}) + EXISTBINSHA=$(rpm -qp --queryformat "%{SHA1HEADER}" ${EXISTRPMFILE}) + if [ "${NEWBINSHA}" == "${EXISTBINSHA}" ]; then + SKIPPACKAGE=1 + echo "Skipping including of ${binary}. Existing ${BINNAME}-${EXISTBINEPOCH}:${EXISTBINVERSION}-${EXISTBINRELEASE} has the same version and checksum" + else + error "Can't publish ${binary#*/}. Existing ${BINNAME}-${EXISTBINEPOCH}:${EXISTBINVERSION}-${EXISTBINRELEASE} has the same version but different checksum" + fi + ;; + *) : ;; + esac + ## + ############################################################ + fi + ############ + ## Signing + ## + if [ -n "${SIGKEYID}" ] ; then + # rpmsign requires pass phrase. use `expect` to skip it + LANG=C expect </dev/null + done + rm -f $(repomanage --keep=1 --old ${LOCAL_REPO_PATH}/x86_64) + rm -f $(repomanage --keep=1 --old ${LOCAL_REPO_PATH}/Source) + # Update and sign repository metadata + [ ! -e ${LOCAL_REPO_PATH}/comps.xml ] && wget ${DEFAULTCOMPSXML} -O ${LOCAL_REPO_PATH}/comps.xml + createrepo --pretty --database --update -g ${LOCAL_REPO_PATH}/comps.xml -o ${LOCAL_REPO_PATH}/x86_64/ ${LOCAL_REPO_PATH}/x86_64 + createrepo --pretty --database --update -o ${LOCAL_REPO_PATH}/Source/ ${LOCAL_REPO_PATH}/Source + if [ -n "${SIGKEYID}" ] ; then + rm -f ${LOCAL_REPO_PATH}/x86_64/repodata/repomd.xml.asc + rm -f ${LOCAL_REPO_PATH}/Source/repodata/repomd.xml.asc + gpg --armor --local-user ${SIGKEYID} --detach-sign ${LOCAL_REPO_PATH}/x86_64/repodata/repomd.xml + gpg --armor --local-user ${SIGKEYID} --detach-sign ${LOCAL_REPO_PATH}/Source/repodata/repomd.xml + [ -f "RPM-GPG-KEY" ] && cp RPM-GPG-KEY ${LOCAL_REPO_PATH}/RPM-GPG-KEY-${PROJECT_NAME}${PROJECT_VERSION} + fi + + # Sync repo to remote host + sync-repo ${LOCAL_REPO_PATH}/ ${RPM_REPO_PATH} ${REPO_REQUEST_PATH_PREFIX} ${REQUEST_NUM} ${LP_BUG} + job_lock ${LOCAL_REPO_PATH}.lock unset + + rm -f ${WRK_DIR}/rpm.publish.setenvfile + + cat > ${WRK_DIR}/rpm.publish.setenvfile <<-EOF + RPM_PUBLISH_SUCCEEDED=true + RPM_DISTRO=${DIST} + RPM_VERSION=${NEWBINEPOCH}:${NEWBINVERSION}-${NEWBINRELEASE} + RPM_REPO_URL=http://${REMOTE_REPO_HOST}/${URL_PREFIX}${RPM_REPO_PATH}/x86_64 + RPM_BINARIES=$(echo ${PACKAGENAMES} | sed 's|^ ||; s| |,|g') + RPM_CHANGE_REVISION=${GERRIT_PATCHSET_REVISION} + LP_BUG=${LP_BUG} + EOF +} + +main "$@" + +exit 0 diff --git a/perestroika/publisher.v5/repoquerysrpm.py b/perestroika/publisher.v5/repoquerysrpm.py new file mode 100755 index 0000000..9ebefa7 --- /dev/null +++ b/perestroika/publisher.v5/repoquerysrpm.py @@ -0,0 +1,66 @@ +#!/usr/bin/python + +import argparse +import gzip +import os +import sys + +from lxml import etree as ET + + +def main(): + parser = argparse.ArgumentParser() + + parser.add_argument( + '-s', '--srpm', dest='srpm', action='store', type=str, + help='srpm', required=True, default='none' + ) + + parser.add_argument( + '-p', '--path', dest='path', action='store', type=str, + help='path', required=True, default='.' + ) + + params, other_params = parser.parse_known_args() + + repomdpath = os.path.join(params.path, 'repodata', 'repomd.xml') + + tree = ET.parse(repomdpath) + repomd = tree.getroot() + + xmlpath = {} + for data in repomd.findall(ET.QName(repomd.nsmap[None], 'data')): + filetype = data.attrib['type'] + xmlpath[filetype] = data.find( + ET.QName(repomd.nsmap[None], 'location')).attrib['href'] + + primaryfile = os.path.join(params.path, xmlpath['primary']) + + with gzip.open(primaryfile, 'rb') as f: + primary_content = f.read() + + primary = ET.fromstring(primary_content) + + filtered = primary.xpath('//rpm:sourcerpm[text()="' + params.srpm + '"]', + namespaces={'rpm': primary.nsmap['rpm']}) + + for item in filtered: + name = item.getparent().getparent().find( + ET.QName(primary.nsmap[None], 'name')).text + arch = item.getparent().getparent().find( + ET.QName(primary.nsmap[None], 'arch')).text + epoch = item.getparent().getparent().find( + ET.QName(primary.nsmap[None], 'version')).attrib['epoch'] + ver = item.getparent().getparent().find( + ET.QName(primary.nsmap[None], 'version')).attrib['ver'] + rel = item.getparent().getparent().find( + ET.QName(primary.nsmap[None], 'version')).attrib['rel'] + location = item.getparent().getparent().find( + ET.QName(primary.nsmap[None], 'location')).attrib['href'] + print '{name} {epoch} {ver} {rel} {arch} {location}'.format( + name=name, epoch=epoch, ver=ver, rel=rel, + arch=arch, location=location) + + +if __name__ == "__main__": + main()