From 17b2bda1a05304025f2a468a17dae70532012ca3 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Sun, 1 Apr 2012 21:02:25 -0700 Subject: [PATCH] Added slave scripts to puppet. Added all of the files from slave_scripts to jenkins_slave. Includes the changes I submitted in https://review.openstack.org/6063 Changed the jobs to use these rather than the ones from openstack-ci. Change-Id: I4bd9e78b3d9c0cbbc6a0b17fc95c60458b06a0f7 --- .../templates/builder_copy_bundle.xml.erb | 2 +- .../templates/builder_coverage.xml.erb | 2 +- .../templates/builder_docs.xml.erb | 2 +- .../templates/builder_gerrit_git_prep.xml.erb | 2 +- .../templates/builder_ppa.xml.erb | 2 +- .../templates/builder_python26.xml.erb | 2 +- .../templates/builder_python27.xml.erb | 2 +- .../templates/builder_tarball.xml.erb | 2 +- .../templates/builder_venv.xml.erb | 2 +- .../slave_scripts/baremetal-archive-logs.sh | 5 ++ .../files/slave_scripts/baremetal-deploy.sh | 44 ++++++++++ .../slave_scripts/baremetal-os-install.sh | 22 +++++ .../files/slave_scripts/build-bundle.sh | 29 +++++++ .../files/slave_scripts/build-venv.sh | 32 +++++++ .../files/slave_scripts/copy-bundle.sh | 15 ++++ .../files/slave_scripts/copy-venv.sh | 43 ++++++++++ .../files/slave_scripts/create-ppa-package.sh | 78 +++++++++++++++++ .../files/slave_scripts/create-tarball.sh | 84 +++++++++++++++++++ .../files/slave_scripts/gerrit-git-prep.sh | 35 ++++++++ .../files/slave_scripts/lvm-kexec-reset.sh | 7 ++ .../jenkins_slave/files/slave_scripts/ping.py | 11 +++ .../slave_scripts/propose_translations.sh | 32 +++++++ .../files/slave_scripts/run-cover.sh | 25 ++++++ .../files/slave_scripts/run-docs.sh | 27 ++++++ .../files/slave_scripts/run-tox.sh | 38 +++++++++ .../files/slave_scripts/update-pip-cache.sh | 10 +++ .../files/slave_scripts/wait_for_nova.sh | 12 +++ .../files/slave_scripts/wait_for_puppet.sh | 21 +++++ .../jenkins_slave/manifests/jenkinsuser.pp | 52 ++++++++++++ 29 files changed, 631 insertions(+), 9 deletions(-) create mode 100755 modules/jenkins_slave/files/slave_scripts/baremetal-archive-logs.sh create mode 100755 modules/jenkins_slave/files/slave_scripts/baremetal-deploy.sh create mode 100755 modules/jenkins_slave/files/slave_scripts/baremetal-os-install.sh create mode 100755 modules/jenkins_slave/files/slave_scripts/build-bundle.sh create mode 100755 modules/jenkins_slave/files/slave_scripts/build-venv.sh create mode 100755 modules/jenkins_slave/files/slave_scripts/copy-bundle.sh create mode 100755 modules/jenkins_slave/files/slave_scripts/copy-venv.sh create mode 100755 modules/jenkins_slave/files/slave_scripts/create-ppa-package.sh create mode 100755 modules/jenkins_slave/files/slave_scripts/create-tarball.sh create mode 100755 modules/jenkins_slave/files/slave_scripts/gerrit-git-prep.sh create mode 100755 modules/jenkins_slave/files/slave_scripts/lvm-kexec-reset.sh create mode 100755 modules/jenkins_slave/files/slave_scripts/ping.py create mode 100755 modules/jenkins_slave/files/slave_scripts/propose_translations.sh create mode 100755 modules/jenkins_slave/files/slave_scripts/run-cover.sh create mode 100755 modules/jenkins_slave/files/slave_scripts/run-docs.sh create mode 100755 modules/jenkins_slave/files/slave_scripts/run-tox.sh create mode 100755 modules/jenkins_slave/files/slave_scripts/update-pip-cache.sh create mode 100755 modules/jenkins_slave/files/slave_scripts/wait_for_nova.sh create mode 100755 modules/jenkins_slave/files/slave_scripts/wait_for_puppet.sh diff --git a/modules/jenkins_jobs/templates/builder_copy_bundle.xml.erb b/modules/jenkins_jobs/templates/builder_copy_bundle.xml.erb index 850d6762bd..72e594d578 100644 --- a/modules/jenkins_jobs/templates/builder_copy_bundle.xml.erb +++ b/modules/jenkins_jobs/templates/builder_copy_bundle.xml.erb @@ -6,6 +6,6 @@ - bash -x ~/openstack-ci/slave_scripts/copy-bundle.sh + /usr/local/jenkins/slave_scripts/copy-bundle.sh diff --git a/modules/jenkins_jobs/templates/builder_coverage.xml.erb b/modules/jenkins_jobs/templates/builder_coverage.xml.erb index b60713483c..b6728aa623 100644 --- a/modules/jenkins_jobs/templates/builder_coverage.xml.erb +++ b/modules/jenkins_jobs/templates/builder_coverage.xml.erb @@ -1,4 +1,4 @@ - bash -x ~/openstack-ci/slave_scripts/run-cover.sh + /usr/local/jenkins/slave_scripts/run-cover.sh diff --git a/modules/jenkins_jobs/templates/builder_docs.xml.erb b/modules/jenkins_jobs/templates/builder_docs.xml.erb index 24d7db9b0d..8e27abc875 100644 --- a/modules/jenkins_jobs/templates/builder_docs.xml.erb +++ b/modules/jenkins_jobs/templates/builder_docs.xml.erb @@ -1,4 +1,4 @@ - bash -x ~/openstack-ci/slave_scripts/run-docs.sh + /usr/local/jenkins/slave_scripts/run-docs.sh diff --git a/modules/jenkins_jobs/templates/builder_gerrit_git_prep.xml.erb b/modules/jenkins_jobs/templates/builder_gerrit_git_prep.xml.erb index 3409055992..f0007e4af5 100644 --- a/modules/jenkins_jobs/templates/builder_gerrit_git_prep.xml.erb +++ b/modules/jenkins_jobs/templates/builder_gerrit_git_prep.xml.erb @@ -1,3 +1,3 @@ - bash -x ~/openstack-ci/slave_scripts/gerrit-git-prep.sh <%= site %> + /usr/local/jenkins/slave_scripts/gerrit-git-prep.sh <%= site %> diff --git a/modules/jenkins_jobs/templates/builder_ppa.xml.erb b/modules/jenkins_jobs/templates/builder_ppa.xml.erb index e67079041a..ca4fec9a1e 100644 --- a/modules/jenkins_jobs/templates/builder_ppa.xml.erb +++ b/modules/jenkins_jobs/templates/builder_ppa.xml.erb @@ -17,6 +17,6 @@ mkdir build #export DO_UPLOAD="no" export PROJECT="<%= project %>" export GERRIT_REFNAME=$BRANCH -/bin/bash -xe ~/openstack-ci/slave_scripts/create-ppa-package.sh +/usr/local/jenkins/slave_scripts/create-ppa-package.sh diff --git a/modules/jenkins_jobs/templates/builder_python26.xml.erb b/modules/jenkins_jobs/templates/builder_python26.xml.erb index a3bc6ca151..7cbb1af523 100644 --- a/modules/jenkins_jobs/templates/builder_python26.xml.erb +++ b/modules/jenkins_jobs/templates/builder_python26.xml.erb @@ -1,3 +1,3 @@ - bash -x ~/openstack-ci/slave_scripts/run-tox.sh 26 + /usr/local/jenkins/slave_scripts/run-tox.sh 26 diff --git a/modules/jenkins_jobs/templates/builder_python27.xml.erb b/modules/jenkins_jobs/templates/builder_python27.xml.erb index a3bc6ca151..7cbb1af523 100644 --- a/modules/jenkins_jobs/templates/builder_python27.xml.erb +++ b/modules/jenkins_jobs/templates/builder_python27.xml.erb @@ -1,3 +1,3 @@ - bash -x ~/openstack-ci/slave_scripts/run-tox.sh 26 + /usr/local/jenkins/slave_scripts/run-tox.sh 26 diff --git a/modules/jenkins_jobs/templates/builder_tarball.xml.erb b/modules/jenkins_jobs/templates/builder_tarball.xml.erb index 0291d8de23..72f551e21e 100644 --- a/modules/jenkins_jobs/templates/builder_tarball.xml.erb +++ b/modules/jenkins_jobs/templates/builder_tarball.xml.erb @@ -1,3 +1,3 @@ - bash -x ~/openstack-ci/slave_scripts/create-tarball.sh + /usr/local/jenkins/slave_scripts/create-tarball.sh diff --git a/modules/jenkins_jobs/templates/builder_venv.xml.erb b/modules/jenkins_jobs/templates/builder_venv.xml.erb index 459cc5bb6d..3c7611f97e 100644 --- a/modules/jenkins_jobs/templates/builder_venv.xml.erb +++ b/modules/jenkins_jobs/templates/builder_venv.xml.erb @@ -1,3 +1,3 @@ - bash -x ~/openstack-ci/slave_scripts/build-bundle.sh + /usr/local/jenkins/slave_scripts/build-bundle.sh diff --git a/modules/jenkins_slave/files/slave_scripts/baremetal-archive-logs.sh b/modules/jenkins_slave/files/slave_scripts/baremetal-archive-logs.sh new file mode 100755 index 0000000000..cf3159b868 --- /dev/null +++ b/modules/jenkins_slave/files/slave_scripts/baremetal-archive-logs.sh @@ -0,0 +1,5 @@ +#!/bin/bash -xe + +for host in $HEAD_HOST ${COMPUTE_HOSTS//,/ }; do + cp /var/log/orchestra/rsyslog/$host/syslog $WORKSPACE/logs/$host-syslog.txt +done diff --git a/modules/jenkins_slave/files/slave_scripts/baremetal-deploy.sh b/modules/jenkins_slave/files/slave_scripts/baremetal-deploy.sh new file mode 100755 index 0000000000..0421529e3d --- /dev/null +++ b/modules/jenkins_slave/files/slave_scripts/baremetal-deploy.sh @@ -0,0 +1,44 @@ +#!/bin/bash -xe + +WORKSPACE=`pwd` +mkdir -p logs +rm -f logs/* +cd `dirname "$0"` + +echo "Jenkins: resetting hosts..." +for host in $HEAD_HOST ${COMPUTE_HOSTS//,/ }; do + scp lvm-kexec-reset.sh root@$host:/var/tmp/ + ssh root@$host /var/tmp/lvm-kexec-reset.sh + sudo rm -f /var/log/orchestra/rsyslog/$host/syslog +done + +# Have rsyslog reopen log files we rm'd from under it +sudo restart rsyslog + +# wait for the host to come up (2 ping responses or timeout after 5 minutes) +echo "Jenkins: Waiting for head host to return after reset..." +sleep 10 +if ! timeout 300 ./ping.py $HEAD_HOST; then + echo "Jenkins: ERROR: Head node did not come back up after reset" + exit 1 +fi + +echo "Jenkins: Pre-populating PIP cache" +for host in $HEAD_HOST ${COMPUTE_HOSTS//,/ }; do + scp -r ~/cache/pip root@$host:/var/cache/pip +done + +echo "Jenkins: Caching images." +cd ~/devstack +source stackrc +for image_url in ${IMAGE_URLS//,/ }; do + # Downloads the image (uec ami+aki style), then extracts it. + IMAGE_FNAME=`echo "$image_url" | python -c "import sys; print sys.stdin.read().split('/')[-1]"` + IMAGE_NAME=`echo "$IMAGE_FNAME" | python -c "import sys; print sys.stdin.read().split('.tar.gz')[0].split('.tgz')[0]"` + if [ ! -f files/$IMAGE_FNAME ]; then + wget -c $image_url -O files/$IMAGE_FNAME + fi +done + +echo "Jenkins: Executing build_bm_multi.sh." +./tools/build_bm_multi.sh diff --git a/modules/jenkins_slave/files/slave_scripts/baremetal-os-install.sh b/modules/jenkins_slave/files/slave_scripts/baremetal-os-install.sh new file mode 100755 index 0000000000..473f975da7 --- /dev/null +++ b/modules/jenkins_slave/files/slave_scripts/baremetal-os-install.sh @@ -0,0 +1,22 @@ +#!/bin/bash -xe + +set -x +sudo cobbler sync +sudo cobbler system edit --netboot-enabled=Y --name=baremetal1 +sudo cobbler system edit --netboot-enabled=Y --name=baremetal2 +sudo cobbler system edit --netboot-enabled=Y --name=baremetal3 +sudo cobbler system edit --netboot-enabled=Y --name=baremetal4 +sudo cobbler system edit --netboot-enabled=Y --name=baremetal5 +sudo cobbler system edit --netboot-enabled=Y --name=baremetal6 +sudo cobbler system edit --netboot-enabled=Y --name=baremetal7 +sudo cobbler system edit --netboot-enabled=Y --name=baremetal8 +sudo cobbler system edit --netboot-enabled=Y --name=baremetal9 +sudo cobbler system reboot --name=baremetal1 +sudo cobbler system reboot --name=baremetal2 +sudo cobbler system reboot --name=baremetal3 +sudo cobbler system reboot --name=baremetal4 +sudo cobbler system reboot --name=baremetal5 +sudo cobbler system reboot --name=baremetal6 +sudo cobbler system reboot --name=baremetal7 +sudo cobbler system reboot --name=baremetal8 +sudo cobbler system reboot --name=baremetal9 diff --git a/modules/jenkins_slave/files/slave_scripts/build-bundle.sh b/modules/jenkins_slave/files/slave_scripts/build-bundle.sh new file mode 100755 index 0000000000..b3b845467f --- /dev/null +++ b/modules/jenkins_slave/files/slave_scripts/build-bundle.sh @@ -0,0 +1,29 @@ +#!/bin/bash -xe + +# Make sure there is a location on this builder to cache pip downloads +mkdir -p ~/cache/pip +export PIP_DOWNLOAD_CACHE=~/cache/pip + +# Start with a clean slate +rm -fr jenkins_venvs +mkdir -p jenkins_venvs + +# Build a venv for every known branch +for branch in `git branch -r |grep "origin/"|grep -v HEAD|sed "s/origin\///"` +do + echo "Building venv for $branch" + git checkout $branch + mkdir -p jenkins_venvs/$branch + bundle=jenkins_venvs/$branch/.cache.bundle + + if [ -f tools/pip-requires ] ; then + if [ -f tools/test-requires ] ; then + pip bundle $bundle -r tools/pip-requires -r tools/test-requires + else + pip bundle $bundle -r tools/pip-requires + fi + else + pip bundle $bundle distribute openstack.nose_plugin + fi +done +git checkout master diff --git a/modules/jenkins_slave/files/slave_scripts/build-venv.sh b/modules/jenkins_slave/files/slave_scripts/build-venv.sh new file mode 100755 index 0000000000..77a9bcbc86 --- /dev/null +++ b/modules/jenkins_slave/files/slave_scripts/build-venv.sh @@ -0,0 +1,32 @@ +#!/bin/bash -xe + +# Make sure there is a location on this builder to cache pip downloads +mkdir -p ~/cache/pip +export PIP_DOWNLOAD_CACHE=~/cache/pip + +# Start with a clean slate +rm -fr jenkins_venvs +mkdir -p jenkins_venvs + +# Update the list of remote refs to pick up new branches +git remote update + +# Build a venv for every known branch +for branch in `git branch -r |grep "origin/"|grep -v HEAD|sed "s/origin\///"` +do + echo "Building venv for $branch" + git checkout $branch + mkdir -p jenkins_venvs/$branch + python tools/install_venv.py + virtualenv --relocatable .venv + if [ -e tools/test-requires ] + then + pip bundle .cache.bundle -r tools/pip-requires -r tools/test-requires + else + pip bundle .cache.bundle -r tools/pip-requires + fi + tar cvfz jenkins_venvs/$branch/venv.tgz .venv .cache.bundle + rm -fr .venv + mv .cache.bundle jenkins_venvs/$branch/ +done +git checkout master diff --git a/modules/jenkins_slave/files/slave_scripts/copy-bundle.sh b/modules/jenkins_slave/files/slave_scripts/copy-bundle.sh new file mode 100755 index 0000000000..a9de591b3a --- /dev/null +++ b/modules/jenkins_slave/files/slave_scripts/copy-bundle.sh @@ -0,0 +1,15 @@ +#!/bin/bash -xe + +# Support jobs, such as nova-docs, which are not yet triggered by gerrit +if [ "x$GERRIT_BRANCH" = "x" ] ; then + GERRIT_BRANCH=master +fi +mv jenkins_venvs/$GERRIT_BRANCH/.cache.bundle . +rm -fr jenkins_venvs + +if [ -f tools/test-requires -a \ + `git diff HEAD^1 tools/test-requires 2>/dev/null | wc -l` -gt 0 -o \ + `git diff HEAD^1 tools/pip-requires 2>/dev/null | wc -l` -gt 0 ] +then + rm .cache.bundle +fi diff --git a/modules/jenkins_slave/files/slave_scripts/copy-venv.sh b/modules/jenkins_slave/files/slave_scripts/copy-venv.sh new file mode 100755 index 0000000000..fd47ac8937 --- /dev/null +++ b/modules/jenkins_slave/files/slave_scripts/copy-venv.sh @@ -0,0 +1,43 @@ +#!/bin/bash -xe + +if [ -n "$GERRIT_REFNAME" ] +then + GERRIT_BRANCH=$GERRIT_REFNAME +fi + +# Support jobs, such as nova-docs, which are not yet triggered by gerrit +if [ "x$GERRIT_BRANCH" = "x" ] ; then + GERRIT_BRANCH=master +fi +mv jenkins_venvs/$GERRIT_BRANCH/venv.tgz . +rm -fr jenkins_venvs +tar xfz venv.tgz +rm venv.tgz +if [ -f .cache.bundle ] ; then + rm -rf .venv + virtualenv --no-site-packages .venv + pip install --upgrade -E .venv file://`pwd`/.cache.bundle + rm .cache.bundle +else + perl -MCwd -ple '$CWD=Cwd::abs_path();s,^VIRTUAL_ENV=.*,VIRTUAL_ENV="$CWD/.venv",' -i .venv/bin/activate +# This gets easy-install.pth as well as $PROJECT.pth. Examples for glance: +# easy-install.pth looks like: /home/jenkins/workspace/glance-venv/.venv/src/keystone +# $PROJECT.pth looks like: /home/jenkins/workspace/glance-venv + perl -MCwd -ple '$CWD=Cwd::abs_path();s,^/.*/workspace/[^/]+(/.*)$,$CWD$1,' -i .venv/lib/python2.7/site-packages/*.pth +# This is needed for the keystone install for glance + perl -MCwd -ple '$CWD=Cwd::abs_path();s,^/.*/.venv/src/(.*),$CWD/.venv/src/$1,' -i .venv/lib/python2.7/site-packages/*.egg-link +# This is needed for the keystone install for glance + perl -MCwd -ple '$CWD=Cwd::abs_path();s,/.*/.venv/src/(.*),$CWD/.venv/src/$1,' -i .venv/bin/* +fi +if [ -f tools/test-requires ] ; then + TEST_REQUIRES='-r tools/test-requires' +fi + +if [ -f tools/test-requires -a \ + `git diff HEAD^1 tools/test-requires 2>/dev/null | wc -l` -gt 0 -o \ + `git diff HEAD^1 tools/pip-requires 2>/dev/null | wc -l` -gt 0 ] +then + rm -rf .venv + virtualenv --no-site-packages .venv + pip -E .venv install --upgrade -r tools/pip-requires $TEST_REQUIRES +fi diff --git a/modules/jenkins_slave/files/slave_scripts/create-ppa-package.sh b/modules/jenkins_slave/files/slave_scripts/create-ppa-package.sh new file mode 100755 index 0000000000..a758a924b8 --- /dev/null +++ b/modules/jenkins_slave/files/slave_scripts/create-ppa-package.sh @@ -0,0 +1,78 @@ +#!/bin/bash -xe + +if [ -z "$PROJECT" ] +then + echo '$PROJECT not set.' + exit 1 +fi + +case "$GERRIT_REFNAME" in + master) + export PPAS="ppa:openstack-ppa/bleeding-edge" + ;; + milestone-proposed) + export PPAS="ppa:openstack-ppa/milestone-proposed" + ;; + *) + echo "No PPA defined for branch $GERRIT_REFNAME" + exit 0 +esac + +HUDSON=http://localhost:8080/ +# We keep packaging for openstack trunk in lp:~o-u-p/$project/ubuntu +# For a release (diablo, essex), it's in lp:~o-u-p/$project/$release +OPENSTACK_RELEASE=${OPENSTACK_RELEASE:-ubuntu} +BZR_BRANCH=${BZR_BRANCH:-lp:~openstack-ubuntu-packagers/$PROJECT/${OPENSTACK_RELEASE}} +PPAS=${PPAS:-ppa:$PROJECT-core/trunk} +PACKAGING_REVNO=${PACKAGING_REVNO:--1} +series=${series:-lucid} + +cd build + +tarball="$(echo dist/$PROJECT*.tar.gz)" +version="${tarball%.tar.gz}" +version="${version#*$PROJECT-}" +base_version=$version +if [ -n "${EXTRAVERSION}" ] +then + version="${version%~*}${EXTRAVERSION}~${version#*~}" +fi +tar xvzf "${tarball}" +echo ln -s "${tarball}" "${PROJECT}_${version}.orig.tar.gz" +ln -s "${tarball}" "${PROJECT}_${version}.orig.tar.gz" + +# Overlay packaging +# (Intentionally using the natty branch. For these PPA builds, we don't need to diverge +# (yet, at least), so it makes the branch management easier this way. +# Note: Doing a checkout and deleting .bzr afterwards instead of just doing an export, +# because export refuses to overlay over an existing directory, so this was easier. +# (We need to not have the .bzr in there, otherwise vcsversion.py might get overwritten) +echo bzr checkout -r ${PACKAGING_REVNO} --lightweight $BZR_BRANCH $PROJECT-* +bzr checkout -r ${PACKAGING_REVNO} --lightweight $BZR_BRANCH $PROJECT-* +cd $PROJECT-* +if [ -d .git ] +then + PACKAGING_REVNO="$(git log --oneline | wc -l)" + rm -rf .git +else + PACKAGING_REVNO="$(bzr revno --tree)" + rm -rf .bzr +fi + +# Please don't change this. It's the only way I'll get notified +# if an upload fails. +export DEBFULLNAME="Soren Hansen" +export DEBEMAIL="soren@openstack.org" + +buildno=$BUILD_NUMBER +pkgversion="${version}-0ubuntu0~${series}${buildno}" +dch -b --force-distribution --v "${pkgversion}" "Automated PPA build. Packaging revision: ${PACKAGING_REVNO}." -D $series +dpkg-buildpackage -rfakeroot -S -sa -nc -k32EE128C +if ! [ "$DO_UPLOAD" = "no" ] +then + for ppa in $PPAS + do + dput --force $ppa "../${PROJECT}_${pkgversion}_source.changes" + done +fi +cd .. diff --git a/modules/jenkins_slave/files/slave_scripts/create-tarball.sh b/modules/jenkins_slave/files/slave_scripts/create-tarball.sh new file mode 100755 index 0000000000..a53a975c2b --- /dev/null +++ b/modules/jenkins_slave/files/slave_scripts/create-tarball.sh @@ -0,0 +1,84 @@ +#!/bin/bash -xe + +# In case we start doing something more sophisticated with other refs +# later (such as tags). +BRANCH=$GERRIT_REFNAME + +if [ $BRANCH == "milestone-proposed" ] +then + REVNOPREFIX="r" +fi +if [[ $BRANCH =~ ^stable/.*$ ]] +then + NOMILESTONE="true" +fi + +# Should be ~ if tarball version is the one we're working *toward*. (By far preferred!) +# Should be + if tarball version is already released and we're moving forward after it. +SEPARATOR=${SEPARATOR:-'~'} + +if [ -z "$PROJECT" ] +then + echo '$PROJECT not set.' + exit 1 +fi + +find_next_version() { + datestamp="${datestamp:-$(date +%Y%m%d)}" + git fetch origin +refs/meta/*:refs/remotes/meta/* + milestonever="$(git show meta/openstack/release:${BRANCH})" + if [ $? != 0 ] + then + if [ "$NOMILESTONE" = "true" ] + then + milestonever="" + else + echo "Milestone file ${BRANCH} not found. Bailing out." >&2 + exit 1 + fi + fi + + version="$milestonever" + if [ -n "$version" ] + then + version="${version}~" + fi + revno="${revno:-$(git log --oneline | wc -l)}" + version="$(printf %s%s.%s%d "$version" "$datestamp" "$REVNOPREFIX" "$revno")" + printf "%s" "$version" +} + +snapshotversion=$(find_next_version) + +rm -f dist/*.tar.gz +if [ -f setup.py ] ; then + # Try tox and cached bundles first + if [ -e ".cache.bundle" ] ; then + if [ -f tox.ini ] ; then + if tox --showconfig | grep testenv | grep jenkinsvenv >/dev/null 2>&1 + then + tox -ejenkinsvenv python setup.py sdist + else + tox -evenv python setup.py sdist + fi + else + rm -rf .venv + mv .cache.bundle .cache.pybundle + virtualenv --no-site-packages .venv + .venv/bin/pip install .cache.pybundle + rm .cache.pybundle + tools/with_venv.sh python setup.py sdist + fi + # Try old style venv's second + elif [ -d .venv -a -f tools/with_venv.sh ] ; then + tools/with_venv.sh python setup.py sdist + # Last but not least, just make a tarball + else + python setup.py sdist + fi + # There should only be one, so this should be safe. + tarball=$(echo dist/*.tar.gz) + + echo mv "$tarball" "dist/$(basename $tarball .tar.gz)${SEPARATOR}${snapshotversion}.tar.gz" + mv "$tarball" "dist/$(basename $tarball .tar.gz)${SEPARATOR}${snapshotversion}.tar.gz" +fi diff --git a/modules/jenkins_slave/files/slave_scripts/gerrit-git-prep.sh b/modules/jenkins_slave/files/slave_scripts/gerrit-git-prep.sh new file mode 100755 index 0000000000..fa714fa609 --- /dev/null +++ b/modules/jenkins_slave/files/slave_scripts/gerrit-git-prep.sh @@ -0,0 +1,35 @@ +#!/bin/bash -xe + +SITE=$1 +if [ -z "$SITE" ] +then + echo "The site name (eg 'openstack') must be the first argument." + exit 1 +fi + +if [ -z "$GERRIT_NEWREV" ] && [ -z "$GERRIT_REFSPEC" ] +then + echo "This job may only be triggered by Gerrit." + exit 1 +fi + +if [[ ! -e .git ]] +then + git clone https://review.$SITE.org/p/$GERRIT_PROJECT . +fi +git remote update || git remote update # attempt to work around bug #925790 +git reset --hard +git clean -x -f -d -q + +if [ ! -z "$GERRIT_REFSPEC" ] +then + git checkout $GERRIT_BRANCH + git reset --hard remotes/origin/$GERRIT_BRANCH + git clean -x -f -d -q + git fetch https://review.$SITE.org/p/$GERRIT_PROJECT $GERRIT_REFSPEC + git merge FETCH_HEAD +else + git checkout $GERRIT_NEWREV + git reset --hard $GERRIT_NEWREV + git clean -x -f -d -q +fi diff --git a/modules/jenkins_slave/files/slave_scripts/lvm-kexec-reset.sh b/modules/jenkins_slave/files/slave_scripts/lvm-kexec-reset.sh new file mode 100755 index 0000000000..0916b05018 --- /dev/null +++ b/modules/jenkins_slave/files/slave_scripts/lvm-kexec-reset.sh @@ -0,0 +1,7 @@ +#!/bin/bash -x +lvremove -f /dev/main/last_root +lvrename /dev/main/root last_root +lvcreate -L20G -s -n root /dev/main/orig_root +APPEND="`cat /proc/cmdline`" +kexec -l /vmlinuz --initrd=/initrd.img --append="$APPEND" +nohup bash -c "sleep 2; kexec -e" /dev/null 2>&1 & diff --git a/modules/jenkins_slave/files/slave_scripts/ping.py b/modules/jenkins_slave/files/slave_scripts/ping.py new file mode 100755 index 0000000000..a359522e62 --- /dev/null +++ b/modules/jenkins_slave/files/slave_scripts/ping.py @@ -0,0 +1,11 @@ +#!/usr/bin/env python + +import sys +from subprocess import * + +p = Popen(["ping", sys.argv[1]], stdout=PIPE) +while True: + line = p.stdout.readline().strip() + if 'bytes from' in line: + p.terminate() + sys.exit(0) diff --git a/modules/jenkins_slave/files/slave_scripts/propose_translations.sh b/modules/jenkins_slave/files/slave_scripts/propose_translations.sh new file mode 100755 index 0000000000..b38d70307a --- /dev/null +++ b/modules/jenkins_slave/files/slave_scripts/propose_translations.sh @@ -0,0 +1,32 @@ +#!/bin/bash -xe + +git config user.name "OpenStack Jenkins" +git config user.email "jenkins@openstack.org" + +# See if there is an open change in the launchpad/translations topic +# If so, amend the commit with new changes since then +previous=`ssh -p 29418 review.openstack.org gerrit query --current-patch-set status:open project:openstack/PROJECT topic:launchpad/translations | grep "^ number:" | awk '{print $2}'` +if [ "x${previous}" != "x" ] ; then + git review -d ${previous} + amend="--amend" +fi + +tar xvfz po.tgz +rm po.tgz +for f in po/*po ; do + lang=`echo $f | cut -f2 -d/ | cut -f1 -d.` + if [ -d $PROJECT/locale/$lang ] ; then + cp $f $PROJECT/locale/$lang/LC_MESSAGES/$PROJECT.po + fi +done +python setup.py extract_messages +git add $PROJECT/locale/$PROJECT.pot +python setup.py update_catalog +for f in po/*po ; do + lang=`echo $f | cut -f2 -d/ | cut -f1 -d.` + if [ -d $PROJECT/locale/$lang ] ; then + git add $PROJECT/locale/$lang/LC_MESSAGES/$PROJECT.po + fi +done +git commit ${amend} -m "Imported Translations from Launchpad" +git review -t launchpad/translations diff --git a/modules/jenkins_slave/files/slave_scripts/run-cover.sh b/modules/jenkins_slave/files/slave_scripts/run-cover.sh new file mode 100755 index 0000000000..a5b3a81388 --- /dev/null +++ b/modules/jenkins_slave/files/slave_scripts/run-cover.sh @@ -0,0 +1,25 @@ +#!/bin/bash -xe + +# If a bundle file is present, call tox with the jenkins version of +# the test environment so it is used. Otherwise, use the normal +# (non-bundle) test environment. Also, run pip freeze on the +# resulting environment at the end so that we have a record of exactly +# what packages we ended up testing. +# + +if [ -f .cache.bundle ] +then + venv=jenkinscover +else + venv=cover +fi + +tox -e$venv +result=$? + +echo "Begin pip freeze output from test virtualenv:" +echo "======================================================================" +.tox/$venv/bin/pip freeze +echo "======================================================================" + +exit $result diff --git a/modules/jenkins_slave/files/slave_scripts/run-docs.sh b/modules/jenkins_slave/files/slave_scripts/run-docs.sh new file mode 100755 index 0000000000..f0679d627a --- /dev/null +++ b/modules/jenkins_slave/files/slave_scripts/run-docs.sh @@ -0,0 +1,27 @@ +#!/bin/bash -xe + +# If a bundle file is present, call tox with the jenkins version of +# the test environment so it is used. Otherwise, use the normal +# (non-bundle) test environment. Also, run pip freeze on the +# resulting environment at the end so that we have a record of exactly +# what packages we ended up testing. +# + +if [ -f .cache.bundle ] +then + venv=jenkinsvenv +else + venv=venv +fi + +mkdir -p doc/build +export HUDSON_PUBLISH_DOCS=1 +tox -e$venv -- python setup.py build_sphinx +result=$? + +echo "Begin pip freeze output from test virtualenv:" +echo "======================================================================" +.tox/$venv/bin/pip freeze +echo "======================================================================" + +exit $result diff --git a/modules/jenkins_slave/files/slave_scripts/run-tox.sh b/modules/jenkins_slave/files/slave_scripts/run-tox.sh new file mode 100755 index 0000000000..b83d65a44c --- /dev/null +++ b/modules/jenkins_slave/files/slave_scripts/run-tox.sh @@ -0,0 +1,38 @@ +#!/bin/bash -xe + +# If a bundle file is present, call tox with the jenkins version of +# the test environment so it is used. Otherwise, use the normal +# (non-bundle) test environment. Also, run pip freeze on the +# resulting environment at the end so that we have a record of exactly +# what packages we ended up testing. +# +# Usage: run-tox.sh PYTHONVERSION +# +# Where PYTHONVERSION is the numeric version identifier used as a suffix +# in the tox.ini file. E.g., "26" or "27" for "py26"/"jenkins26" or +# "py27"/"jenkins27" respectively. + +version=$1 + +if [ -z "$version" ] +then + echo "The tox environment python version (eg '27') must be the first argument." + exit 1 +fi + +if [ -f .cache.bundle ] +then + venv=jenkins$version +else + venv=py$version +fi + +tox -e$venv +result=$? + +echo "Begin pip freeze output from test virtualenv:" +echo "======================================================================" +.tox/$venv/bin/pip freeze +echo "======================================================================" + +exit $result diff --git a/modules/jenkins_slave/files/slave_scripts/update-pip-cache.sh b/modules/jenkins_slave/files/slave_scripts/update-pip-cache.sh new file mode 100755 index 0000000000..e1f08f0990 --- /dev/null +++ b/modules/jenkins_slave/files/slave_scripts/update-pip-cache.sh @@ -0,0 +1,10 @@ +#!/bin/bash -xe + +mkdir -p ~/cache/pip +VENV=`mktemp -d` +virtualenv --no-site-packages $VENV +cd $VENV +. bin/activate +PIP_DOWNLOAD_CACHE=~/cache/pip pip install `cat ~/devstack/files/pips/*` +cd +rm -fr $VENV diff --git a/modules/jenkins_slave/files/slave_scripts/wait_for_nova.sh b/modules/jenkins_slave/files/slave_scripts/wait_for_nova.sh new file mode 100755 index 0000000000..564bb5859a --- /dev/null +++ b/modules/jenkins_slave/files/slave_scripts/wait_for_nova.sh @@ -0,0 +1,12 @@ +#!/bin/bash -xe + +URL=$1 + +echo "Jenkins: Waiting for Nova to start on infrastructure node" +RET=7 +while [ $RET != 0 ]; do + curl -s $URL >/dev/null + RET=$? + sleep 1 +done +echo "Jenkins: Nova is running." diff --git a/modules/jenkins_slave/files/slave_scripts/wait_for_puppet.sh b/modules/jenkins_slave/files/slave_scripts/wait_for_puppet.sh new file mode 100755 index 0000000000..9812a15a98 --- /dev/null +++ b/modules/jenkins_slave/files/slave_scripts/wait_for_puppet.sh @@ -0,0 +1,21 @@ +#!/bin/bash -xe + +# wait_for_pupet.sh LOGFILE HOSTNAME [HOSTNAME...] +# Search LOGFILE for puppet completion on each host + +FINISH_RE="puppet-agent\[.*\]: Finished catalog run in .* seconds" +LOGFILE=$1 +shift +HOSTS=$@ + +echo "Jenkins: Waiting for puppet to complete on all nodes" +DONE=0 +while [ $DONE != 1 ]; do + DONE=1 + for hostname in $HOSTS + do + if !(grep "$hostname $FINISH_RE" $LOGFILE >/dev/null); then DONE=0; fi + done + sleep 5 +done +echo "Jenkins: Puppet is complete." diff --git a/modules/jenkins_slave/manifests/jenkinsuser.pp b/modules/jenkins_slave/manifests/jenkinsuser.pp index 37a103f56c..f781f31bbd 100644 --- a/modules/jenkins_slave/manifests/jenkinsuser.pp +++ b/modules/jenkins_slave/manifests/jenkinsuser.pp @@ -205,4 +205,56 @@ define jenkinsuser($ensure = present, $ssh_key) { ], } + + file { '/usr/local/jenkins': + owner => 'root', + group => 'root', + mode => 755, + ensure => 'directory', + } + + file { '/usr/local/jenkins/slave_scripts': + owner => 'root', + group => 'root', + mode => 755, + ensure => 'directory', + require => File['/usr/local/jenkins'], + } + + + $slave_scripts = [ + 'slave_scripts/baremetal-archive-logs.sh', + 'slave_scripts/baremetal-deploy.sh', + 'slave_scripts/baremetal-os-install.sh', + 'slave_scripts/build-bundle.sh', + 'slave_scripts/build-venv.sh', + 'slave_scripts/copy-bundle.sh', + 'slave_scripts/copy-venv.sh', + 'slave_scripts/create-ppa-package.sh', + 'slave_scripts/create-tarball.sh', + 'slave_scripts/gerrit-git-prep.sh', + 'slave_scripts/lvm-kexec-reset.sh', + 'slave_scripts/ping.py', + 'slave_scripts/propose_translations.sh', + 'slave_scripts/run-cover.sh', + 'slave_scripts/run-docs.sh', + 'slave_scripts/run-tox.sh', + 'slave_scripts/update-pip-cache.sh', + 'slave_scripts/wait_for_nova.sh', + 'slave_scripts/wait_for_puppet.sh', + ] + + file { $slave_scripts: + name => "/usr/local/jenkins/slave_scripts/${name}", + owner => 'root', + group => 'root', + mode => 750, + ensure => 'present', + require => File['/usr/local/jenkins/slave_scripts'], + source => [ + "puppet:///modules/jenkins_slave/${name}", + ], + } + + }