Merge "Convert gate to file backend rather than partition"

tests/setup_RedHat.sh

@@ -1,49 +0,0 @@
-#!/bin/bash
-
-set -o xtrace
-set -o errexit
-
-DEV=$1
-
-# (SamYaple)TODO: Remove the path overriding
-export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
-
-cat | sudo tee /etc/yum.repos.d/docker.repo << EOF
-[docker]
-name=Docker Main Repository
-baseurl=https://yum.dockerproject.org/repo/main/centos/7
-enabled=1
-gpgcheck=1
-gpgkey=https://yum.dockerproject.org/gpg
-EOF
-
-sudo yum install -y libffi-devel openssl-devel docker-engine xfsprogs
-
-# Only do FS optimization if we have a secondary disk
-if [[ -b /dev/${DEV} ]]; then
-    # Setup backing disk for use with Docker. This is to ensure we use the ephemeral
-    # disk provided to the build instance. It ensures the correct disk and storage
-    # driver are used for Docker. It is recommend to use the thin provisioning
-    # driver. https://github.com/docker/docker/blob/master/man/docker.1.md
-    sudo parted /dev/${DEV} -s -- mklabel msdos mkpart pri 1 -1
-    # Figure out the path to the partitioned device
-    PARTDEV=$(ls "/dev/${DEV}"* | egrep "/dev/${DEV}p?1")
-    sudo pvcreate ${PARTDEV}
-    sudo vgcreate kolla01 ${PARTDEV}
-    sudo lvcreate -n thin01 -L 60G kolla01
-    sudo lvcreate -n thin01meta -L 2G kolla01
-    yes | sudo lvconvert --type thin-pool --poolmetadata kolla01/thin01meta kolla01/thin01
-
-    # Setup Docker
-    sudo sed -i -r 's,(ExecStart)=(.+),\1=/usr/bin/docker daemon --storage-driver devicemapper --storage-opt dm.fs=xfs --storage-opt dm.thinpooldev=kolla01-thin01 --storage-opt dm.use_deferred_removal=true,' /usr/lib/systemd/system/docker.service
-    sudo systemctl daemon-reload
-fi
-
-sudo systemctl start docker
-sudo docker info
-
-# disable ipv6 until we're sure routes to fedora mirrors work properly
-sudo sh -c 'echo "net.ipv6.conf.all.disable_ipv6 = 1" >> /etc/sysctl.conf'
-sudo /usr/sbin/sysctl -p
-
-echo "Completed $0."

tests/setup_gate.sh

@@ -1,123 +0,0 @@
-#!/bin/bash
-
-set -o xtrace
-set -o errexit
-
-# Just for mandre :)
-if [[ ! -f /etc/sudoers.d/jenkins ]]; then
-    echo "jenkins ALL=(:docker) NOPASSWD: ALL" | sudo tee /etc/sudoers.d/jenkins
-fi
-
-function dump_node_info {
-    # NOTE(SamYaple): function for debugging gate
-    set +o errexit
-    local OLD_PATH="${PATH}"
-    PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
-    sudo parted -l
-
-    sudo mount
-
-    df -h
-
-    uname -a
-
-    cat /etc/*release*
-
-    PATH="${OLD_PATH}"
-    set -o errexit
-}
-
-function detect_disk {
-    # TODO(SamYaple): This check could be much better, but should work for now
-    if [[ $(hostname | grep rax) ]]; then
-        export DEV="xvde"
-    else
-        echo "Assuming this is a hpcloud box"
-        export DEV="vdb"
-    fi
-}
-
-function setup_config {
-    # generate the config
-    tox -e genconfig
-    # Copy configs
-    sudo cp -a etc/kolla /etc/
-}
-
-function detect_distro {
-    DISTRO=$(ansible all -i "localhost," -msetup -clocal | awk -F\" '/ansible_os_family/ {print $4}')
-}
-
-function setup_ssh {
-    # Generate a new keypair that Ansible will use
-    ssh-keygen -f /home/jenkins/.ssh/kolla -N ''
-    cat /home/jenkins/.ssh/kolla.pub >> /home/jenkins/.ssh/authorized_keys
-
-    # Push the the public key around to all of the nodes
-    for ip in $(cat /etc/nodepool/sub_nodes_private); do
-        scp /home/jenkins/.ssh/kolla.pub ${ip}:/home/jenkins/.ssh/authorized_keys
-        # TODO(SamYaple): Remove this root key pushing once Kolla doesn't
-        # require root anymore.
-        ssh ${ip} -i /home/jenkins/.ssh/kolla 'sudo mkdir -p /root/.ssh; sudo cp /home/jenkins/.ssh/* /root/.ssh/'
-    done
-
-    # From now on use the new IdentityFile for connecting to other hosts
-    echo "IdentityFile /home/jenkins/.ssh/kolla" >> /home/jenkins/.ssh/config
-}
-
-function setup_inventory {
-    local counter=0
-
-    detect_distro
-    if [[ "${DISTRO}" == "Debian" ]]; then
-        ANSIBLE_CONNECTION_TYPE=ssh
-    else
-        ANSIBLE_CONNECTION_TYPE=local
-    fi
-
-    echo -e "127.0.0.1\tlocalhost" > /tmp/hosts
-    for ip in $(cat /etc/nodepool/{node_private,sub_nodes_private}); do
-        : $((counter++))
-        echo -e "${ip}\tnode${counter} $(ssh ${ip} hostname)" >> /tmp/hosts
-        echo "node${counter} ansible_connection=${ANSIBLE_CONNECTION_TYPE}" >> ${RAW_INVENTORY}
-    done
-
-    sudo chown root: /tmp/hosts
-    sudo chmod 644 /tmp/hosts
-    sudo mv /tmp/hosts /etc/hosts
-}
-
-function setup_ansible {
-    RAW_INVENTORY=/tmp/kolla/raw_inventory
-    mkdir /tmp/kolla
-
-    sudo -H pip install "ansible<2" "docker-py>=1.6.0"
-
-    setup_inventory
-
-    # Record the running state of the environment as seen by the setup module
-    ansible all -i ${RAW_INVENTORY} -m setup > /tmp/logs/ansible/initial-setup
-}
-
-function setup_node {
-    detect_disk
-    ansible-playbook -i ${RAW_INVENTORY} -edocker_dev=${DEV} tests/setup_nodes.yml
-}
-
-function setup_logging {
-    # This directory is the directory that is copied with the devstack-logs
-    # publisher. It must exist at /home/jenkins/workspace/<job-name>/logs
-    mkdir logs
-
-    # For ease of access we symlink that logs directory to a known path
-    ln -s $(pwd)/logs /tmp/logs
-    mkdir -p /tmp/logs/{ansible,build}
-}
-
-setup_logging
-(dump_node_info 2>&1) > /tmp/logs/node_info_first
-setup_ssh
-setup_ansible
-setup_node
-setup_config
-(dump_node_info 2>&1) > /tmp/logs/node_info_last

tests/setup_gate.sh

@@ -0,0 +1 @@
+../tools/setup_gate.sh

tests/test_build.py

@@ -33,7 +33,7 @@ class BuildTest(base.BaseTestCase):
         super(BuildTest, self).setUp()
         self.useFixture(log_fixture.SetLogLevel([__name__],
                                                 logging.logging.INFO))
-        self.build_args = [__name__, "--debug"]
+        self.build_args = [__name__, "--debug", '--threads', '4']
 
     @testtools.skipUnless(os.environ.get('DOCKER_BUILD_TEST'),
                           'Skip the docker build test')

tools/dump_info.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+set -o xtrace
+
+function dump_node_info {
+    # NOTE(SamYaple): function for debugging gate
+    set +o errexit
+    local OLD_PATH="${PATH}"
+    PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+    sudo parted -l
+
+    sudo mount
+
+    df -h
+
+    uname -a
+
+    cat /etc/*release*
+
+    cat /proc/meminfo
+
+    PATH="${OLD_PATH}"
+    set -o errexit
+}
+
+(dump_node_info 2>&1) > /tmp/logs/node_info_$(date +%s)

tools/setup_Debian.sh

@@ -3,7 +3,30 @@
 set -o xtrace
 set -o errexit
 
-DEV=$1
+function setup_disk {
+    sudo swapoff -a
+    sudo dd if=/dev/zero of=/swapfile bs=1M count=4096
+    sudo chmod 0600 /swapfile
+    sudo mkswap /swapfile
+    sudo /sbin/swapon /swapfile
+
+    sudo dd if=/dev/zero of=/docker bs=1M count=10240
+    losetup -f /docker
+    DEV=$(losetup -a | awk -F: '/\/docker/ {print $1}')
+
+    # Format Disks and setup Docker to use BTRFS
+    sudo parted ${DEV} -s -- mklabel msdos
+    sudo rm -rf /var/lib/docker
+    sudo mkdir /var/lib/docker
+
+    # We want to snapshot the entire docker directory so we have to first create a
+    # subvolume and use that as the root for the docker directory.
+    sudo mkfs.btrfs -f ${DEV}
+    sudo mount ${DEV} /var/lib/docker
+    sudo btrfs subvolume create /var/lib/docker/docker
+    sudo umount /var/lib/docker
+    sudo mount -o noatime,subvol=docker ${DEV} /var/lib/docker
+}
 
 # (SamYaple)TODO: Remove the path overriding
 export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
@@ -14,28 +37,10 @@ sudo apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58
 sudo apt-get update
 sudo apt-get install -y --no-install-recommends docker-engine btrfs-tools
 
-# Only do FS optimization if we have a secondary disk
-if [[ -b /dev/${DEV} ]]; then
-    # The reason for using BTRFS is stability. There are numerous issues with the
-    # devicemapper backed on Ubuntu and AUFS is slow. BTRFS is very solid as a
-    # backend in my experince. I use ie almost exclusively.
-    # Format Disks and setup Docker to use BTRFS
-    sudo umount /dev/${DEV} || true
-    sudo parted /dev/${DEV} -s -- mklabel msdos
 sudo service docker stop
+setup_disk
 echo 'DOCKER_OPTS="-s btrfs"' | sudo tee /etc/default/docker
-    sudo rm -rf /var/lib/docker/*
-
-    # We want to snapshot the entire docker directory so we have to first create a
-    # subvolume and use that as the root for the docker directory.
-    sudo mkfs.btrfs -f /dev/${DEV}
-    sudo mount /dev/${DEV} /var/lib/docker
-    sudo btrfs subvolume create /var/lib/docker/docker
-    sudo umount /var/lib/docker
-    sudo mount -o noatime,compress=lzo,space_cache,subvol=docker /dev/${DEV} /var/lib/docker
-
 sudo service docker start
-fi
 
 sudo docker info
 

tools/setup_RedHat.sh

@@ -0,0 +1,58 @@
+#!/bin/bash
+
+set -o xtrace
+set -o errexit
+
+function setup_disk {
+    sudo swapoff -a
+    sudo dd if=/dev/zero of=/swapfile bs=1M count=4096
+    sudo chmod 0600 /swapfile
+    sudo mkswap /swapfile
+    sudo /sbin/swapon /swapfile
+
+    sudo dd if=/dev/zero of=/docker bs=1M count=10240
+    losetup -f /docker
+    DEV=$(losetup -a | awk -F: '/\/docker/ {print $1}')
+
+    # Format Disks and setup Docker to use BTRFS
+    sudo parted ${DEV} -s -- mklabel msdos
+    sudo rm -rf /var/lib/docker
+    sudo mkdir /var/lib/docker
+
+    # We want to snapshot the entire docker directory so we have to first create a
+    # subvolume and use that as the root for the docker directory.
+    sudo mkfs.btrfs -f ${DEV}
+    sudo mount ${DEV} /var/lib/docker
+    sudo btrfs subvolume create /var/lib/docker/docker
+    sudo umount /var/lib/docker
+    sudo mount -o noatime,subvol=docker ${DEV} /var/lib/docker
+}
+
+# (SamYaple)TODO: Remove the path overriding
+export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+
+cat | sudo tee /etc/yum.repos.d/docker.repo << EOF
+[docker]
+name=Docker Main Repository
+baseurl=https://yum.dockerproject.org/repo/main/centos/7
+enabled=1
+gpgcheck=1
+gpgkey=https://yum.dockerproject.org/gpg
+EOF
+
+sudo yum install -y libffi-devel openssl-devel docker-engine btrfs-progs
+
+setup_disk
+
+# Setup Docker
+sudo sed -i -r 's,(ExecStart)=(.+),\1=/usr/bin/docker daemon --storage-driver btrfs,' /usr/lib/systemd/system/docker.service
+sudo systemctl daemon-reload
+
+sudo systemctl start docker
+sudo docker info
+
+# disable ipv6 until we're sure routes to fedora mirrors work properly
+sudo sh -c 'echo "net.ipv6.conf.all.disable_ipv6 = 1" >> /etc/sysctl.conf'
+sudo /usr/sbin/sysctl -p
+
+echo "Completed $0."

tools/setup_gate.sh

@@ -0,0 +1,92 @@
+#!/bin/bash
+
+set -o xtrace
+set -o errexit
+
+# Just for mandre :)
+if [[ ! -f /etc/sudoers.d/jenkins ]]; then
+    echo "jenkins ALL=(:docker) NOPASSWD: ALL" | sudo tee /etc/sudoers.d/jenkins
+fi
+
+function setup_config {
+    # generate the config
+    tox -e genconfig
+    # Copy configs
+    sudo cp -a etc/kolla /etc/
+}
+
+function detect_distro {
+    DISTRO=$(ansible all -i "localhost," -msetup -clocal | awk -F\" '/ansible_os_family/ {print $4}')
+}
+
+function setup_ssh {
+    # Generate a new keypair that Ansible will use
+    ssh-keygen -f /home/jenkins/.ssh/kolla -N ''
+    cat /home/jenkins/.ssh/kolla.pub >> /home/jenkins/.ssh/authorized_keys
+
+    # Push the the public key around to all of the nodes
+    for ip in $(cat /etc/nodepool/sub_nodes_private); do
+        scp /home/jenkins/.ssh/kolla.pub ${ip}:/home/jenkins/.ssh/authorized_keys
+        # TODO(SamYaple): Remove this root key pushing once Kolla doesn't
+        # require root anymore.
+        ssh ${ip} -i /home/jenkins/.ssh/kolla 'sudo mkdir -p /root/.ssh; sudo cp /home/jenkins/.ssh/* /root/.ssh/'
+    done
+
+    # From now on use the new IdentityFile for connecting to other hosts
+    echo "IdentityFile /home/jenkins/.ssh/kolla" >> /home/jenkins/.ssh/config
+}
+
+function setup_inventory {
+    local counter=0
+
+    detect_distro
+    if [[ "${DISTRO}" == "Debian" ]]; then
+        ANSIBLE_CONNECTION_TYPE=ssh
+    else
+        ANSIBLE_CONNECTION_TYPE=local
+    fi
+
+    echo -e "127.0.0.1\tlocalhost" > /tmp/hosts
+    for ip in $(cat /etc/nodepool/{node_private,sub_nodes_private}); do
+        : $((counter++))
+        echo -e "${ip}\tnode${counter} $(ssh ${ip} hostname)" >> /tmp/hosts
+        echo "node${counter} ansible_connection=${ANSIBLE_CONNECTION_TYPE}" >> ${RAW_INVENTORY}
+    done
+
+    sudo chown root: /tmp/hosts
+    sudo chmod 644 /tmp/hosts
+    sudo mv /tmp/hosts /etc/hosts
+}
+
+function setup_ansible {
+    RAW_INVENTORY=/tmp/kolla/raw_inventory
+    mkdir /tmp/kolla
+
+    sudo -H pip install "ansible<2" "docker-py>=1.6.0"
+
+    setup_inventory
+
+    # Record the running state of the environment as seen by the setup module
+    ansible all -i ${RAW_INVENTORY} -m setup > /tmp/logs/ansible/initial-setup
+}
+
+function setup_node {
+    ansible-playbook -i ${RAW_INVENTORY} tools/setup_nodes.yml
+}
+
+function setup_logging {
+    # This directory is the directory that is copied with the devstack-logs
+    # publisher. It must exist at /home/jenkins/workspace/<job-name>/logs
+    mkdir logs
+
+    # For ease of access we symlink that logs directory to a known path
+    ln -s $(pwd)/logs /tmp/logs
+    mkdir -p /tmp/logs/{ansible,build}
+}
+
+setup_logging
+tools/dump_info.sh
+setup_ssh
+setup_ansible
+setup_node
+setup_config

tools/setup_nodes.yml

@@ -25,4 +25,4 @@
       path: /tmp/{{ inventory_hostname }}
 
   - name: Run node setup
-    shell: sudo /tmp/setup.sh {{ docker_dev }}
+    shell: sudo /tmp/setup.sh

tox.ini

@@ -31,8 +31,9 @@ commands = {posargs}
 commands = python setup.py build_sphinx
 
 [testenv:setupenv]
-whitelist_externals = bash
+commands =
-commands = bash -c tests/setup_gate.sh
+  {toxinidir}/tools/setup_gate.sh
+  {toxinidir}/tools/dump_info.sh
 
 [testenv:build-centos-binary]
 whitelist_externals = find
@@ -44,6 +45,7 @@ commands =
   find . -type f -name "*.pyc" -delete
   bash -c "if [ ! -d .testrepository ]; then testr init; fi"
   sudo -E -g docker testr run test_build.BuildTestCentosBinary
+  {toxinidir}/tools/dump_info.sh
 
 [testenv:build-centos-source]
 whitelist_externals = find
@@ -55,6 +57,7 @@ commands =
   find . -type f -name "*.pyc" -delete
   bash -c "if [ ! -d .testrepository ]; then testr init; fi"
   sudo -E -g docker testr run test_build.BuildTestCentosSource
+  {toxinidir}/tools/dump_info.sh
 
 [testenv:build-ubuntu-source]
 whitelist_externals = find
@@ -66,6 +69,7 @@ commands =
   find . -type f -name "*.pyc" -delete
   bash -c "if [ ! -d .testrepository ]; then testr init; fi"
   sudo -E -g docker testr run test_build.BuildTestUbuntuSource
+  {toxinidir}/tools/dump_info.sh
 
 [testenv:deploy-centos-binary]
 whitelist_externals = find
@@ -77,7 +81,8 @@ commands =
   find . -type f -name "*.pyc" -delete
   bash -c "if [ ! -d .testrepository ]; then testr init; fi"
   sudo -E -g docker testr run test_build.DeployTestCentosBinary
-  sudo tests/deploy_aio.sh centos binary
+  sudo {toxinidir}/tools/deploy_aio.sh centos binary
+  {toxinidir}/tools/dump_info.sh
 
 [testenv:deploy-centos-source]
 whitelist_externals = find
@@ -89,7 +94,8 @@ commands =
   find . -type f -name "*.pyc" -delete
   bash -c "if [ ! -d .testrepository ]; then testr init; fi"
   sudo -E -g docker testr run test_build.DeployTestCentosSource
-  sudo tests/deploy_aio.sh centos source
+  sudo {toxinidir}/tools/deploy_aio.sh centos source
+  {toxinidir}/tools/dump_info.sh
 
 [testenv:deploy-ubuntu-source]
 whitelist_externals = find
@@ -101,7 +107,8 @@ commands =
   find . -type f -name "*.pyc" -delete
   bash -c "if [ ! -d .testrepository ]; then testr init; fi"
   sudo -E -g docker testr run test_build.DeployTestUbuntuSource
-  sudo tests/deploy_aio.sh ubuntu source
+  sudo {toxinidir}/tools/deploy_aio.sh ubuntu source
+  {toxinidir}/tools/dump_info.sh
 
 [testenv:deploy-multinode-ubuntu-source]
 whitelist_externals = find
@@ -113,6 +120,7 @@ commands =
   find . -type f -name "*.pyc" -delete
   bash -c "if [ ! -d .testrepository ]; then testr init; fi"
   sudo -E -g docker testr run test_build.BuildTestUbuntuSource
+  {toxinidir}/tools/dump_info.sh
 
 [testenv:genconfig]
 whitelist_externals = which