Setup_env script should install Tempest from tag

setup_env script should be able to install Tempest from specific tag
release. Key -t added to setup_env script. It specifies Tempest tag
release which will be installed as a test runner. By default last tag
from Tempest github repository will be used. Installed by setup script,
Tempest test runner is only single available option for now. Because of
it, option --tempest-dir option is removed from refstack-client

https://storyboard.openstack.org/#!/story/309
https://storyboard.openstack.org/#!/story/324

Change-Id: I182c3870a7ce4bbcb188836d189538dfd54b1c26

README.rst

@@ -5,12 +5,16 @@ refstack-client is a command line utility that allows you to execute Tempest
 test runs based on configurations you specify.  When finished running Tempest
 it sends the passed test data back to the Refstack API server.
 
-**Setup**
+**Environment setup**
 
 We've created an "easy button" for Ubuntu, Centos, RHEL and openSuSe.
 
 $ ./setup_env
 
+**Options:**
+   a. -t option allows to specify tag in Tempest repository which will be
+   installed. By default, Tempest from last tag release will be used.
+
 **Usage**
 
 1. Prepare a tempest configuration file that is customized to your cloud
@@ -37,7 +41,6 @@ $ ./setup_env
       server instead of the default Refstack API server.
    e. Adding --offline option will have your test results not be uploaded.
 
-
    **Upload:**
 
    If you previously ran a test with refstack-client using the --offline

refstack_client/refstack_client.py

@@ -37,6 +37,13 @@ from keystoneclient.v2_0 import client as ksclient
 from subunit_processor import SubunitProcessor
 
 
+def get_input():
+    """
+    Wrapper for raw_input. Necessary for testing.
+    """
+    return raw_input().lower()  # pragma: no cover
+
+
 class RefstackClient:
     log_format = "%(asctime)s %(name)s:%(lineno)d %(levelname)s %(message)s"
 
@@ -49,6 +56,7 @@ class RefstackClient:
         self.logger.addHandler(self.console_log_handle)
 
         self.args = args
+        self.tempest_dir = '.tempest'
 
         if self.args.verbose > 1:
             self.logger.setLevel(logging.DEBUG)
@@ -66,18 +74,18 @@ class RefstackClient:
             exit(1)
 
         # Check that the Tempest directory is an existing directory.
-        if not os.path.isdir(self.args.tempest_dir):
+        if not os.path.isdir(self.tempest_dir):
             self.logger.error("Tempest directory given is not a directory or "
-                              "does not exist: %s" % self.args.tempest_dir)
+                              "does not exist: %s" % self.tempest_dir)
             exit(1)
 
-        self.tempest_script = os.path.join(self.args.tempest_dir,
+        self.tempest_script = os.path.join(self.tempest_dir,
                                            'run_tempest.sh')
 
         self.conf_file = self.args.conf_file
         self.conf = ConfigParser.SafeConfigParser()
         self.conf.read(self.args.conf_file)
-        self.tempest_script = os.path.join(self.args.tempest_dir,
+        self.tempest_script = os.path.join(self.tempest_dir,
                                            'run_tempest.sh')
 
     def _prep_upload(self):
@@ -166,7 +174,7 @@ class RefstackClient:
         '''Execute Tempest test against the cloud.'''
         self._prep_test()
         results_file = self._get_next_stream_subunit_output_file(
-            self.args.tempest_dir)
+            self.tempest_dir)
         cpid = self._get_cpid_from_keystone(self.conf)
 
         self.logger.info("Starting Tempest test...")
@@ -175,7 +183,7 @@ class RefstackClient:
         # Run the tempest script, specifying the conf file, the flag
         # telling it to not use a virtual environment (-N), and the flag
         # telling it to run the tests serially (-t).
-        cmd = (self.tempest_script, '-C', self.conf_file, '-N', '-t')
+        cmd = (self.tempest_script, '-C', self.conf_file, '-V', '-t')
 
         # Add the tempest test cases to test as arguments. If no test
         # cases are specified, then all Tempest API tests will be run.
@@ -232,7 +240,7 @@ class RefstackClient:
 
 def parse_cli_args(args=None):
 
-    usage_string = ('refstack-client [-h] {upload,test} ...\n\n'
+    usage_string = ('refstack-client [-h] {upload,test,setup} ...\n\n'
                     'To see help on specific argument, do:\n'
                     'refstack-client <ARG> -h')
 
@@ -249,33 +257,29 @@ def parse_cli_args(args=None):
                              action='count',
                              help='Show verbose output.')
 
-    shared_args.add_argument('--url',
+    url_arg = argparse.ArgumentParser(add_help=False)
-                             action='store',
+    url_arg.add_argument('--url',
-                             required=False,
+                         action='store',
-                             default='https://api.refstack.org',
+                         required=False,
-                             type=str,
+                         default='https://api.refstack.org',
-                             help='Refstack API URL to upload results to '
+                         type=str,
-                                  '(--url https://127.0.0.1:8000).')
+                         help='Refstack API URL to upload results to '
+                              '(--url https://127.0.0.1:8000).')
 
     # Upload command
-    parser_upload = subparsers.add_parser('upload', parents=[shared_args],
+    parser_upload = subparsers.add_parser(
-                                          help='Upload an existing result '
+        'upload', parents=[shared_args, url_arg],
-                                               'file. ')
+        help='Upload an existing result file.'
+    )
     parser_upload.add_argument('file',
                                type=str,
                                help='Path of JSON results file.')
     parser_upload.set_defaults(func="upload")
 
     # Test command
-    parser_test = subparsers.add_parser('test', parents=[shared_args],
+    parser_test = subparsers.add_parser(
-                                        help='Run Tempest against a cloud.')
+        'test', parents=[shared_args, url_arg],
-
+        help='Run Tempest against a cloud.')
-    parser_test.add_argument('--tempest-dir',
-                             action='store',
-                             required=False,
-                             dest='tempest_dir',
-                             default='.venv/src/tempest',
-                             help='Path of the Tempest project directory.')
 
     parser_test.add_argument('-c', '--conf-file',
                              action='store',

refstack_client/tests/smoke/run_in_docker

@@ -2,4 +2,4 @@
 cp -r /refstack-client /test
 cd /test
 ./setup_env
-/bin/bash -c ". .venv/bin/activate; exec ./refstack-client -vv -c tempest.conf -t tempest.api.identity.admin.test_users"
+/bin/bash -c ". .venv/bin/activate; exec ./refstack-client test -vv -c tempest.conf -t tempest.api.identity.admin.test_users"

refstack_client/tests/smoke/test_distros.py

@@ -53,7 +53,7 @@ class TestSequenceFunctions(unittest.TestCase):
     def test_opensuse_13(self):
         # offcial opensuse image has outdated certificates
         # we can't use it while this issue isn't fixed
-        distro_image = 'mmckeen/opensuse-13-1'
+        distro_image = 'opensuse/13.2'
         self.run_test(distro_image)
 
 if __name__ == '__main__':

refstack_client/tests/unit/test_client.py

@@ -18,6 +18,7 @@ import logging
 import json
 import os
 import tempfile
+import subprocess
 
 import mock
 from mock import MagicMock
@@ -42,7 +43,7 @@ class TestRefstackClient(unittest.TestCase):
         self.addCleanup(patcher.stop)
         return thing
 
-    def mock_argv(self, conf_file_name=None, tempest_dir=None, verbose=None):
+    def mock_argv(self, conf_file_name=None, verbose=None):
         """
         Build argv for test.
         :param conf_file_name: Configuration file name
@@ -51,11 +52,8 @@ class TestRefstackClient(unittest.TestCase):
         """
         if conf_file_name is None:
             conf_file_name = self.conf_file_name
-        if tempest_dir is None:
-            tempest_dir = self.test_path
         argv = ['test',
                 '-c', conf_file_name,
-                '--tempest-dir', tempest_dir,
                 '--test-cases', 'tempest.api.compute',
                 '--url', '0.0.0.0']
         if verbose:
@@ -89,16 +87,19 @@ class TestRefstackClient(unittest.TestCase):
         """
         args = rc.parse_cli_args(self.mock_argv())
         client = rc.RefstackClient(args)
+        client.tempest_dir = self.test_path
         client._prep_test()
         self.assertEqual(client.logger.level, logging.ERROR)
 
         args = rc.parse_cli_args(self.mock_argv(verbose='-v'))
         client = rc.RefstackClient(args)
+        client.tempest_dir = self.test_path
         client._prep_test()
         self.assertEqual(client.logger.level, logging.INFO)
 
         args = rc.parse_cli_args(self.mock_argv(verbose='-vv'))
         client = rc.RefstackClient(args)
+        client.tempest_dir = self.test_path
         client._prep_test()
         self.assertEqual(client.logger.level, logging.DEBUG)
 
@@ -109,6 +110,7 @@ class TestRefstackClient(unittest.TestCase):
         """
         args = rc.parse_cli_args(self.mock_argv())
         client = rc.RefstackClient(args)
+        client.tempest_dir = self.test_path
         output_file = client._get_next_stream_subunit_output_file(
             self.test_path)
 
@@ -134,6 +136,7 @@ class TestRefstackClient(unittest.TestCase):
         """
         args = rc.parse_cli_args(self.mock_argv())
         client = rc.RefstackClient(args)
+        client.tempest_dir = self.test_path
         client._prep_test()
         self.mock_keystone()
         cpid = client._get_cpid_from_keystone(client.conf)
@@ -149,6 +152,7 @@ class TestRefstackClient(unittest.TestCase):
         """
         args = rc.parse_cli_args(self.mock_argv())
         client = rc.RefstackClient(args)
+        client.tempest_dir = self.test_path
         client._prep_test()
         client.conf.remove_option('identity', 'admin_tenant_id')
         client.conf.set('identity', 'admin_tenant_name', 'admin_tenant_name')
@@ -167,6 +171,7 @@ class TestRefstackClient(unittest.TestCase):
         """
         args = rc.parse_cli_args(self.mock_argv(verbose='-vv'))
         client = rc.RefstackClient(args)
+        client.tempest_dir = self.test_path
         client._prep_test()
         client.conf.remove_option('identity', 'admin_tenant_id')
         self.assertRaises(SystemExit, client._get_cpid_from_keystone,
@@ -221,7 +226,7 @@ class TestRefstackClient(unittest.TestCase):
         """
         args = rc.parse_cli_args(self.mock_argv(verbose='-vv'))
         client = rc.RefstackClient(args)
-
+        client.tempest_dir = self.test_path
         mock_popen = self.patch(
             'refstack_client.refstack_client.subprocess.Popen',
             return_value=MagicMock(returncode=0))
@@ -232,7 +237,7 @@ class TestRefstackClient(unittest.TestCase):
         client.test()
         mock_popen.assert_called_with(
             ('%s/run_tempest.sh' % self.test_path, '-C', self.conf_file_name,
-             '-N', '-t', '--', 'tempest.api.compute'),
+             '-V', '-t', '--', 'tempest.api.compute'),
             stderr=None
         )
 
@@ -249,7 +254,7 @@ class TestRefstackClient(unittest.TestCase):
         argv.append('--offline')
         args = rc.parse_cli_args(argv)
         client = rc.RefstackClient(args)
-
+        client.tempest_dir = self.test_path
         mock_popen = self.patch(
             'refstack_client.refstack_client.subprocess.Popen',
             return_value=MagicMock(returncode=0))
@@ -260,7 +265,7 @@ class TestRefstackClient(unittest.TestCase):
         client.test()
         mock_popen.assert_called_with(
             ('%s/run_tempest.sh' % self.test_path, '-C', self.conf_file_name,
-             '-N', '-t', '--', 'tempest.api.compute'),
+             '-V', '-t', '--', 'tempest.api.compute'),
             stderr=None
         )
 
@@ -280,7 +285,7 @@ class TestRefstackClient(unittest.TestCase):
         """
         Test when a nonexistent Tempest directory is passed in.
         """
-        args = rc.parse_cli_args(self.mock_argv(tempest_dir='/does/not/exist'))
+        args = rc.parse_cli_args(self.mock_argv())
         client = rc.RefstackClient(args)
         self.assertRaises(SystemExit, client.test)
 
@@ -293,6 +298,7 @@ class TestRefstackClient(unittest.TestCase):
         self.mock_keystone()
         args = rc.parse_cli_args(self.mock_argv(verbose='-vv'))
         client = rc.RefstackClient(args)
+        client.tempest_dir = self.test_path
         client.logger.error = MagicMock()
         client.test()
         self.assertTrue(client.logger.error.called)
@@ -324,3 +330,63 @@ class TestRefstackClient(unittest.TestCase):
                                   '--url', 'http://api.test.org'])
         client = rc.RefstackClient(args)
         self.assertRaises(SystemExit, client.upload)
+
+    def _set_mocks_for_setup(self):
+        """
+        Setup mocks for testing setup command in positive case
+        """
+        env = dict()
+        env['args'] = rc.parse_cli_args(['setup', '-r', 'havana-eol',
+                                         '--tempest-dir', '/tmp/tempest'])
+        env['raw_input'] = self.patch(
+            'refstack_client.refstack_client.get_input',
+            return_value='yes'
+        )
+        env['exists'] = self.patch(
+            'refstack_client.refstack_client.os.path.exists',
+            return_value=True
+        )
+        env['rmtree'] = self.patch(
+            'refstack_client.refstack_client.shutil.rmtree',
+            return_value=True
+        )
+        env['test_commit_sha'] = '42'
+        env['tag'] = MagicMock(
+            **{'commit.hexsha': env['test_commit_sha']}
+        )
+        env['tag'].configure_mock(name='havana-eol')
+        env['git.reset'] = MagicMock()
+        env['repo'] = MagicMock(
+            tags=[env['tag']],
+            **{'git.reset': env['git.reset']}
+        )
+        self.patch(
+            'refstack_client.refstack_client.git.Repo.clone_from',
+            return_value=env['repo']
+        )
+        env['os.chdir'] = self.patch(
+            'refstack_client.refstack_client.os.chdir'
+        )
+        env['subprocess.check_output'] = self.patch(
+            'refstack_client.refstack_client.subprocess.check_output',
+            return_value='Ok!'
+        )
+        return env
+
+    def _check_mocks_for_setup(self, env):
+        """
+        Check mocks after successful run 'setup' command
+        """
+        env['exists'].assert_called_once_with('/tmp/tempest')
+        env['rmtree'].assert_called_once_with('/tmp/tempest')
+        env['git.reset'].assert_called_once_with(
+            env['test_commit_sha'], hard=True
+        )
+        env['os.chdir'].assert_has_calls([mock.call('/tmp/tempest'),
+                                          mock.call(os.getcwd())])
+        env['subprocess.check_output'].assert_has_calls([
+            mock.call(['virtualenv', '.venv'],
+                      stderr=subprocess.STDOUT),
+            mock.call(['.venv//bin//pip', 'install', '-r', 'requirements.txt'],
+                      stderr=subprocess.STDOUT)
+        ])

requirements.txt

@@ -1 +1,3 @@
--e git+https://github.com/openstack/tempest.git#egg=tempest
+python-keystoneclient>=0.10.0
+gitpython>=0.3.2.RC1
+python-subunit>=0.0.18

setup_env

@@ -1,74 +1,143 @@
-#!/bin/sh
+#!/bin/bash
-# Setup binary requirements
+
+# Prints help
+function usage {
+    SCRIPT_NAME="basename ${BASH_SOURCE[0]}"
+    echo "Usage: ${SCRIPT_NAME} [OPTION]..."
+    echo "Setup Refstack client with test environment"
+    echo ""
+    echo "  -h,                      Print this usage message"
+    echo "  -t                       Tempest test runner tag release. Default is latest tag"
+}
+
+#Check that parameter is a valid tag in tempest repository
+function check_tag {
+        tags="$(git tag)"
+        for tag in ${tags}; do
+                [[ "${tag}" == "$1" ]] && return 0;
+        done
+        return 1
+}
+
+#Install git
 if [ -n "$(command -v apt-get)" ]; then
-# For apt-get-based Linux distributions (Ubuntu, Debian)
+    # For apt-get-based Linux distributions (Ubuntu, Debian)
     # If we run script in container we need sudo
-    apt-get update
     if [ ! -n "$(command -v sudo)" ]; then
+        apt-get update
         apt-get -y install sudo
-    fi
+    else
-    # Workaroud for Ubuntu 10.04.
-    # There is no package 'git' in default ubuntu 10.04 repo's
-    sudo apt-get -s install git
-    if [ $? = 100 ]; then
-        sudo apt-get -y install python-software-properties
-        sudo add-apt-repository ppa:git-core/ppa
         sudo apt-get update
     fi
-    sudo apt-get -y install git curl wget tar unzip python-dev build-essential libssl-dev libxslt-dev libsasl2-dev libffi-dev libbz2-dev
+    sudo apt-get -y install git
-
-    # Setup pip for python
-    curl https://bootstrap.pypa.io/get-pip.py | sudo -E python -
 elif [ -n "$(command -v yum)" ]; then
-# For yum-based distributions (RHEL, Centos)
+    # For yum-based distributions (RHEL, Centos)
     # If we run script in container we need sudo
     if [ ! -f sudo ]; then
         yum -y install sudo
     fi
-    sudo yum -y install git curl wget tar unzip make python-devel.x86_64 gcc gcc-c++ libffi-devel libxml2-devel bzip2-devel libxslt-devel openssl-devel
+    sudo yum -y install git
-
-    # Setup pip for python
-    curl https://bootstrap.pypa.io/get-pip.py | sudo -E python -
 elif [ -n "$(command) -v zypper" ]; then
-# For zypper-based distributions (openSuSe, SELS)
+    # For zypper-based distributions (openSuSe, SELS)
     # If we run script in container we need sudo
     if [ ! -f sudo ]; then
-        zypper --no-gpg-checks --non-interactive install sudo
+        zypper --gpg-auto-import-keys --non-interactive refresh
+        zypper --non-interactive install sudo
+    else
+        sudo zypper --gpg-auto-import-keys --non-interactive refresh
     fi
-    sudo zypper --no-gpg-checks --non-interactive refresh
+    sudo zypper --non-interactive install git
-    sudo zypper --non-interactive install git curl wget tar unzip make python-devel.x86_64 gcc gcc-c++ libffi-devel libxml2-devel zlib-devel libxslt-devel libopenssl-devel
-
-    # Setup pip for python
-    sudo zypper --non-interactive install python-pip
 else
     echo "Neither apt-get nor yum nor zypper found"
     exit 1
 fi
 
+WORKDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+#Checkout tempest on spceified tag
+if [ -d "${WORKDIR}/.tempest" ]; then
+    while true; do
+        read -p "Installed tempest found.We should remove it. All data from previouse test runs will be deleted. Continue (y/n) ?" yn
+        case ${yn} in
+            [Yy]* ) rm -rf ${WORKDIR}/.tempest; break;;
+            [Nn]* ) exit 1;;
+            * ) echo "Please answer yes or no.";;
+        esac
+    done
+fi
+git clone https://github.com/openstack/tempest.git ${WORKDIR}/.tempest
+cd ${WORKDIR}/.tempest
+TAG="$(git describe --abbrev=0 --tags)"
+while getopts t:h FLAG; do
+    case ${FLAG} in
+        t)
+            TAG=$OPTARG
+            if check_tag TAG ; then
+                echo -e \\n"Tag $TAG not found in tempest github repository"
+                exit 1
+            fi
+            ;;
+        h)  #show help
+            usage
+            ;;
+        \?) #unrecognized option - show help
+            echo -e \\n"Option -$OPTARG not allowed."
+            usage
+            ;;
+    esac
+done
+shift $((OPTIND-1))  #This tells getopts to move on to the next argument.
+git checkout -q tags/${TAG}
+cd ${WORKDIR}
+
+# Setup binary requirements
+if [ -n "$(command -v apt-get)" ]; then
+    # For apt-get-based Linux distributions (Ubuntu, Debian)
+    sudo apt-get -y install curl wget tar unzip python-dev build-essential libssl-dev libxslt-dev libsasl2-dev libffi-dev libbz2-dev
+elif [ -n "$(command -v yum)" ]; then
+    # For yum-based distributions (RHEL, Centos)
+    sudo yum -y install curl wget tar unzip make python-devel.x86_64 gcc gcc-c++ libffi-devel libxml2-devel bzip2-devel libxslt-devel openssl-devel
+elif [ -n "$(command) -v zypper" ]; then
+    # For zypper-based distributions (openSuSe, SELS)
+    sudo zypper --non-interactive install curl wget tar unzip make python-devel.x86_64 gcc gcc-c++ libffi-devel libxml2-devel zlib-devel libxslt-devel libopenssl-devel python-xml
+else
+    echo "Neither apt-get nor yum nor zypper found"
+    exit 1
+fi
+
+#Build local python interpreter if needed
 if [ ! -n "$(command -v python2.7)" ]; then
-    echo "python2.7 not found. Building one..."
+    PY_VERSION="2.7.8"
-    WORKDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+    echo "python2.7 not found. Building python ${PY_VERSION}..."
     mkdir ${WORKDIR}/.localpython
     mkdir ${WORKDIR}/.python_src
     cd ${WORKDIR}/.python_src
-    wget http://www.python.org/ftp/python/2.7.8/Python-2.7.8.tgz
+    wget http://www.python.org/ftp/python/${PY_VERSION}/Python-${PY_VERSION}.tgz
-    tar -zxvf Python-2.7.8.tgz
+    tar zxvf Python-${PY_VERSION}.tgz
-    cd Python-2.7.8
+    cd Python-${PY_VERSION}
 
     ./configure --prefix=${WORKDIR}/.localpython
     make && make install
     cd ${WORKDIR}
+    rm -rf ${WORKDIR}/.python_src
     PYPATH="${WORKDIR}/.localpython/bin/python"
 else
 echo "python2.7 found!"
 PYPATH="python2.7"
 fi
 
-# Setup python requirements
+#Setup virtual environments for refstack-client and tempest
-sudo pip install virtualenv
+VENV_VERSION='1.11.6'
-# Create virtual environment
+wget https://pypi.python.org/packages/source/v/virtualenv/virtualenv-${VENV_VERSION}.tar.gz
-if [ ! -d .venv ]; then
+tar xvfz virtualenv-${VENV_VERSION}.tar.gz
-    virtualenv .venv --python="$PYPATH"
+cd virtualenv-${VENV_VERSION}
+if [ -d ${WORKDIR}/.venv ]; then
+    rm -rf ${WORKDIR}/.venv
 fi
-# Setup virtual environment
+python virtualenv.py ${WORKDIR}/.venv --python="${PYPATH}"
-.venv/bin/pip install -r requirements.txt
+python virtualenv.py ${WORKDIR}/.tempest/.venv --python="${PYPATH}"
+cd ..
+rm -rf virtualenv-${VENV_VERSION}
+rm virtualenv-${VENV_VERSION}.tar.gz
+${WORKDIR}/.venv/bin/pip install -r ${WORKDIR}/requirements.txt
+${WORKDIR}/.tempest/.venv/bin/pip install -r ${WORKDIR}/.tempest/requirements.txt

test-requirements.txt

@@ -1,7 +1,6 @@
 pep8==1.4.5
 pyflakes>=0.7.2,<0.7.4
 flake8==2.0
-python-subunit>=0.0.18
 # required to build documentation
 sphinx>=1.1.2,!=1.2.0,<1.3
 testrepository>=0.0.18