Launch functional test

* Functional test is used to test deployment, samples etc.
* Remove `tox -e cli` and merge test cases into functional test
* Remove unused test cases

Change-Id: I87832278551da788e53984c7c4d90ea5775b53b9

.zuul.d/rally-tox-functional.yaml

@@ -0,0 +1,21 @@
+- job:
+    name: rally-dsvm-tox-functional
+    parent: devstack
+    description:
+      Run functional test for rally-openstack project.
+      Uses tox with the ``functional`` environment.
+    required-projects:
+      - name: openstack-dev/devstack
+      - name: openstack-infra/devstack-gate
+      - name: openstack/rally-openstack
+      # NOTE(andreykurilin): it is a required project to fetch the latest
+      #   version and test master of rally-openstack with master of rally
+      - name: openstack/rally
+    timeout: 7200
+    roles:
+      - zuul: openstack-dev/devstack
+    vars:
+      devstack_plugins:
+        rally-openstack: https://git.openstack.org/openstack/rally-openstack
+      rally_tox_env: "functional"
+    run: tests/ci/playbooks/run-rally-tox.yaml

.zuul.d/zuul.yaml

@@ -48,6 +48,7 @@
         - rally-tox-py36
         - rally-tox-py37
         - rally-tox-cover
+        - rally-dsvm-tox-functional
         - rally-docker-check
         - rally-task-simple-job
         - rally-task-basic-with-existing-users:
@@ -99,6 +100,7 @@
         - rally-tox-py36
         - rally-tox-py37
         - rally-tox-cover
+        - rally-dsvm-tox-functional
         - rally-docker-check
         - rally-task-simple-job
         #- rally-task-heat

tests/check_samples/utils.py

@@ -1,32 +0,0 @@
-# All Rights Reserved.
-#
-#    Licensed under the Apache License, Version 2.0 (the "License"); you may
-#    not use this file except in compliance with the License. You may obtain
-#    a copy of the License at
-#
-#         http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-#    License for the specific language governing permissions and limitations
-#    under the License.
-
-import os
-import subprocess
-
-from tests.functional import utils
-
-
-DEPLOYMENT_FILE = "/tmp/rally_functests_main_deployment.json"
-
-
-class Rally(utils.Rally):
-
-    def __init__(self, force_new_db=False):
-        self._DEPLOYMENT_CREATE_ARGS = " --file %s" % DEPLOYMENT_FILE
-        if not os.path.exists(DEPLOYMENT_FILE):
-            subprocess.call(["rally", "--log-file", "/dev/null",
-                             "deployment", "config"],
-                            stdout=open(DEPLOYMENT_FILE, "w"))
-        super(Rally, self).__init__(force_new_db=force_new_db)

tests/ci/playbooks/roles/prepare-for-rally-task/tasks/main.yaml

@@ -175,10 +175,16 @@
           nova flavor-create m1.nano 42 64 0 1
       fi
 
+- name: Check the existence of rally_task
+  stat:
+    path: '{{ zuul.project.src_dir }}/{{ rally_task }}'
+  register: rally_task_file_stat
+
 - name: Copy task file
   become: True
   become_user: stack
   command: cp "{{ zuul.project.src_dir }}/{{ rally_task }}" "{{ rally_home_dir }}/task.yaml"
+  when: rally_task_file_stat.stat.exists == True
 
 - name: Check the existence of task_args_file
   stat:

tests/ci/playbooks/roles/run-rally-tox/tasks/main.yaml

@@ -0,0 +1,10 @@
+- name: Run tox command
+  become: True
+  become_user: stack
+  shell:
+    executable: /bin/sh
+    cmd: |
+      set -e
+      cd {{ zuul.project.src_dir }}
+      export REQUESTS_CA_BUNDLE=/opt/stack/data/ca-bundle.pem
+      tox -e {{ rally_tox_env }}

tests/ci/playbooks/run-rally-tox.yaml

@@ -0,0 +1,15 @@
+- hosts: all
+  roles:
+    - run-devstack
+
+- hosts: controller
+  vars:
+    rally_home_dir: '/opt/stack/.rally'
+    rally_fake_image_path: '{{ rally_home_dir }}/extra/fake-image.img'
+    rally_use_existing_users: false
+    rally_existing_users_config: '{{ rally_home_dir }}/with-existing-users-config'
+    rally_task_args_file: "100-percent-not-exist-file"
+    rally_task: "100-percent-not-exist-file"
+  roles:
+    - prepare-for-rally-task
+    - run-rally-tox

tests/ci/rally_functional_job.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+LOCAL_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+DB_CONNECTION="$(rally db show)"
+
+if [[ $DB_CONNECTION == sqlite* ]]; then
+    CONCURRENCY=0
+else
+    # in case of not sqlite db backends we cannot launch tests in parallel due
+    # to possible conflicts
+    CONCURRENCY=1
+    # currently, RCI_KEEP_DB variable is used to not create new databases per
+    # each test
+    export RCI_KEEP_DB=1
+fi
+
+python $LOCAL_DIR/pytest_launcher.py "tests/functional" --concurrency $CONCURRENCY --posargs=$1

tests/functional/extra/fake_dir1/fake_plugin1.py

@@ -1,24 +0,0 @@
-# Copyright 2015: Mirantis Inc.
-# All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-from rally.task import scenario
-
-
-@scenario.configure(name="FakeScenarioPlugin1.list")
-class FakeScenarioPlugin1(scenario.Scenario):
-
-    def run(self):
-        """Sample fake scenario."""
-        pass

tests/functional/extra/fake_dir2/fake_plugin2.py

@@ -1,24 +0,0 @@
-# Copyright 2015: Mirantis Inc.
-# All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-from rally.task import scenario
-
-
-@scenario.configure(name="FakeScenarioPlugin2.list")
-class FakeScenarioPlugin2(scenario.Scenario):
-
-    def run(self):
-        """Sample fake scenario."""
-        pass

tests/functional/extra/fake_platforms.py

@@ -1,43 +0,0 @@
-# All Rights Reserved.
-#
-#    Licensed under the Apache License, Version 2.0 (the "License"); you may
-#    not use this file except in compliance with the License. You may obtain
-#    a copy of the License at
-#
-#         http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-#    License for the specific language governing permissions and limitations
-#    under the License.
-
-from rally.env import platform
-
-
-@platform.configure(name="good", platform="fake")
-class GoodPlatform(platform.Platform):
-
-    CONFIG_SCHEMA = {}
-
-    def create(self):
-        return {}, {}
-
-    def destroy(self):
-        pass
-
-    def cleanup(self, task_uuid=None):
-        return {
-            "message": "Coming soon!",
-            "discovered": 0,
-            "deleted": 0,
-            "failed": 0,
-            "resources": {},
-            "errors": []
-        }
-
-    def check_health(self):
-        return {"available": True}
-
-    def info(self):
-        return {"info": {"a": 1}}

tests/functional/extra/test_fake_scenario.json

@@ -1,18 +0,0 @@
-{
-    "FakeScenarioPlugin1.list": [
-        {
-            "runner": {
-                "type": "constant",
-                "times": 5
-            }
-        }
-    ],
-    "FakeScenarioPlugin2.list": [
-        {
-            "runner": {
-                "type": "constant",
-                "times": 5
-            }
-        }
-    ]
-}

tests/functional/test_certification_task.py

@@ -19,18 +19,14 @@ import traceback
 import unittest
 
 import rally_openstack
-from tests.check_samples import utils
+from tests.functional import utils
 
 
 class TestPreCreatedTasks(unittest.TestCase):
 
-    def test_check_success(self):
-        rally = utils.Rally()
-        rally("deployment check")
-
     def test_task_samples_is_valid(self):
 
-        rally = utils.Rally()
+        rally = utils.RallyWithSpecifiedDeployment()
         full_path = os.path.join(
             os.path.dirname(rally_openstack.__file__), os.pardir,
             "tasks", "openstack")

tests/functional/test_cli_deployment.py

@@ -51,11 +51,6 @@ class DeploymentTestCase(unittest.TestCase):
               "--filename /tmp/.tmp.deployment")
         self.assertIn("t_create_file", rally("deployment list"))
 
-    def test_create_empty(self):
-        rally = utils.Rally()
-        rally("deployment create --name t_empty")
-        self.assertEqual("{}", rally("deployment config").strip())
-
     def test_destroy(self):
         rally = utils.Rally()
         rally.env.update(TEST_ENV)
@@ -120,9 +115,6 @@ class DeploymentTestCase(unittest.TestCase):
                                               rally.env)
         self.assertEqual(uuid, current_deployment)
 
-    # TODO(andreykurilin): Do not forget to move thes tests while splitting
-    #   rally to main framework and openstack plugins
-
     def test_create_from_env_openstack_deployment(self):
         rally = utils.Rally()
         rally.env.update(TEST_ENV)

tests/functional/test_cli_env.py

@@ -1,92 +0,0 @@
-# Copyright 2013: ITLook, Inc.
-# All Rights Reserved.
-#
-#    Licensed under the Apache License, Version 2.0 (the "License"); you may
-#    not use this file except in compliance with the License. You may obtain
-#    a copy of the License at
-#
-#         http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-#    License for the specific language governing permissions and limitations
-#    under the License.
-
-import json
-import os
-import tempfile
-
-import unittest
-
-from tests.functional import utils
-
-
-class EnvTestCase(unittest.TestCase):
-
-    def test_create_no_spec(self):
-        rally = utils.Rally()
-        rally("env create --name empty --description de")
-        self.assertIn("empty", rally("env list"))
-        env_data = rally("env show --json", getjson=True)
-        self.assertEqual("empty", env_data["name"])
-        self.assertEqual("de", env_data["description"])
-        self.assertEqual({}, env_data["extras"])
-        self.assertEqual({}, env_data["platforms"])
-
-    def _create_spec(self, spec):
-        f = tempfile.NamedTemporaryFile(delete=False)
-
-        def unlink():
-            os.unlink(f.name)
-
-        self.addCleanup(unlink)
-
-        f.write(json.dumps(spec, indent=2))
-        f.close()
-        return f.name
-
-    def test_create_check_info_destroy_delete_with_spec(self):
-        rally = utils.Rally(plugin_path="tests/functional/extra")
-
-        spec = self._create_spec({"good@fake": {}})
-        rally("env create --name real --spec %s" % spec)
-        env = rally("env show --json", getjson=True)
-        self.assertIn("fake", env["platforms"])
-
-        env_info = rally("env info --json", getjson=True)
-        self.assertEqual({"good@fake": {"info": {"a": 1}}}, env_info)
-
-        rally("env check --json")
-
-    def test_list_empty(self):
-        rally = utils.Rally()
-        # TODO(boris-42): Clean this up somehow
-        rally("env destroy MAIN")
-        rally("env delete MAIN")
-        self.assertEqual([], rally("env list --json", getjson=True))
-        self.assertIn("There are no environments", rally("env list"))
-
-    def test_list(self):
-        rally = utils.Rally()
-        envs = rally("env list --json", getjson=True)
-        self.assertEqual(1, len(envs))
-        self.assertEqual("MAIN", envs[0]["name"])
-        self.assertIn("MAIN", rally("env list"))
-
-    def test_use(self):
-
-        def show_helper():
-            return rally("env show --json", getjson=True)
-
-        rally = utils.Rally()
-        self.assertEqual("MAIN", show_helper()["name"])
-        empty_uuid = rally("env create --name empty --json",
-                           getjson=True)["uuid"]
-        self.assertEqual("empty", show_helper()["name"])
-        rally("env use MAIN")
-        self.assertEqual("MAIN", show_helper()["name"])
-        rally("env use %s" % empty_uuid)
-        self.assertEqual("empty", show_helper()["name"])
-        rally("env create --name empty2 --description de --no-use")
-        self.assertEqual("empty", show_helper()["name"])

tests/functional/test_cli_plugin.py

@@ -1,72 +0,0 @@
-# Copyright 2015: Mirantis Inc.
-# All Rights Reserved.
-#
-#    Licensed under the Apache License, Version 2.0 (the "License"); you may
-#    not use this file except in compliance with the License. You may obtain
-#    a copy of the License at
-#
-#         http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-#    License for the specific language governing permissions and limitations
-#    under the License.
-
-import unittest
-
-from tests.functional import utils
-
-
-class PluginTestCase(unittest.TestCase):
-
-    def test_show_one(self):
-        rally = utils.Rally()
-        result = rally("plugin show Dummy.dummy")
-        self.assertIn("NAME", result)
-        self.assertIn("PLATFORM", result)
-        self.assertIn("Dummy.dummy", result)
-        self.assertIn("MODULE", result)
-
-    def test_show_multiple(self):
-        rally = utils.Rally()
-        result = rally("plugin show Dummy")
-        self.assertIn("Multiple plugins found:", result)
-        self.assertIn("Dummy.dummy", result)
-        self.assertIn("Dummy.dummy_exception", result)
-        self.assertIn("Dummy.dummy_random_fail_in_atomic", result)
-
-    def test_show_not_found(self):
-        rally = utils.Rally()
-        name = "Dummy666666"
-        result = rally("plugin show %s" % name)
-        self.assertIn("Plugin %s not found" % name, result)
-
-    def test_show_not_found_in_specific_platform(self):
-        rally = utils.Rally()
-        name = "Dummy"
-        platform = "non_existing"
-        result = rally(
-            "plugin show --name %(name)s --platform %(platform)s"
-            % {"name": name, "platform": platform})
-        self.assertIn(
-            "Plugin %(name)s@%(platform)s not found"
-            % {"name": name, "platform": platform},
-            result)
-
-    def test_list(self):
-        rally = utils.Rally()
-        result = rally("plugin list Dummy")
-        self.assertIn("Dummy.dummy", result)
-        self.assertIn("Dummy.dummy_exception", result)
-        self.assertIn("Dummy.dummy_random_fail_in_atomic", result)
-
-    def test_list_not_found_platform(self):
-        rally = utils.Rally()
-        result = rally("plugin list --platform some")
-        self.assertIn("Platform some not found", result)
-
-    def test_list_not_found_name(self):
-        rally = utils.Rally()
-        result = rally("plugin list Dummy2222")
-        self.assertIn("Plugin Dummy2222 not found", result)

tests/functional/test_cli_verify.py

@@ -1,29 +0,0 @@
-# Copyright 2014: Mirantis Inc.
-# All Rights Reserved.
-#
-#    Licensed under the Apache License, Version 2.0 (the "License"); you may
-#    not use this file except in compliance with the License. You may obtain
-#    a copy of the License at
-#
-#         http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-#    License for the specific language governing permissions and limitations
-#    under the License.
-
-##############################################################################
-#
-#                        THIS MODULE IS DEPRECATED.
-#                 DON'T ADD TESTS FOR "rally verify" HERE.
-#
-# This module is no longer used for testing "rally verify" command.
-# Functional testing for this command is moved to separate job.
-# https://review.openstack.org/#/c/137232
-#
-# Please look at tests/ci/rally-verify.sh for more details.
-#
-##############################################################################
-
-pass

tests/functional/test_lib_api.py

@@ -1,30 +0,0 @@
-# Copyright 2016: Mirantis Inc.
-# All Rights Reserved.
-#
-#    Licensed under the Apache License, Version 2.0 (the "License"); you may
-#    not use this file except in compliance with the License. You may obtain
-#    a copy of the License at
-#
-#         http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-#    License for the specific language governing permissions and limitations
-#    under the License.
-
-import os
-import subprocess
-import unittest
-
-from tests.functional import utils
-
-
-class LibAPITestCase(unittest.TestCase):
-
-    def test_rally_lib(self):
-        rally = utils.Rally(force_new_db=True)
-        cdir = os.path.dirname(os.path.realpath(__file__))
-        app = os.path.join(cdir, "../ci/rally_app.py")
-        subprocess.check_output(
-            ["python", app, "--config-file", rally.config_filename])

tests/functional/test_task_samples.py

@@ -24,13 +24,12 @@ import unittest
 
 import six
 
-import rally as rally_module
 from rally import api
 from rally.common import broker
 from rally.common import yamlutils as yaml
 from rally import plugins
-from rally_openstack.contexts.keystone import users
+import rally_openstack as rally_openstack_module
-from tests.check_samples import utils
+from tests.functional import utils
 
 
 class TestTaskSamples(unittest.TestCase):
@@ -56,8 +55,8 @@ class TestTaskSamples(unittest.TestCase):
 
     @plugins.ensure_plugins_are_loaded
     def test_task_samples_are_valid(self):
-        rally = utils.Rally(force_new_db=True)
+        from rally_openstack.contexts.keystone import users
-
+        rally = utils.RallyWithSpecifiedDeployment(force_new_db=True)
         # let's use pre-created users to make TestTaskSamples quicker
         rapi = api.API(config_file=rally.config_filename)
         deployment = rapi.deployment._get("MAIN")
@@ -120,7 +119,7 @@ class TestTaskSamples(unittest.TestCase):
         def publisher(queue):
             """List all samples and render task configs"""
             samples_path = os.path.join(
-                os.path.dirname(rally_module.__file__), os.pardir,
+                os.path.dirname(rally_openstack_module.__file__), os.pardir,
                 "samples", "tasks")
 
             for dirname, dirnames, filenames in os.walk(samples_path):

tests/functional/utils.py

@@ -215,6 +215,20 @@ class Rally(object):
                     rep.write("%s\n" % output)
 
 
+class RallyWithSpecifiedDeployment(Rally):
+
+    DEPLOYMENT_FILE = "/tmp/rally_functests_main_deployment.json"
+
+    def __init__(self, force_new_db=False):
+        self._DEPLOYMENT_CREATE_ARGS = " --file %s" % self.DEPLOYMENT_FILE
+        if not os.path.exists(self.DEPLOYMENT_FILE):
+            subprocess.call(["rally", "--log-file", "/dev/null",
+                             "deployment", "config"],
+                            stdout=open(self.DEPLOYMENT_FILE, "w"))
+        super(RallyWithSpecifiedDeployment, self).__init__(
+            force_new_db=force_new_db)
+
+
 def get_global(global_key, env):
     home_dir = env.get("HOME")
     with open("%s/.rally/globals" % home_dir) as f:

tox.ini

@@ -24,7 +24,7 @@ commands =
   python {toxinidir}/tests/ci/pytest_launcher.py tests/unit --posargs={posargs}
 distribute = false
 basepython = python2.7
-passenv = http_proxy HTTP_PROXY https_proxy HTTPS_PROXY no_proxy NO_PROXY
+passenv = http_proxy HTTP_PROXY https_proxy HTTPS_PROXY no_proxy NO_PROXY REQUESTS_CA_BUNDLE
 
 [testenv:pep8]
 commands = flake8
@@ -60,12 +60,6 @@ commands = oslo_debug_helper -t tests {posargs}
 basepython = python3.5
 commands = oslo_debug_helper -t tests {posargs}
 
-[testenv:cli]
-sitepackages = True
-commands =
-  find . -type f -name "*.pyc" -delete
-  python {toxinidir}/tests/ci/pytest_launcher.py tests/check_samples --posargs={posargs}
-
 [testenv:functional]
 sitepackages = True
 commands =