[verification] Introduce Tempest verifier

Co-Authored-By: Yaroslav Lobankov <ylobankov@mirantis.com>
Change-Id: I06c7e1d2c7956a1ad92e62484d84141ee926be08

rally/consts.py

@@ -24,27 +24,6 @@ for each enum. (e.g TaskStatus)
 from rally.common import utils
 
 
-class _TempestTestsAPI(utils.ImmutableMixin, utils.EnumMixin):
-    BAREMETAL = "baremetal"
-    CLUSTERING = "clustering"
-    COMPUTE = "compute"
-    DATA_PROCESSING = "data_processing"
-    DATABASE = "database"
-    IDENTITY = "identity"
-    IMAGE = "image"
-    MESSAGING = "messaging"
-    NETWORK = "network"
-    OBJECT_STORAGE = "object_storage"
-    ORCHESTRATION = "orchestration"
-    TELEMETRY = "telemetry"
-    VOLUME = "volume"
-
-
-class _TempestTestsSets(utils.ImmutableMixin, utils.EnumMixin):
-    FULL = "full"
-    SMOKE = "smoke"
-    SCENARIO = "scenario"
-
 JSON_SCHEMA = "http://json-schema.org/draft-04/schema"
 
 
@@ -239,8 +218,6 @@ EndpointPermission = _EndpointPermission()
 ServiceType = _ServiceType()
 Service = _Service()
 EndpointType = _EndpointType()
-TempestTestsAPI = _TempestTestsAPI()
-TempestTestsSets = _TempestTestsSets()
 HookStatus = _HookStatus()
 TagType = _TagType()
 VerifierStatus = _VerifierStatus()

rally/plugins/openstack/verification/tempest/config.ini

@@ -0,0 +1,55 @@
+[DEFAULT]
+debug = True
+log_file = tempest.log
+use_stderr = False
+
+[auth]
+use_dynamic_credentials = True
+
+[compute]
+image_ref =
+image_ref_alt =
+flavor_ref =
+flavor_ref_alt =
+fixed_network_name =
+
+[compute-feature-enabled]
+live_migration = False
+resize = True
+vnc_console = True
+attach_encrypted_volume = False
+
+[data-processing]
+
+[identity]
+
+[image-feature-enabled]
+deactivate_image = True
+
+[input-scenario]
+ssh_user_regex = [["^.*[Cc]irros.*$", "cirros"], ["^.*[Tt]est[VvMm].*$", "cirros"], ["^.*rally_verify.*$", "cirros"]]
+
+[network]
+
+[network-feature-enabled]
+ipv6_subnet_attributes = True
+ipv6 = True
+
+[object-storage]
+
+[oslo_concurrency]
+
+[orchestration]
+instance_type =
+
+[scenario]
+img_file =
+
+[service_available]
+
+[validation]
+run_validation = True
+image_ssh_user = cirros
+
+[volume-feature-enabled]
+bootable = True

rally/plugins/openstack/verification/tempest/config.py

@@ -0,0 +1,594 @@
+# Copyright 2014: Mirantis Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import inspect
+import os
+import re
+
+from oslo_config import cfg
+import requests
+import six
+from six.moves import configparser
+from six.moves.urllib import parse
+
+from rally.common.i18n import _
+from rally.common import logging
+from rally.common import objects
+from rally.common import utils
+from rally import exceptions
+from rally import osclients
+from rally.plugins.openstack.wrappers import glance
+from rally.plugins.openstack.wrappers import network
+from rally.task import utils as task_utils
+from rally.verification import context
+
+
+LOG = logging.getLogger(__name__)
+
+
+TEMPEST_OPTS = [
+    cfg.StrOpt("img_url",
+               deprecated_opts=[cfg.DeprecatedOpt("cirros_img_url",
+                                                  group="image")],
+               default="http://download.cirros-cloud.net/"
+                       "0.3.4/cirros-0.3.4-x86_64-disk.img",
+               help="image URL"),
+    cfg.StrOpt("img_disk_format",
+               deprecated_opts=[cfg.DeprecatedOpt("disk_format",
+                                                  group="image")],
+               default="qcow2",
+               help="Image disk format to use when creating the image"),
+    cfg.StrOpt("img_container_format",
+               deprecated_opts=[cfg.DeprecatedOpt("container_format",
+                                                  group="image")],
+               default="bare",
+               help="Image container format to use when creating the image"),
+    cfg.StrOpt("img_name_regex",
+               deprecated_opts=[cfg.DeprecatedOpt("name_regex",
+                                                  group="image")],
+               default="^.*(cirros|testvm).*$",
+               help="Regular expression for name of a public image to "
+                    "discover it in the cloud and use it for the tests. "
+                    "Note that when Rally is searching for the image, case "
+                    "insensitive matching is performed. Specify nothing "
+                    "('img_name_regex =') if you want to disable discovering. "
+                    "In this case Rally will create needed resources by "
+                    "itself if the values for the corresponding config "
+                    "options are not specified in the Tempest config file"),
+    cfg.StrOpt("swift_operator_role",
+               deprecated_group="role",
+               default="Member",
+               help="Role required for users "
+                    "to be able to create Swift containers"),
+    cfg.StrOpt("swift_reseller_admin_role",
+               deprecated_group="role",
+               default="ResellerAdmin",
+               help="User role that has reseller admin"),
+    cfg.StrOpt("heat_stack_owner_role",
+               deprecated_group="role",
+               default="heat_stack_owner",
+               help="Role required for users "
+                    "to be able to manage Heat stacks"),
+    cfg.StrOpt("heat_stack_user_role",
+               deprecated_group="role",
+               default="heat_stack_user",
+               help="Role for Heat template-defined users"),
+    cfg.IntOpt("flavor_ref_ram",
+               default="64",
+               help="Primary flavor RAM size used by most of the test cases"),
+    cfg.IntOpt("flavor_ref_alt_ram",
+               default="128",
+               help="Alternate reference flavor RAM size used by test that"
+               "need two flavors, like those that resize an instance"),
+    cfg.IntOpt("heat_instance_type_ram",
+               default="64",
+               help="RAM size flavor used for orchestration test cases")
+]
+
+CONF = cfg.CONF
+CONF.register_opts(TEMPEST_OPTS, "tempest")
+CONF.import_opt("glance_image_delete_timeout",
+                "rally.plugins.openstack.scenarios.glance.utils",
+                "benchmark")
+CONF.import_opt("glance_image_delete_poll_interval",
+                "rally.plugins.openstack.scenarios.glance.utils",
+                "benchmark")
+
+
+def _create_or_get_data_dir():
+    data_dir = os.path.join(
+        os.path.expanduser("~"), ".rally", "verification", "tempest", "data")
+    if not os.path.exists(data_dir):
+        os.makedirs(data_dir)
+
+    return data_dir
+
+
+def _download_image(image_path, image=None):
+    if image:
+        LOG.debug("Downloading image '%s' "
+                  "from Glance to %s" % (image.name, image_path))
+        with open(image_path, "wb") as image_file:
+            for chunk in image.data():
+                image_file.write(chunk)
+    else:
+        LOG.debug("Downloading image from %s "
+                  "to %s" % (CONF.tempest.img_url, image_path))
+        try:
+            response = requests.get(CONF.tempest.img_url, stream=True)
+        except requests.ConnectionError as err:
+            msg = _("Failed to download image. "
+                    "Possibly there is no connection to Internet. "
+                    "Error: %s.") % (str(err) or "unknown")
+            raise exceptions.TempestConfigCreationFailure(msg)
+
+        if response.status_code == 200:
+            with open(image_path, "wb") as image_file:
+                for chunk in response.iter_content(chunk_size=1024):
+                    if chunk:   # filter out keep-alive new chunks
+                        image_file.write(chunk)
+                        image_file.flush()
+        else:
+            if response.status_code == 404:
+                msg = _("Failed to download image. Image was not found.")
+            else:
+                msg = _("Failed to download image. "
+                        "HTTP error code %d.") % response.status_code
+            raise exceptions.TempestConfigCreationFailure(msg)
+
+    LOG.debug("The image has been successfully downloaded!")
+
+
+def write_configfile(path, conf_object):
+    with open(path, "w") as configfile:
+        conf_object.write(configfile)
+
+
+def read_configfile(path):
+    with open(path) as f:
+        return f.read()
+
+
+def add_extra_options(extra_options, conf_object):
+    for section in extra_options:
+        if section not in (conf_object.sections() + ["DEFAULT"]):
+            conf_object.add_section(section)
+        for option, value in extra_options[section].items():
+            conf_object.set(section, option, value)
+
+    return conf_object
+
+
+def extend_configfile(configfile, extra_options):
+    conf = configparser.ConfigParser()
+
+    conf.read(configfile)
+    add_extra_options(extra_options, conf)
+    write_configfile(configfile, conf)
+    raw_conf = six.StringIO()
+    conf.write(raw_conf)
+    return raw_conf.getvalue()
+
+
+class TempestConfigfileManager(utils.RandomNameGeneratorMixin):
+    """Class to create a Tempest config file."""
+
+    def __init__(self, deployment):
+        self.deployment = deployment
+
+        self.credential = deployment["admin"]
+        self.clients = osclients.Clients(objects.Credential(**self.credential))
+        self.keystone = self.clients.verified_keystone()
+        self.available_services = self.clients.services().values()
+
+        self.data_dir = _create_or_get_data_dir()
+
+        self.conf = configparser.ConfigParser()
+        self.conf.read(os.path.join(os.path.dirname(__file__), "config.ini"))
+
+    def _get_service_url(self, service_name):
+        s_type = self._get_service_type_by_service_name(service_name)
+        available_endpoints = self.keystone.service_catalog.get_endpoints()
+        service_endpoints = available_endpoints.get(s_type, [])
+        for endpoint in service_endpoints:
+            # If endpoints were returned by Keystone API V2
+            if "publicURL" in endpoint:
+                return endpoint["publicURL"]
+            # If endpoints were returned by Keystone API V3
+            if endpoint["interface"] == "public":
+                return endpoint["url"]
+
+    def _get_service_type_by_service_name(self, service_name):
+        for s_type, s_name in self.clients.services().items():
+            if s_name == service_name:
+                return s_type
+
+    def _configure_auth(self, section_name="auth"):
+        self.conf.set(section_name, "admin_username",
+                      self.credential["username"])
+        self.conf.set(section_name, "admin_password",
+                      self.credential["password"])
+        self.conf.set(section_name, "admin_project_name",
+                      self.credential["tenant_name"])
+        # Keystone v3 related parameter
+        self.conf.set(section_name, "admin_domain_name",
+                      self.credential["user_domain_name"] or "Default")
+
+    # Sahara has two service types: 'data_processing' and 'data-processing'.
+    # 'data_processing' is deprecated, but it can be used in previous OpenStack
+    # releases. So we need to configure the 'catalog_type' option to support
+    # environments where 'data_processing' is used as service type for Sahara.
+    def _configure_data_processing(self, section_name="data-processing"):
+        if "sahara" in self.available_services:
+            self.conf.set(section_name, "catalog_type",
+                          self._get_service_type_by_service_name("sahara"))
+
+    def _configure_identity(self, section_name="identity"):
+        self.conf.set(section_name, "region",
+                      self.credential["region_name"])
+
+        auth_url = self.credential["auth_url"]
+        if "/v2" not in auth_url and "/v3" not in auth_url:
+            auth_version = "v2"
+            auth_url_v2 = parse.urljoin(auth_url, "/v2.0")
+        else:
+            url_path = parse.urlparse(auth_url).path
+            auth_version = url_path[1:3]
+            auth_url_v2 = auth_url.replace(url_path, "/v2.0")
+        self.conf.set(section_name, "auth_version", auth_version)
+        self.conf.set(section_name, "uri", auth_url_v2)
+        self.conf.set(section_name, "uri_v3",
+                      auth_url_v2.replace("/v2.0", "/v3"))
+
+        self.conf.set(section_name, "disable_ssl_certificate_validation",
+                      str(self.credential["https_insecure"]))
+        self.conf.set(section_name, "ca_certificates_file",
+                      self.credential["https_cacert"])
+
+    # The compute section is configured in context class for Tempest resources.
+    # Options which are configured there: 'image_ref', 'image_ref_alt',
+    # 'flavor_ref', 'flavor_ref_alt'.
+
+    def _configure_network(self, section_name="network"):
+        if "neutron" in self.available_services:
+            neutronclient = self.clients.neutron()
+            public_nets = [net for net
+                           in neutronclient.list_networks()["networks"]
+                           if net["status"] == "ACTIVE" and
+                           net["router:external"] is True]
+            if public_nets:
+                net_id = public_nets[0]["id"]
+                self.conf.set(section_name, "public_network_id", net_id)
+        else:
+            novaclient = self.clients.nova()
+            net_name = next(net.human_id for net in novaclient.networks.list()
+                            if net.human_id is not None)
+            self.conf.set("compute", "fixed_network_name", net_name)
+            self.conf.set("validation", "network_for_ssh", net_name)
+
+    def _configure_network_feature_enabled(
+            self, section_name="network-feature-enabled"):
+        if "neutron" in self.available_services:
+            neutronclient = self.clients.neutron()
+            extensions = neutronclient.list_ext("extensions", "/extensions",
+                                                retrieve_all=True)
+            aliases = [ext["alias"] for ext in extensions["extensions"]]
+            aliases_str = ",".join(aliases)
+            self.conf.set(section_name, "api_extensions", aliases_str)
+
+    def _configure_oslo_concurrency(self, section_name="oslo_concurrency"):
+        lock_path = os.path.join(self.data_dir,
+                                 "lock_files_%s" % self.deployment["uuid"])
+        if not os.path.exists(lock_path):
+            os.makedirs(lock_path)
+        self.conf.set(section_name, "lock_path", lock_path)
+
+    def _configure_object_storage(self, section_name="object-storage"):
+        self.conf.set(section_name, "operator_role",
+                      CONF.tempest.swift_operator_role)
+        self.conf.set(section_name, "reseller_admin_role",
+                      CONF.tempest.swift_reseller_admin_role)
+
+    def _configure_scenario(self, section_name="scenario"):
+        self.conf.set(section_name, "img_dir", self.data_dir)
+
+    def _configure_service_available(self, section_name="service_available"):
+        services = ["cinder", "glance", "heat", "ironic", "neutron", "nova",
+                    "sahara", "swift"]
+        for service in services:
+            # Convert boolean to string because ConfigParser fails
+            # on attempt to get option with boolean value
+            self.conf.set(section_name, service,
+                          str(service in self.available_services))
+
+    def _configure_validation(self, section_name="validation"):
+        if "neutron" in self.available_services:
+            self.conf.set(section_name, "connect_method", "floating")
+        else:
+            self.conf.set(section_name, "connect_method", "fixed")
+
+    def _configure_orchestration(self, section_name="orchestration"):
+        self.conf.set(section_name, "stack_owner_role",
+                      CONF.tempest.heat_stack_owner_role)
+        self.conf.set(section_name, "stack_user_role",
+                      CONF.tempest.heat_stack_user_role)
+
+    def create(self, conf_path, extra_options=None):
+        for name, method in inspect.getmembers(self, inspect.ismethod):
+            if name.startswith("_configure_"):
+                method()
+
+        if extra_options:
+            add_extra_options(extra_options, self.conf)
+
+        write_configfile(conf_path, self.conf)
+
+        return read_configfile(conf_path)
+
+
+@context.configure("tempest_configuration", order=900)
+class TempestResourcesContext(context.VerifierContext):
+    """Context class to create/delete resources needed for Tempest."""
+
+    RESOURCE_NAME_FORMAT = "rally_verify_XXXXXXXX_XXXXXXXX"
+
+    def __init__(self, ctx):
+        super(TempestResourcesContext, self).__init__(ctx)
+        credential = self.verifier.deployment["admin"]
+        self.clients = osclients.Clients(objects.Credential(**credential))
+
+        self.conf = configparser.ConfigParser()
+        self.conf_path = self.verifier.manager.configfile
+        self.data_dir = os.path.join(os.path.expanduser("~"), ".rally",
+                                     "verification", "tempest", "data")
+        self.image_name = "tempest-image"
+
+        self._created_roles = []
+        self._created_images = []
+        self._created_flavors = []
+        self._created_networks = []
+
+    def setup(self):
+        self.available_services = self.clients.services().values()
+
+        self.conf.read(self.conf_path)
+
+        self.data_dir = _create_or_get_data_dir()
+
+        self._create_tempest_roles()
+
+        self._configure_option("scenario", "img_file", self.image_name,
+                               helper_method=self._download_image)
+        self._configure_option("compute", "image_ref",
+                               helper_method=self._discover_or_create_image)
+        self._configure_option("compute", "image_ref_alt",
+                               helper_method=self._discover_or_create_image)
+        self._configure_option("compute", "flavor_ref",
+                               helper_method=self._discover_or_create_flavor,
+                               flv_ram=CONF.tempest.flavor_ref_ram)
+        self._configure_option("compute", "flavor_ref_alt",
+                               helper_method=self._discover_or_create_flavor,
+                               flv_ram=CONF.tempest.flavor_ref_alt_ram)
+        if "neutron" in self.available_services:
+            neutronclient = self.clients.neutron()
+            if neutronclient.list_networks(shared=True)["networks"]:
+                # If the OpenStack cloud has some shared networks, we will
+                # create our own shared network and specify its name in the
+                # Tempest config file. Such approach will allow us to avoid
+                # failures of Tempest tests with error "Multiple possible
+                # networks found". Otherwise the default behavior defined in
+                # Tempest will be used and Tempest itself will manage network
+                # resources.
+                LOG.debug("Shared networks found. "
+                          "'fixed_network_name' option should be configured")
+                self._configure_option(
+                    "compute", "fixed_network_name",
+                    helper_method=self._create_network_resources)
+        if "heat" in self.available_services:
+            self._configure_option(
+                "orchestration", "instance_type",
+                helper_method=self._discover_or_create_flavor,
+                flv_ram=CONF.tempest.heat_instance_type_ram)
+
+        write_configfile(self.conf_path, self.conf)
+
+    def cleanup(self):
+        # Tempest tests may take more than 1 hour and we should remove all
+        # cached clients sessions to avoid tokens expiration when deleting
+        # Tempest resources.
+        self.clients.clear()
+
+        self._cleanup_tempest_roles()
+        self._cleanup_images()
+        self._cleanup_flavors()
+        if "neutron" in self.available_services:
+            self._cleanup_network_resources()
+
+        write_configfile(self.conf_path, self.conf)
+
+    def _create_tempest_roles(self):
+        keystoneclient = self.clients.verified_keystone()
+        roles = [CONF.tempest.swift_operator_role,
+                 CONF.tempest.swift_reseller_admin_role,
+                 CONF.tempest.heat_stack_owner_role,
+                 CONF.tempest.heat_stack_user_role]
+        existing_roles = set(role.name for role in keystoneclient.roles.list())
+
+        for role in roles:
+            if role not in existing_roles:
+                LOG.debug("Creating role '%s'" % role)
+                self._created_roles.append(keystoneclient.roles.create(role))
+
+    def _discover_image(self):
+        LOG.debug("Trying to discover a public image with name matching "
+                  "regular expression '%s'. Note that case insensitive "
+                  "matching is performed." % CONF.tempest.img_name_regex)
+        glance_wrapper = glance.wrap(self.clients.glance, self)
+        images = glance_wrapper.list_images(status="active",
+                                            visibility="public")
+        for image in images:
+            if image.name and re.match(CONF.tempest.img_name_regex,
+                                       image.name, re.IGNORECASE):
+                LOG.debug("The following public "
+                          "image discovered: '%s'" % image.name)
+                return image
+
+        LOG.debug("There is no public image with name matching "
+                  "regular expression '%s'" % CONF.tempest.img_name_regex)
+
+    def _download_image(self):
+        image_path = os.path.join(self.data_dir, self.image_name)
+        if os.path.isfile(image_path):
+            LOG.debug("Image is already downloaded to %s" % image_path)
+            return
+
+        if CONF.tempest.img_name_regex:
+            image = self._discover_image()
+            if image:
+                return _download_image(image_path, image)
+
+        _download_image(image_path)
+
+    def _configure_option(self, section, option, value=None,
+                          helper_method=None, *args, **kwargs):
+        option_value = self.conf.get(section, option)
+        if not option_value:
+            LOG.debug("Option '%s' from '%s' section "
+                      "is not configured" % (option, section))
+            if helper_method:
+                res = helper_method(*args, **kwargs)
+                if res:
+                    value = res["name"] if "network" in option else res.id
+            LOG.debug("Setting value '%s' for option '%s'" % (value, option))
+            self.conf.set(section, option, value)
+            LOG.debug("Option '{opt}' is configured. "
+                      "{opt} = {value}".format(opt=option, value=value))
+        else:
+            LOG.debug("Option '{opt}' is already configured "
+                      "in Tempest config file. {opt} = {opt_val}"
+                      .format(opt=option, opt_val=option_value))
+
+    def _discover_or_create_image(self):
+        if CONF.tempest.img_name_regex:
+            image = self._discover_image()
+            if image:
+                LOG.debug("Using image '%s' (ID = %s) "
+                          "for the tests" % (image.name, image.id))
+                return image
+
+        params = {
+            "name": self.generate_random_name(),
+            "disk_format": CONF.tempest.img_disk_format,
+            "container_format": CONF.tempest.img_container_format,
+            "image_location": os.path.join(self.data_dir, self.image_name),
+            "visibility": "public"
+        }
+        LOG.debug("Creating image '%s'" % params["name"])
+        glance_wrapper = glance.wrap(self.clients.glance, self)
+        image = glance_wrapper.create_image(**params)
+        LOG.debug("Image '%s' (ID = %s) has been "
+                  "successfully created!" % (image.name, image.id))
+        self._created_images.append(image)
+
+        return image
+
+    def _discover_or_create_flavor(self, flv_ram):
+        novaclient = self.clients.nova()
+
+        LOG.debug("Trying to discover a flavor with the following "
+                  "properties: RAM = %dMB, VCPUs = 1, disk = 0GB" % flv_ram)
+        for flavor in novaclient.flavors.list():
+            if (flavor.ram == flv_ram and
+                    flavor.vcpus == 1 and flavor.disk == 0):
+                LOG.debug("The following flavor discovered: '{0}'. "
+                          "Using flavor '{0}' (ID = {1}) for the tests"
+                          .format(flavor.name, flavor.id))
+                return flavor
+
+        LOG.debug("There is no flavor with the mentioned properties")
+
+        params = {
+            "name": self.generate_random_name(),
+            "ram": flv_ram,
+            "vcpus": 1,
+            "disk": 0
+        }
+        LOG.debug("Creating flavor '%s' with the following properties: RAM "
+                  "= %dMB, VCPUs = 1, disk = 0GB" % (params["name"], flv_ram))
+        flavor = novaclient.flavors.create(**params)
+        LOG.debug("Flavor '%s' (ID = %s) has been "
+                  "successfully created!" % (flavor.name, flavor.id))
+        self._created_flavors.append(flavor)
+
+        return flavor
+
+    def _create_network_resources(self):
+        neutron_wrapper = network.NeutronWrapper(self.clients, self)
+        tenant_id = self.clients.keystone.auth_ref.project_id
+        LOG.debug("Creating network resources: network, subnet, router")
+        net = neutron_wrapper.create_network(
+            tenant_id, subnets_num=1, add_router=True,
+            network_create_args={"shared": True})
+        LOG.debug("Network resources have been successfully created!")
+        self._created_networks.append(net)
+
+        return net
+
+    def _cleanup_tempest_roles(self):
+        keystoneclient = self.clients.keystone()
+        for role in self._created_roles:
+            LOG.debug("Deleting role '%s'" % role.name)
+            keystoneclient.roles.delete(role.id)
+            LOG.debug("Role '%s' has been deleted" % role.name)
+
+    def _cleanup_images(self):
+        glance_wrapper = glance.wrap(self.clients.glance, self)
+        for image in self._created_images:
+            LOG.debug("Deleting image '%s'" % image.name)
+            self.clients.glance().images.delete(image.id)
+            task_utils.wait_for_status(
+                image, ["deleted", "pending_delete"],
+                check_deletion=True,
+                update_resource=glance_wrapper.get_image,
+                timeout=CONF.benchmark.glance_image_delete_timeout,
+                check_interval=CONF.benchmark.
+                glance_image_delete_poll_interval)
+            LOG.debug("Image '%s' has been deleted" % image.name)
+            self._remove_opt_value_from_config("compute", image.id)
+
+    def _cleanup_flavors(self):
+        novaclient = self.clients.nova()
+        for flavor in self._created_flavors:
+            LOG.debug("Deleting flavor '%s'" % flavor.name)
+            novaclient.flavors.delete(flavor.id)
+            LOG.debug("Flavor '%s' has been deleted" % flavor.name)
+            self._remove_opt_value_from_config("compute", flavor.id)
+            self._remove_opt_value_from_config("orchestration", flavor.id)
+
+    def _cleanup_network_resources(self):
+        neutron_wrapper = network.NeutronWrapper(self.clients, self)
+        for net in self._created_networks:
+            LOG.debug("Deleting network resources: router, subnet, network")
+            neutron_wrapper.delete_network(net)
+            self._remove_opt_value_from_config("compute", net["name"])
+            LOG.debug("Network resources have been deleted")
+
+    def _remove_opt_value_from_config(self, section, opt_value):
+        for option, value in self.conf.items(section):
+            if opt_value == value:
+                LOG.debug("Removing value '%s' for option '%s' "
+                          "from Tempest config file" % (opt_value, option))
+                self.conf.set(section, option, "")
+                LOG.debug("Value '%s' has been removed" % opt_value)

rally/plugins/openstack/verification/tempest/consts.py

@@ -0,0 +1,46 @@
+# Copyright 2016: Mirantis Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from rally.common import utils
+
+
+class _TempestApiTestSets(utils.ImmutableMixin, utils.EnumMixin):
+    BAREMETAL = "baremetal"
+    CLUSTERING = "clustering"
+    COMPUTE = "compute"
+    DATA_PROCESSING = "data_processing"
+    DATABASE = "database"
+    IDENTITY = "identity"
+    IMAGE = "image"
+    MESSAGING = "messaging"
+    NETWORK = "network"
+    OBJECT_STORAGE = "object_storage"
+    ORCHESTRATION = "orchestration"
+    TELEMETRY = "telemetry"
+    VOLUME = "volume"
+
+
+class _TempestScenarioTestSets(utils.ImmutableMixin, utils.EnumMixin):
+    SCENARIO = "scenario"
+
+
+class _TempestTestSets(utils.ImmutableMixin, utils.EnumMixin):
+    FULL = "full"
+    SMOKE = "smoke"
+
+
+TempestApiTestSets = _TempestApiTestSets()
+TempestScenarioTestSets = _TempestScenarioTestSets()
+TempestTestSets = _TempestTestSets()

rally/plugins/openstack/verification/tempest/manager.py

@@ -0,0 +1,178 @@
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import os
+import re
+import shutil
+import subprocess
+
+import yaml
+
+from rally.common.i18n import _LE
+from rally.common import logging
+from rally import exceptions
+from rally.plugins.common.verification import testr
+from rally.plugins.openstack.verification.tempest import config
+from rally.plugins.openstack.verification.tempest import consts
+from rally.verification import manager
+from rally.verification import utils
+
+
+LOG = logging.getLogger(__name__)
+
+
+@manager.configure(name="tempest", namespace="openstack",
+                   default_repo="https://git.openstack.org/openstack/tempest",
+                   context={"tempest_configuration": {},
+                            "testr_verifier": {}})
+class TempestManager(testr.TestrLauncher):
+    """Plugin for Tempest management."""
+
+    @property
+    def run_environ(self):
+        env = super(TempestManager, self).run_environ
+        env["TEMPEST_CONFIG_DIR"] = os.path.dirname(self.configfile)
+        env["TEMPEST_CONFIG"] = os.path.basename(self.configfile)
+        # TODO(andreykurilin): move it to Testr base class
+        env["OS_TEST_PATH"] = os.path.join(self.repo_dir,
+                                           "tempest/test_discover")
+        return env
+
+    @property
+    def configfile(self):
+        return os.path.join(self.home_dir, "tempest.conf")
+
+    def get_configuration(self):
+        return config.read_configfile(self.configfile)
+
+    def configure(self, extra_options=None):
+        cm = config.TempestConfigfileManager(self.verifier.deployment)
+        raw_configfile = cm.create(self.configfile, extra_options)
+        return raw_configfile
+
+    def extend_configuration(self, extra_options):
+        return config.extend_configfile(self.configfile, extra_options)
+
+    def override_configuration(self, new_content):
+        with open(self.configfile, "w") as f:
+            f.write(new_content)
+
+    def install_extension(self, source, version=None, extra=None):
+        """Install a Tempest plugin."""
+        if extra:
+            raise NotImplementedError(
+                _LE("'%s' verifiers don't support extra options for "
+                    "extension installations.")
+                % self.get_name())
+        version = version or "master"
+        egg = re.sub("\.git$", "", os.path.basename(source.strip("/")))
+        full_source = "git+{0}@{1}#egg={2}".format(source, version, egg)
+        # NOTE(ylobankov): Use 'develop mode' installation to provide an
+        #                  ability to advanced users to change tests or
+        #                  develop new ones in verifier repo on the fly.
+        cmd = ["pip", "install",
+               "--src", os.path.join(self.base_dir, "extensions"),
+               "-e", full_source]
+        if self.verifier.system_wide:
+            cmd.insert(2, "--no-deps")
+        utils.check_output(cmd, cwd=self.base_dir, env=self.environ)
+
+        # Very often Tempest plugins are inside projects and requirements
+        # for plugins are listed in the test-requirements.txt file.
+        test_reqs_path = os.path.join(self.base_dir, "extensions",
+                                      egg, "test-requirements.txt")
+        # TODO(andreykurilin): check that packages from test-requirements are
+        #   present in system in case of system_wide installation
+        if not self.verifier.system_wide and os.path.exists(test_reqs_path):
+            utils.check_output(["pip", "install", "-r", test_reqs_path],
+                               cwd=self.base_dir, env=self.environ)
+
+    def list_extensions(self):
+        """List all installed Tempest plugins."""
+        # TODO(andreykurilin): find a better way to list tempest plugins
+        cmd = ("from tempest.test_discover import plugins; "
+               "plugins_manager = plugins.TempestTestPluginManager(); "
+               "plugins_map = plugins_manager.get_plugin_load_tests_tuple(); "
+               "plugins_list = ["
+               "    {'name': p.name, "
+               "     'entry_point': p.entry_point_target, "
+               "     'location': plugins_map[p.name][1]} "
+               "    for p in plugins_manager.ext_plugins.extensions]; "
+               "print(plugins_list)")
+        try:
+            return yaml.load(
+                utils.check_output(["python", "-c", cmd], cwd=self.base_dir,
+                                   env=self.environ).strip())
+        except subprocess.CalledProcessError:
+            raise exceptions.RallyException(
+                "Cannot list installed Tempest plugins for verifier %s." %
+                self.verifier)
+
+    def uninstall_extension(self, name):
+        """Uninstall a Tempest plugin."""
+        for ext in self.list_extensions():
+            if ext["name"] == name and os.path.exists(ext["location"]):
+                shutil.rmtree(ext["location"])
+                break
+        else:
+            raise exceptions.RallyException(
+                "There is no Tempest plugin with name '%s'. "
+                "Are you sure that it was installed?" % name)
+
+    def list_tests(self, pattern=""):
+        """List all tests."""
+        if pattern:
+            pattern = self._transform_pattern(pattern)
+        return super(TempestManager, self).list_tests(pattern)
+
+    @classmethod
+    def validate_args(cls, args):
+        """Validate given arguments."""
+        super(TempestManager, cls).validate_args(args)
+
+        if args.get("pattern"):
+            pattern = args["pattern"].split("=", 1)
+            if len(pattern) == 1:
+                pass  # it is just a regex
+            elif pattern[0] == "set":
+                available_sets = (list(consts.TempestTestSets) +
+                                  list(consts.TempestApiTestSets) +
+                                  list(consts.TempestScenarioTestSets))
+                if pattern[1] not in available_sets:
+                    raise exceptions.ValidationError(
+                        "Test set '%s' not found in available "
+                        "Tempest test sets. Available sets are '%s'."
+                        % (pattern[1], "', '".join(available_sets)))
+            else:
+                raise exceptions.ValidationError(
+                    "'pattern' argument should be a regexp or set name "
+                    "(format: 'tempest.api.identity.v3', 'set=smoke').")
+
+    def _transform_pattern(self, pattern):
+        """Transforms tempest-specific pattern to testr format."""
+        parsed_pattern = pattern.split("=", 1)
+        if len(parsed_pattern) == 2:
+            if parsed_pattern[0] == "set":
+                if parsed_pattern[1] in consts.TempestTestSets:
+                    return "smoke" if parsed_pattern[1] == "smoke" else ""
+                elif parsed_pattern[1] in consts.TempestApiTestSets:
+                    return "tempest.api.%s" % parsed_pattern[1]
+                else:
+                    return "tempest.%s" % parsed_pattern[1]
+        return pattern
+
+    def _process_run_args(self, run_args):
+        if run_args.get("pattern"):
+            run_args["pattern"] = self._transform_pattern(run_args["pattern"])
+        return run_args

tests/unit/plugins/openstack/verification/tempest/test_config.py

@@ -0,0 +1,633 @@
+# Copyright 2014: Mirantis Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import os
+
+import ddt
+import mock
+from oslo_config import cfg
+import requests
+
+from rally import exceptions
+from rally.plugins.openstack.verification.tempest import config
+from tests.unit import fakes
+from tests.unit import test
+
+CONF = cfg.CONF
+
+
+CREDS = {
+    "admin": {
+        "username": "admin",
+        "tenant_name": "admin",
+        "password": "admin-12345",
+        "auth_url": "http://test:5000/v2.0/",
+        "permission": "admin",
+        "region_name": "test",
+        "https_insecure": False,
+        "https_cacert": "/path/to/cacert/file",
+        "user_domain_name": "admin",
+        "project_domain_name": "admin"
+    },
+    "uuid": "fake_deployment"
+}
+
+PATH = "rally.plugins.openstack.verification.tempest.config"
+
+
+@ddt.ddt
+class TempestConfigTestCase(test.TestCase):
+
+    def setUp(self):
+        super(TempestConfigTestCase, self).setUp()
+
+        mock.patch("rally.osclients.Clients").start()
+
+        self.tempest_conf = config.TempestConfigfileManager(CREDS)
+
+    @ddt.data({"publicURL": "test_url"},
+              {"interface": "public", "url": "test_url"})
+    def test__get_service_url(self, endpoint):
+        mock_catalog = mock.MagicMock()
+        mock_catalog.get_endpoints.return_value = {
+            "test_service_type": [endpoint]}
+
+        self.tempest_conf.keystone.service_catalog = mock_catalog
+        self.tempest_conf.clients.services.return_value = {
+            "test_service_type": "test_service"}
+        self.assertEqual(
+            self.tempest_conf._get_service_url("test_service"), "test_url")
+
+    def test__configure_auth(self):
+        self.tempest_conf._configure_auth()
+
+        expected = (
+            ("admin_username", CREDS["admin"]["username"]),
+            ("admin_password", CREDS["admin"]["password"]),
+            ("admin_project_name", CREDS["admin"]["tenant_name"]),
+            ("admin_domain_name", CREDS["admin"]["user_domain_name"]))
+        result = self.tempest_conf.conf.items("auth")
+        for item in expected:
+            self.assertIn(item, result)
+
+    @ddt.data("data_processing", "data-processing")
+    def test__configure_data_processing(self, service_type):
+        self.tempest_conf.available_services = ["sahara"]
+
+        self.tempest_conf.clients.services.return_value = {
+            service_type: "sahara"}
+        self.tempest_conf._configure_data_processing()
+        self.assertEqual(
+            self.tempest_conf.conf.get(
+                "data-processing", "catalog_type"), service_type)
+
+    def test__configure_identity(self):
+        self.tempest_conf._configure_identity()
+
+        expected = (
+            ("region", CREDS["admin"]["region_name"]),
+            ("auth_version", "v2"),
+            ("uri", CREDS["admin"]["auth_url"][:-1]),
+            ("uri_v3", CREDS["admin"]["auth_url"].replace("/v2.0/", "/v3")),
+            ("disable_ssl_certificate_validation",
+             str(CREDS["admin"]["https_insecure"])),
+            ("ca_certificates_file", CREDS["admin"]["https_cacert"]))
+        result = self.tempest_conf.conf.items("identity")
+        for item in expected:
+            self.assertIn(item, result)
+
+    def test__configure_network_if_neutron(self):
+        self.tempest_conf.available_services = ["neutron"]
+        client = self.tempest_conf.clients.neutron()
+        client.list_networks.return_value = {
+            "networks": [
+                {
+                    "status": "ACTIVE",
+                    "id": "test_id",
+                    "router:external": True
+                }
+            ]
+        }
+
+        self.tempest_conf._configure_network()
+        self.assertEqual(
+            self.tempest_conf.conf.get("network",
+                                       "public_network_id"), "test_id")
+
+    def test__configure_network_if_nova(self):
+        self.tempest_conf.available_services = ["nova"]
+        client = self.tempest_conf.clients.nova()
+        client.networks.list.return_value = [
+            mock.MagicMock(human_id="fake-network")]
+
+        self.tempest_conf._configure_network()
+
+        expected = {"compute": ("fixed_network_name", "fake-network"),
+                    "validation": ("network_for_ssh", "fake-network")}
+        for section, option in expected.items():
+            result = self.tempest_conf.conf.items(section)
+            self.assertIn(option, result)
+
+    def test__configure_network_feature_enabled(self):
+        self.tempest_conf.available_services = ["neutron"]
+        client = self.tempest_conf.clients.neutron()
+        client.list_ext.return_value = {
+            "extensions": [
+                {"alias": "dvr"},
+                {"alias": "extra_dhcp_opt"},
+                {"alias": "extraroute"}
+            ]
+        }
+
+        self.tempest_conf._configure_network_feature_enabled()
+        client.list_ext.assert_called_once_with("extensions", "/extensions",
+                                                retrieve_all=True)
+        self.assertEqual(self.tempest_conf.conf.get(
+            "network-feature-enabled", "api_extensions"),
+            "dvr,extra_dhcp_opt,extraroute")
+
+    @mock.patch("os.makedirs")
+    @mock.patch("os.path.exists", return_value=False)
+    def test__configure_oslo_concurrency(self, mock_exists, mock_makedirs):
+        self.tempest_conf._configure_oslo_concurrency()
+
+        lock_path = os.path.join(
+            self.tempest_conf.data_dir, "lock_files_fake_deployment")
+        mock_makedirs.assert_called_with(lock_path)
+        self.assertEqual(
+            self.tempest_conf.conf.get(
+                "oslo_concurrency", "lock_path"), lock_path)
+
+    def test__configure_object_storage(self):
+        self.tempest_conf._configure_object_storage()
+
+        expected = (
+            ("operator_role", CONF.tempest.swift_operator_role),
+            ("reseller_admin_role", CONF.tempest.swift_reseller_admin_role))
+        result = self.tempest_conf.conf.items("object-storage")
+        for item in expected:
+            self.assertIn(item, result)
+
+    def test__configure_orchestration(self):
+        self.tempest_conf._configure_orchestration()
+
+        expected = (
+            ("stack_owner_role", CONF.tempest.heat_stack_owner_role),
+            ("stack_user_role", CONF.tempest.heat_stack_user_role))
+        result = self.tempest_conf.conf.items("orchestration")
+        for item in expected:
+            self.assertIn(item, result)
+
+    def test__configure_scenario(self):
+        self.tempest_conf._configure_scenario()
+
+        expected = (("img_dir", self.tempest_conf.data_dir),)
+        result = self.tempest_conf.conf.items("scenario")
+        for item in expected:
+            self.assertIn(item, result)
+
+    def test__configure_service_available(self):
+        available_services = ("nova", "cinder", "glance", "sahara")
+        self.tempest_conf.available_services = available_services
+        self.tempest_conf._configure_service_available()
+
+        expected = (
+            ("neutron", "False"), ("heat", "False"), ("nova", "True"),
+            ("swift", "False"), ("cinder", "True"), ("sahara", "True"),
+            ("glance", "True"))
+        result = self.tempest_conf.conf.items("service_available")
+        for item in expected:
+            self.assertIn(item, result)
+
+    @ddt.data({}, {"service": "neutron", "connect_method": "floating"})
+    @ddt.unpack
+    def test__configure_validation(self, service="nova",
+                                   connect_method="fixed"):
+        self.tempest_conf.available_services = [service]
+        self.tempest_conf._configure_validation()
+
+        expected = (("run_validation", "True"),
+                    ("connect_method", connect_method))
+        result = self.tempest_conf.conf.items("validation")
+        for item in expected:
+            self.assertIn(item, result)
+
+    @mock.patch("rally.plugins.openstack.verification."
+                "tempest.config.read_configfile")
+    @mock.patch("rally.plugins.openstack.verification."
+                "tempest.config.write_configfile")
+    @mock.patch("inspect.getmembers")
+    def test_create(self, mock_inspect_getmembers, mock_write_configfile,
+                    mock_read_configfile):
+        configure_something_method = mock.MagicMock()
+        mock_inspect_getmembers.return_value = [("_configure_something",
+                                                 configure_something_method)]
+
+        fake_extra_conf = {"section": {"option": "value"}}
+
+        self.assertEqual(mock_read_configfile.return_value,
+                         self.tempest_conf.create("/path/to/fake/conf",
+                                                  fake_extra_conf))
+        self.assertEqual(configure_something_method.call_count, 1)
+        self.assertIn(("option", "value"),
+                      self.tempest_conf.conf.items("section"))
+        self.assertEqual(mock_write_configfile.call_count, 1)
+
+
+@ddt.ddt
+class TempestResourcesContextTestCase(test.TestCase):
+
+    def setUp(self):
+        super(TempestResourcesContextTestCase, self).setUp()
+
+        mock.patch("rally.osclients.Clients").start()
+        self.mock_isfile = mock.patch("os.path.isfile",
+                                      return_value=True).start()
+
+        cfg = {"verifier": mock.Mock(deployment=CREDS),
+               "verification": {"uuid": "uuid"}}
+        cfg["verifier"].manager.configfile = "/fake/path/to/config"
+        self.context = config.TempestResourcesContext(cfg)
+        self.context.conf.add_section("compute")
+        self.context.conf.add_section("orchestration")
+        self.context.conf.add_section("scenario")
+
+    @mock.patch("six.moves.builtins.open", side_effect=mock.mock_open(),
+                create=True)
+    def test__download_image_from_glance(self, mock_open):
+        self.mock_isfile.return_value = False
+        img_path = os.path.join(self.context.data_dir, "foo")
+        img = mock.MagicMock()
+        img.data.return_value = "data"
+
+        config._download_image(img_path, img)
+        mock_open.assert_called_once_with(img_path, "wb")
+        mock_open().write.assert_has_calls([mock.call("d"),
+                                            mock.call("a"),
+                                            mock.call("t"),
+                                            mock.call("a")])
+
+    @mock.patch("six.moves.builtins.open", side_effect=mock.mock_open())
+    @mock.patch("requests.get", return_value=mock.MagicMock(status_code=200))
+    def test__download_image_from_url_success(self, mock_get, mock_open):
+        self.mock_isfile.return_value = False
+        img_path = os.path.join(self.context.data_dir, "foo")
+        mock_get.return_value.iter_content.return_value = "data"
+
+        config._download_image(img_path)
+        mock_get.assert_called_once_with(CONF.tempest.img_url, stream=True)
+        mock_open.assert_called_once_with(img_path, "wb")
+        mock_open().write.assert_has_calls([mock.call("d"),
+                                            mock.call("a"),
+                                            mock.call("t"),
+                                            mock.call("a")])
+
+    @mock.patch("requests.get")
+    @ddt.data(404, 500)
+    def test__download_image_from_url_failure(self, status_code, mock_get):
+        self.mock_isfile.return_value = False
+        mock_get.return_value = mock.MagicMock(status_code=status_code)
+        self.assertRaises(
+            exceptions.TempestConfigCreationFailure, config._download_image,
+            os.path.join(self.context.data_dir, "foo"))
+
+    @mock.patch("requests.get", side_effect=requests.ConnectionError())
+    def test__download_image_from_url_connection_error(
+            self, mock_requests_get):
+        self.mock_isfile.return_value = False
+        self.assertRaises(
+            exceptions.TempestConfigCreationFailure, config._download_image,
+            os.path.join(self.context.data_dir, "foo"))
+
+    @mock.patch("rally.plugins.openstack.wrappers."
+                "network.NeutronWrapper.create_network")
+    @mock.patch("six.moves.builtins.open", side_effect=mock.mock_open())
+    def test_options_configured_manually(
+            self, mock_open, mock_neutron_wrapper_create_network):
+        self.context.available_services = ["glance", "heat", "nova", "neutron"]
+
+        self.context.conf.set("compute", "image_ref", "id1")
+        self.context.conf.set("compute", "image_ref_alt", "id2")
+        self.context.conf.set("compute", "flavor_ref", "id3")
+        self.context.conf.set("compute", "flavor_ref_alt", "id4")
+        self.context.conf.set("compute", "fixed_network_name", "name1")
+        self.context.conf.set("orchestration", "instance_type", "id5")
+        self.context.conf.set("scenario", "img_file", "id6")
+
+        self.context.__enter__()
+
+        glanceclient = self.context.clients.glance()
+        novaclient = self.context.clients.nova()
+
+        self.assertEqual(glanceclient.images.create.call_count, 0)
+        self.assertEqual(novaclient.flavors.create.call_count, 0)
+        self.assertEqual(mock_neutron_wrapper_create_network.call_count, 0)
+
+    def test__create_tempest_roles(self):
+        role1 = CONF.tempest.swift_operator_role
+        role2 = CONF.tempest.swift_reseller_admin_role
+        role3 = CONF.tempest.heat_stack_owner_role
+        role4 = CONF.tempest.heat_stack_user_role
+
+        client = self.context.clients.verified_keystone()
+        client.roles.list.return_value = [fakes.FakeRole(name=role1),
+                                          fakes.FakeRole(name=role2)]
+        client.roles.create.side_effect = [fakes.FakeFlavor(name=role3),
+                                           fakes.FakeFlavor(name=role4)]
+
+        self.context._create_tempest_roles()
+        self.assertEqual(client.roles.create.call_count, 2)
+
+        created_roles = [role.name for role in self.context._created_roles]
+        self.assertIn(role3, created_roles)
+        self.assertIn(role4, created_roles)
+
+    @mock.patch("rally.plugins.openstack.wrappers.glance.wrap")
+    def test__discover_image(self, mock_wrap):
+        client = mock_wrap.return_value
+        client.list_images.return_value = [fakes.FakeImage(name="Foo"),
+                                           fakes.FakeImage(name="CirrOS")]
+
+        image = self.context._discover_image()
+        self.assertEqual("CirrOS", image.name)
+
+    @mock.patch("six.moves.builtins.open", side_effect=mock.mock_open(),
+                create=True)
+    @mock.patch("rally.plugins.openstack.wrappers.glance.wrap")
+    @mock.patch("os.path.isfile", return_value=False)
+    def test__download_image(self, mock_isfile, mock_wrap, mock_open):
+        img_1 = mock.MagicMock()
+        img_1.name = "Foo"
+        img_2 = mock.MagicMock()
+        img_2.name = "CirrOS"
+        img_2.data.return_value = "data"
+        mock_wrap.return_value.list_images.return_value = [img_1, img_2]
+
+        self.context._download_image()
+        img_path = os.path.join(self.context.data_dir, self.context.image_name)
+        mock_open.assert_called_once_with(img_path, "wb")
+        mock_open().write.assert_has_calls([mock.call("d"),
+                                            mock.call("a"),
+                                            mock.call("t"),
+                                            mock.call("a")])
+
+    # We can choose any option to test the '_configure_option' method. So let's
+    # configure the 'flavor_ref' option.
+    def test__configure_option(self):
+        helper_method = mock.MagicMock()
+        helper_method.side_effect = [fakes.FakeFlavor(id="id1")]
+
+        self.context.conf.set("compute", "flavor_ref", "")
+        self.context._configure_option("compute", "flavor_ref",
+                                       helper_method=helper_method, flv_ram=64)
+        self.assertEqual(helper_method.call_count, 1)
+
+        result = self.context.conf.get("compute", "flavor_ref")
+        self.assertEqual("id1", result)
+
+    @mock.patch("rally.plugins.openstack.wrappers.glance.wrap")
+    def test__discover_or_create_image_when_image_exists(self, mock_wrap):
+        client = mock_wrap.return_value
+        client.list_images.return_value = [fakes.FakeImage(name="CirrOS")]
+
+        image = self.context._discover_or_create_image()
+        self.assertEqual("CirrOS", image.name)
+        self.assertEqual(0, client.create_image.call_count)
+        self.assertEqual(0, len(self.context._created_images))
+
+    @mock.patch("rally.plugins.openstack.wrappers.glance.wrap")
+    def test__discover_or_create_image(self, mock_wrap):
+        client = mock_wrap.return_value
+
+        image = self.context._discover_or_create_image()
+        self.assertEqual(image, client.create_image.return_value)
+        self.assertEqual(self.context._created_images[0],
+                         client.create_image.return_value)
+        client.create_image.assert_called_once_with(
+            container_format=CONF.tempest.img_container_format,
+            image_location=mock.ANY,
+            disk_format=CONF.tempest.img_disk_format,
+            name=mock.ANY,
+            visibility="public")
+
+    def test__discover_or_create_flavor_when_flavor_exists(self):
+        client = self.context.clients.nova()
+        client.flavors.list.return_value = [fakes.FakeFlavor(id="id1", ram=64,
+                                                             vcpus=1, disk=0)]
+
+        flavor = self.context._discover_or_create_flavor(64)
+        self.assertEqual("id1", flavor.id)
+        self.assertEqual(0, len(self.context._created_flavors))
+
+    def test__discover_or_create_flavor(self):
+        client = self.context.clients.nova()
+        client.flavors.create.side_effect = [fakes.FakeFlavor(id="id1")]
+
+        flavor = self.context._discover_or_create_flavor(64)
+        self.assertEqual("id1", flavor.id)
+        self.assertEqual("id1", self.context._created_flavors[0].id)
+
+    def test__create_network_resources(self):
+        client = self.context.clients.neutron()
+        fake_network = {
+            "id": "nid1",
+            "name": "network",
+            "status": "status"}
+
+        client.create_network.side_effect = [{"network": fake_network}]
+        client.create_router.side_effect = [{"router": {"id": "rid1"}}]
+        client.create_subnet.side_effect = [{"subnet": {"id": "subid1"}}]
+
+        network = self.context._create_network_resources()
+        self.assertEqual("nid1", network["id"])
+        self.assertEqual("nid1", self.context._created_networks[0]["id"])
+        self.assertEqual("rid1",
+                         self.context._created_networks[0]["router_id"])
+        self.assertEqual("subid1",
+                         self.context._created_networks[0]["subnets"][0])
+
+    def test__cleanup_tempest_roles(self):
+        self.context._created_roles = [fakes.FakeRole(), fakes.FakeRole()]
+
+        self.context._cleanup_tempest_roles()
+        client = self.context.clients.keystone()
+        self.assertEqual(client.roles.delete.call_count, 2)
+
+    @mock.patch("rally.plugins.openstack.wrappers.glance.wrap")
+    def test__cleanup_images(self, mock_wrap):
+        self.context._created_images = [fakes.FakeImage(id="id1"),
+                                        fakes.FakeImage(id="id2")]
+
+        self.context.conf.set("compute", "image_ref", "id1")
+        self.context.conf.set("compute", "image_ref_alt", "id2")
+
+        wrapper = mock_wrap.return_value
+        wrapper.get_image.side_effect = [
+            fakes.FakeImage(id="id1", status="DELETED"),
+            fakes.FakeImage(id="id2"),
+            fakes.FakeImage(id="id2", status="DELETED")]
+
+        self.context._cleanup_images()
+        client = self.context.clients.glance()
+        client.images.delete.assert_has_calls([mock.call("id1"),
+                                               mock.call("id2")])
+
+        self.assertEqual("", self.context.conf.get("compute", "image_ref"))
+        self.assertEqual("", self.context.conf.get("compute", "image_ref_alt"))
+
+    def test__cleanup_flavors(self):
+        self.context._created_flavors = [fakes.FakeFlavor(id="id1"),
+                                         fakes.FakeFlavor(id="id2"),
+                                         fakes.FakeFlavor(id="id3")]
+
+        self.context.conf.set("compute", "flavor_ref", "id1")
+        self.context.conf.set("compute", "flavor_ref_alt", "id2")
+        self.context.conf.set("orchestration", "instance_type", "id3")
+
+        self.context._cleanup_flavors()
+        client = self.context.clients.nova()
+        self.assertEqual(client.flavors.delete.call_count, 3)
+
+        self.assertEqual("", self.context.conf.get("compute", "flavor_ref"))
+        self.assertEqual("", self.context.conf.get("compute",
+                                                   "flavor_ref_alt"))
+        self.assertEqual("", self.context.conf.get("orchestration",
+                                                   "instance_type"))
+
+    @mock.patch("rally.plugins.openstack.wrappers."
+                "network.NeutronWrapper.delete_network")
+    def test__cleanup_network_resources(
+            self, mock_neutron_wrapper_delete_network):
+        self.context._created_networks = [{"name": "net-12345"}]
+        self.context.conf.set("compute", "fixed_network_name", "net-12345")
+
+        self.context._cleanup_network_resources()
+        self.assertEqual(mock_neutron_wrapper_delete_network.call_count, 1)
+        self.assertEqual("", self.context.conf.get("compute",
+                                                   "fixed_network_name"))
+
+    @mock.patch("%s.write_configfile" % PATH)
+    @mock.patch("%s.TempestResourcesContext._configure_option" % PATH)
+    @mock.patch("%s.TempestResourcesContext._create_tempest_roles" % PATH)
+    @mock.patch("%s._create_or_get_data_dir" % PATH)
+    @mock.patch("%s.osclients.Clients" % PATH)
+    def test_setup(self, mock_clients, mock__create_or_get_data_dir,
+                   mock__create_tempest_roles, mock__configure_option,
+                   mock_write_configfile):
+        mock_clients.return_value.services.return_value = {}
+        verifier = mock.MagicMock(deployment=CREDS)
+        cfg = config.TempestResourcesContext({"verifier": verifier})
+        cfg.conf = mock.Mock()
+
+        # case #1: no neutron and heat
+        cfg.setup()
+
+        cfg.conf.read.assert_called_once_with(verifier.manager.configfile)
+        mock__create_or_get_data_dir.assert_called_once_with()
+        mock__create_tempest_roles.assert_called_once_with()
+        mock_write_configfile.assert_called_once_with(
+            verifier.manager.configfile, cfg.conf)
+        self.assertEqual(
+            [mock.call("scenario", "img_file", cfg.image_name,
+                       helper_method=cfg._download_image),
+             mock.call("compute", "image_ref",
+                       helper_method=cfg._discover_or_create_image),
+             mock.call("compute", "image_ref_alt",
+                       helper_method=cfg._discover_or_create_image),
+             mock.call("compute", "flavor_ref",
+                       helper_method=cfg._discover_or_create_flavor,
+                       flv_ram=config.CONF.tempest.flavor_ref_ram),
+             mock.call("compute", "flavor_ref_alt",
+                       helper_method=cfg._discover_or_create_flavor,
+                       flv_ram=config.CONF.tempest.flavor_ref_alt_ram)],
+            mock__configure_option.call_args_list)
+
+        cfg.conf.reset_mock()
+        mock__create_or_get_data_dir.reset_mock()
+        mock__create_tempest_roles.reset_mock()
+        mock_write_configfile.reset_mock()
+        mock__configure_option.reset_mock()
+
+        # case #2: neutron and heat are presented
+        mock_clients.return_value.services.return_value = {
+            "network": "neutron", "orchestration": "heat"}
+        cfg.setup()
+
+        cfg.conf.read.assert_called_once_with(verifier.manager.configfile)
+        mock__create_or_get_data_dir.assert_called_once_with()
+        mock__create_tempest_roles.assert_called_once_with()
+        mock_write_configfile.assert_called_once_with(
+            verifier.manager.configfile, cfg.conf)
+        self.assertEqual(
+            [mock.call("scenario", "img_file", cfg.image_name,
+                       helper_method=cfg._download_image),
+             mock.call("compute", "image_ref",
+                       helper_method=cfg._discover_or_create_image),
+             mock.call("compute", "image_ref_alt",
+                       helper_method=cfg._discover_or_create_image),
+             mock.call("compute", "flavor_ref",
+                       helper_method=cfg._discover_or_create_flavor,
+                       flv_ram=config.CONF.tempest.flavor_ref_ram),
+             mock.call("compute", "flavor_ref_alt",
+                       helper_method=cfg._discover_or_create_flavor,
+                       flv_ram=config.CONF.tempest.flavor_ref_alt_ram),
+             mock.call("compute", "fixed_network_name",
+                       helper_method=cfg._create_network_resources),
+             mock.call("orchestration", "instance_type",
+                       helper_method=cfg._discover_or_create_flavor,
+                       flv_ram=config.CONF.tempest.heat_instance_type_ram)],
+            mock__configure_option.call_args_list)
+
+
+class UtilsTestCase(test.TestCase):
+
+    @mock.patch("six.moves.builtins.open", side_effect=mock.mock_open())
+    def test_write_configfile(self, mock_open):
+        conf_path = "/path/to/fake/conf"
+        conf_data = mock.Mock()
+
+        config.write_configfile(conf_path, conf_data)
+        mock_open.assert_called_once_with(conf_path, "w")
+        conf_data.write.assert_called_once_with(mock_open.side_effect())
+
+    @mock.patch("six.moves.builtins.open", side_effect=mock.mock_open())
+    def test_read_configfile(self, mock_open):
+        conf_path = "/path/to/fake/conf"
+
+        config.read_configfile(conf_path)
+        mock_open.assert_called_once_with(conf_path)
+        mock_open.side_effect().read.assert_called_once_with()
+
+    @mock.patch("rally.plugins.openstack.verification.tempest.config."
+                "six.StringIO")
+    @mock.patch("rally.plugins.openstack.verification.tempest.config."
+                "write_configfile")
+    @mock.patch("rally.plugins.openstack.verification.tempest.config."
+                "add_extra_options")
+    @mock.patch("rally.plugins.openstack.verification.tempest.config."
+                "configparser")
+    def test_extend_configfile(self, mock_configparser, mock_add_extra_options,
+                               mock_write_configfile, mock_string_io):
+        conf_path = "/path/to/fake/conf"
+        extra_options = mock.Mock()
+
+        config.extend_configfile(conf_path, extra_options)
+
+        mock_configparser.ConfigParser.assert_called_once_with()
+        conf = mock_configparser.ConfigParser.return_value
+        conf.read.assert_called_once_with(conf_path)
+        mock_add_extra_options.assert_called_once_with(extra_options, conf)
+        conf.write.assert_called_once_with(mock_string_io.return_value)
+        mock_string_io.return_value.getvalue.assert_called_once_with()

tests/unit/plugins/openstack/verification/tempest/test_manager.py

@@ -0,0 +1,233 @@
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import json
+import os
+import subprocess
+
+import mock
+
+from rally import exceptions
+from rally.plugins.openstack.verification.tempest import manager
+from tests.unit import test
+
+
+PATH = "rally.plugins.openstack.verification.tempest.manager"
+
+
+class TempestManagerTestCase(test.TestCase):
+    def test_run_environ_property(self):
+        mock.patch("%s.testr.TestrLauncher.run_environ" % PATH,
+                   new={"some": "key"}).start()
+        tempest = manager.TempestManager(mock.MagicMock(uuid="uuuiiiddd"))
+        env = {"some": "key",
+               "OS_TEST_PATH": os.path.join(tempest.repo_dir,
+                                            "tempest/test_discover"),
+               "TEMPEST_CONFIG": "tempest.conf",
+               "TEMPEST_CONFIG_DIR": os.path.dirname(tempest.configfile)}
+
+        self.assertEqual(env, tempest.run_environ)
+
+    def test_configfile_property(self):
+        tempest = manager.TempestManager(mock.MagicMock(uuid="uuuiiiddd"))
+        self.assertEqual(os.path.join(tempest.home_dir, "tempest.conf"),
+                         tempest.configfile)
+
+    @mock.patch("%s.config.read_configfile" % PATH)
+    def test_get_configuration(self, mock_read_configfile):
+        tempest = manager.TempestManager(mock.MagicMock(uuid="uuuiiiddd"))
+        self.assertEqual(mock_read_configfile.return_value,
+                         tempest.get_configuration())
+        mock_read_configfile.assert_called_once_with(tempest.configfile)
+
+    @mock.patch("%s.config.TempestConfigfileManager" % PATH)
+    def test_configure(self, mock_tempest_configfile_manager):
+        tempest = manager.TempestManager(mock.MagicMock(uuid="uuuiiiddd"))
+        cm = mock_tempest_configfile_manager.return_value
+        extra_options = mock.Mock()
+
+        self.assertEqual(cm.create.return_value,
+                         tempest.configure(extra_options))
+        mock_tempest_configfile_manager.assert_called_once_with(
+            tempest.verifier.deployment)
+        cm.create.assert_called_once_with(tempest.configfile, extra_options)
+
+    @mock.patch("%s.config.extend_configfile" % PATH)
+    def test_extend_configuration(self, mock_extend_configfile):
+        tempest = manager.TempestManager(mock.MagicMock(uuid="uuuiiiddd"))
+        extra_options = mock.Mock()
+        self.assertEqual(mock_extend_configfile.return_value,
+                         tempest.extend_configuration(extra_options))
+        mock_extend_configfile.assert_called_once_with(tempest.configfile,
+                                                       extra_options)
+
+    @mock.patch("six.moves.builtins.open", side_effect=mock.mock_open())
+    def test_override_configuration(self, mock_open):
+        tempest = manager.TempestManager(mock.MagicMock(uuid="uuuiiiddd"))
+        new_content = mock.Mock()
+
+        tempest.override_configuration(new_content)
+
+        mock_open.assert_called_once_with(tempest.configfile, "w")
+        mock_open.side_effect().write.assert_called_once_with(new_content)
+
+    @mock.patch("%s.os.path.exists" % PATH)
+    @mock.patch("%s.utils.check_output" % PATH)
+    def test_install_extension(self, mock_check_output, mock_exists):
+        tempest = manager.TempestManager(mock.MagicMock(uuid="uuuiiiddd",
+                                                        system_wide=True))
+        e = self.assertRaises(NotImplementedError, tempest.install_extension,
+                              None, None, {"key": "value"})
+        self.assertIn("verifiers don't support extra options", "%s" % e)
+
+        # case #1 system-wide installation
+        source = "https://github.com/example/example"
+        tempest.install_extension(source)
+
+        path = os.path.join(tempest.base_dir, "extensions")
+        mock_check_output.assert_called_once_with(
+            ["pip", "install", "--no-deps", "--src", path, "-e",
+             "git+https://github.com/example/example@master#egg=example"],
+            cwd=tempest.base_dir, env=tempest.environ)
+        self.assertFalse(mock_exists.called)
+
+        mock_check_output.reset_mock()
+
+        # case #2 virtual env with specified version
+        tempest.verifier.system_wide = False
+        version = "some"
+        tempest.install_extension(source, version=version)
+
+        test_reqs_path = os.path.join(tempest.base_dir, "extensions",
+                                      "example", "test-requirements.txt")
+        self.assertEqual([
+            mock.call([
+                "pip", "install", "--src", path, "-e",
+                "git+https://github.com/example/example@some#egg=example"],
+                cwd=tempest.base_dir, env=tempest.environ),
+            mock.call(["pip", "install", "-r", test_reqs_path],
+                      cwd=tempest.base_dir, env=tempest.environ)],
+            mock_check_output.call_args_list)
+        mock_exists.assert_called_once_with(test_reqs_path)
+
+    @mock.patch("%s.utils.check_output" % PATH)
+    def test_list_extensions(self, mock_check_output):
+        plugins_list = [
+            {"name": "some", "entry_point": "foo.bar", "location": "/tmp"},
+            {"name": "another", "entry_point": "bar.foo", "location": "/tmp"}
+        ]
+        mock_check_output.return_value = json.dumps(plugins_list)
+
+        tempest = manager.TempestManager(mock.MagicMock(uuid="uuuiiiddd"))
+
+        self.assertEqual(plugins_list, tempest.list_extensions())
+        self.assertEqual(1, mock_check_output.call_count)
+        mock_check_output.reset_mock()
+
+        mock_check_output.side_effect = subprocess.CalledProcessError("", "")
+        self.assertRaises(exceptions.RallyException, tempest.list_extensions)
+        self.assertEqual(1, mock_check_output.call_count)
+
+    @mock.patch("%s.TempestManager.list_extensions" % PATH)
+    @mock.patch("%s.os.path.exists" % PATH)
+    @mock.patch("%s.shutil.rmtree" % PATH)
+    def test_uninstall_extension(self, mock_rmtree, mock_exists,
+                                 mock_list_extensions):
+        plugins_list = [
+            {"name": "some", "entry_point": "foo.bar", "location": "/tmp"},
+            {"name": "another", "entry_point": "bar.foo", "location": "/tmp"}
+        ]
+        mock_list_extensions.return_value = plugins_list
+
+        tempest = manager.TempestManager(mock.MagicMock(uuid="uuuiiiddd"))
+
+        tempest.uninstall_extension("some")
+        mock_rmtree.assert_called_once_with(plugins_list[0]["location"])
+        mock_list_extensions.assert_called_once_with()
+
+        mock_rmtree.reset_mock()
+        mock_list_extensions.reset_mock()
+
+        self.assertRaises(exceptions.RallyException,
+                          tempest.uninstall_extension, "unexist")
+
+        mock_list_extensions.assert_called_once_with()
+        self.assertFalse(mock_rmtree.called)
+
+    @mock.patch("%s.TempestManager._transform_pattern" % PATH)
+    @mock.patch("%s.testr.TestrLauncher.list_tests" % PATH)
+    def test_list_tests(self, mock_testr_launcher_list_tests,
+                        mock__transform_pattern):
+        tempest = manager.TempestManager(mock.MagicMock(uuid="uuuiiiddd"))
+
+        self.assertEqual(mock_testr_launcher_list_tests.return_value,
+                         tempest.list_tests())
+        mock_testr_launcher_list_tests.assert_called_once_with("")
+        self.assertFalse(mock__transform_pattern.called)
+        mock_testr_launcher_list_tests.reset_mock()
+
+        pattern = mock.Mock()
+
+        self.assertEqual(mock_testr_launcher_list_tests.return_value,
+                         tempest.list_tests(pattern))
+        mock_testr_launcher_list_tests.assert_called_once_with(
+            mock__transform_pattern.return_value)
+        mock__transform_pattern.assert_called_once_with(pattern)
+
+    @mock.patch("%s.testr.TestrLauncher.validate_args" % PATH)
+    def test_validate_args(self, mock_testr_launcher_validate_args):
+        manager.TempestManager.validate_args({})
+        manager.TempestManager.validate_args({"pattern": "some.test"})
+        manager.TempestManager.validate_args({"pattern": "set=smoke"})
+        manager.TempestManager.validate_args({"pattern": "set=compute"})
+        manager.TempestManager.validate_args({"pattern": "set=full"})
+
+        e = self.assertRaises(exceptions.ValidationError,
+                              manager.TempestManager.validate_args,
+                              {"pattern": "foo=bar"})
+        self.assertEqual("Validation error: 'pattern' argument should be a "
+                         "regexp or set name (format: 'tempest.api.identity."
+                         "v3', 'set=smoke').", "%s" % e)
+
+        e = self.assertRaises(exceptions.ValidationError,
+                              manager.TempestManager.validate_args,
+                              {"pattern": "set=foo"})
+        self.assertIn("Test set 'foo' not found in available Tempest test "
+                      "sets. Available sets are ", "%s" % e)
+
+    def test__transform_pattern(self):
+        tempest = manager.TempestManager(mock.MagicMock(uuid="uuuiiiddd"))
+
+        self.assertEqual("foo", tempest._transform_pattern("foo"))
+        self.assertEqual("foo=bar", tempest._transform_pattern("foo=bar"))
+        self.assertEqual("", tempest._transform_pattern("set=full"))
+        self.assertEqual("smoke", tempest._transform_pattern("set=smoke"))
+        self.assertEqual("tempest.bar", tempest._transform_pattern("set=bar"))
+        self.assertEqual("tempest.api.compute",
+                         tempest._transform_pattern("set=compute"))
+
+    @mock.patch("%s.TempestManager._transform_pattern" % PATH)
+    def test__process_run_args(self, mock__transform_pattern):
+        tempest = manager.TempestManager(mock.MagicMock(uuid="uuuiiiddd"))
+
+        self.assertEqual({}, tempest._process_run_args({}))
+        self.assertFalse(mock__transform_pattern.called)
+
+        self.assertEqual({"foo": "bar"},
+                         tempest._process_run_args({"foo": "bar"}))
+        self.assertFalse(mock__transform_pattern.called)
+
+        pattern = mock.Mock()
+        self.assertEqual({"pattern": mock__transform_pattern.return_value},
+                         tempest._process_run_args({"pattern": pattern}))
+        mock__transform_pattern.assert_called_once_with(pattern)