Improve flavor detection

The patch adds a functionality for detecting two smallest
flavors available in the system in case, creation of resources
is not allowed and m1.nano and m1.micro flavors are not available.

Change-Id: Idc4fcd784385113a71fc8c33edd9c30be9c2bfd0
Story: 2002932
Task: 22919

config_tempest/flavors.py

@@ -13,6 +13,8 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+from operator import itemgetter
+
 from config_tempest.constants import LOG
 
 
@@ -27,65 +29,114 @@ class Flavors(object):
         self.client = client
         self.allow_creation = allow_creation
         self._conf = conf
+        self.flavor_list = self.client.list_flavors()['flavors']
 
     def create_tempest_flavors(self):
         """Find or create flavors and set them in conf.
 
         If 'flavor_ref' and 'flavor_ref_alt' are specified in conf, it will
-        first try to find those - otherwise it will try finding or creating
+        try to find them, if not found, it raises an Exception.
-        'm1.nano' and 'm1.micro' and overwrite those options in conf.
+        Otherwise it will try finding or creating 'm1.nano' and 'm1.micro'
+        flavors and set their ids in conf.
         """
-        # m1.nano flavor
+        prefs = [
+            {'key': 'flavor_ref', 'name': 'm1.nano', 'ram': 64},
+            {'key': 'flavor_ref_alt', 'name': 'm1.micro', 'ram': 128}
+        ]
+        for pref in prefs:
             flavor_id = None
-        if self._conf.has_option('compute', 'flavor_ref'):
+            if self._conf.has_option('compute', pref['key']):
-            flavor_id = self._conf.get('compute', 'flavor_ref')
+                flavor_id = self._conf.get('compute', pref['key'])
-        flavor_id = self.find_or_create_flavor(flavor_id, 'm1.nano', ram=64)
+                flavor_id = self.find_flavor_by_id(flavor_id)
-        self._conf.set('compute', 'flavor_ref', flavor_id)
+                if flavor_id is None:
+                    raise Exception("%s id '%s' specified by user doesn't"
+                                    " exist", pref['key'], flavor_id)
+            else:
+                # create m1.nano/m1.micro flavor
+                flavor_id = self.create_flavor(pref['name'], ram=pref['ram'])
+                self._conf.set('compute', pref['key'], flavor_id)
 
-        # m1.micro flavor
+    def create_flavor(self, flavor_name, ram=64, vcpus=1, disk=0):
-        alt_flavor_id = None
+        """Create flavors or try to discover two smallest ones available.
-        if self._conf.has_option('compute', 'flavor_ref_alt'):
-            alt_flavor_id = self._conf.get('compute', 'flavor_ref_alt')
-        alt_flavor_id = self.find_or_create_flavor(alt_flavor_id, 'm1.micro',
-                                                   ram=128)
-        self._conf.set('compute', 'flavor_ref_alt', alt_flavor_id)
 
-    def find_or_create_flavor(self, flavor_id, flavor_name,
+        :param flavor_name: flavor name to be created (usually m1.nano or
-                              ram=64, vcpus=1, disk=0):
+                            m1.micro)
-        """Try finding flavor by ID or name, create if not found.
-
-        :param flavor_id: first try finding the flavor by this
-        :param flavor_name: find by this if it was not found by ID, create new
-            flavor with this name if not found at allCLIENT_MOCK
         :param ram: memory of created flavor in MB
         :param vcpus: number of VCPUs for the flavor
         :param disk: size of disk for flavor in GB
         """
-        flavor = None
+        flavor_id = self.find_flavor_by_name(flavor_name)
-        flavors = self.client.list_flavors()['flavors']
+        if flavor_id is not None:
-        # try finding it by the ID first
+            LOG.info("(no change) Found flavor '%s'", flavor_name)
-        if flavor_id:
+            return flavor_id
-            found = [f for f in flavors if f['id'] == flavor_id]
+        elif self.allow_creation:
-            if found:
-                flavor = found[0]
-        # if not found, try finding it by name
-        if flavor_name and not flavor:
-            found = [f for f in flavors if f['name'] == flavor_name]
-            if found:
-                flavor = found[0]
-
-        if not flavor and not self.allow_creation:
-            raise Exception("Flavor '%s' not found, but resource creation"
-                            " isn't allowed. Either use '--create' or provide"
-                            " an existing flavor" % flavor_name)
-
-        if not flavor:
             LOG.info("Creating flavor '%s'", flavor_name)
-            flavor = self.client.create_flavor(name=flavor_name,
+            resp = self.client.create_flavor(name=flavor_name,
                                              ram=ram, vcpus=vcpus,
                                              disk=disk, id=None)
-            return flavor['flavor']['id']
+            return resp['flavor']['id']
         else:
-            LOG.info("(no change) Found flavor '%s'", flavor['name'])
+            if len(self.flavor_list) < 2:
+                raise Exception("Creation of flavors is not allowed and not "
+                                "enough available flavors found. Either use --"
+                                "create argument or create flavors manually.")
+            # return id of the discovered flavor
+            return self.discover_smallest_flavor(flavor_name)
 
-        return flavor['id']
+    def find_flavor_by_id(self, flavor_id):
+        """Look for a flavor by its id.
+
+        :type flavor_id: string
+        :return: flavor id or None if not found
+        :rtype: string or None
+        """
+        found = [f for f in self.flavor_list if f['id'] == flavor_id]
+        if found:
+            LOG.info("Found flavor '%s' by it's id '%s'",
+                     found[0]['name'], flavor_id)
+            # return flavor's id
+            return found[0]['id']
+        return None
+
+    def find_flavor_by_name(self, flavor_name):
+        """Look for a flavor by its name.
+
+        :type flavor_name: string
+        :return: flavor id or None if not found
+        :rtype: string or None
+        """
+        found = [f for f in self.flavor_list if f['name'] == flavor_name]
+        if found:
+            # return flavor's id
+            return found[0]['id']
+        return None
+
+    def discover_smallest_flavor(self, flavor_name=""):
+        """Discover the two smallest available flavors in the system.
+
+        If flavor_name contains "micro", the method returns the second
+        smallest flavor found.
+        :param flavor_name: [m1.nano, m1.micro]
+        """
+        LOG.warning("Flavor '%s' not found and creation is not allowed. "
+                    "Tying to autodetect the smallest flavor available.",
+                    flavor_name)
+        flavors = []
+        for flavor in self.flavor_list:
+            f = self.client.show_flavor(flavor['id'])['flavor']
+            flavors.append((f['name'], f['id'], f['ram'],
+                            f['disk'], f['vcpus']))
+
+        # order by ram, disk size and vcpus number and take first two of them
+        flavors = sorted(flavors, key=itemgetter(2, 3, 4))[:2]
+
+        f = None
+        if "micro" in flavor_name:
+            # take the second smaller one
+            f = flavors[1]
+        else:
+            f = flavors[0]
+        LOG.warning("Found '%s' flavor (id: '%s', ram: '%s', disk: '%s', "
+                    "vcpus: '%s') ", f[0], f[1], f[2], f[3], f[4])
+        # return flavor's id
+        return f[0]

config_tempest/tests/test_flavors.py

@@ -14,11 +14,15 @@
 # under the License.
 
 from fixtures import MonkeyPatch
+import logging
 import mock
 
 from config_tempest.flavors import Flavors
 from config_tempest.tests.base import BaseConfigTempestTest
 
+# disable logging when running unit tests
+logging.disable(logging.CRITICAL)
+
 
 class TestFlavors(BaseConfigTempestTest):
     """Flavors test class
@@ -36,77 +40,103 @@ class TestFlavors(BaseConfigTempestTest):
         super(TestFlavors, self).setUp()
         self.conf = self._get_conf("v2.0", "v3")
         self.client = self._get_clients(self.conf).flavors
+        return_value = {"flavors": [{"id": "MyFakeID", "name": "MyID"}]}
+        mock_function = mock.Mock(return_value=return_value)
+        self.useFixture(MonkeyPatch(self.CLIENT_MOCK + '.list_flavors',
+                                    mock_function))
         self.Service = Flavors(self.client, True, self.conf)
 
-    def _mock_create_tempest_flavor(self, mock_function):
+    def test_create_tempest_flavors(self):
-        func2mock = 'config_tempest.flavors.Flavors.find_or_create_flavor'
+        self.Service.flavor_list = []
+        mock_function = mock.Mock(return_value="FakeID")
+        func2mock = 'config_tempest.flavors.Flavors.create_flavor'
         self.useFixture(MonkeyPatch(func2mock, mock_function))
         self.Service.create_tempest_flavors()
-
-    def _mock_find_or_create_flavor(self, return_value, func2mock, flavor_name,
-                                    expected_resp, allow_creation=False,
-                                    flavor_id=None):
-        self.Service.allow_creation = allow_creation
-        mock_function = mock.Mock(return_value=return_value)
-        self.useFixture(MonkeyPatch(self.CLIENT_MOCK + func2mock,
-                                    mock_function))
-        resp = self.Service.find_or_create_flavor(flavor_id=flavor_id,
-                                                  flavor_name=flavor_name)
-        self.assertEqual(resp, expected_resp)
-
-    def test_create_tempest_flavors(self):
-        mock_function = mock.Mock(return_value="FakeID")
-        self._mock_create_tempest_flavor(mock_function)
         self.assertEqual(self.conf.get('compute', 'flavor_ref'), "FakeID")
         self.assertEqual(self.conf.get('compute', 'flavor_ref_alt'), "FakeID")
-        calls = [mock.call(None, 'm1.nano', ram=64),
+        calls = [mock.call('m1.nano', ram=64),
-                 mock.call(None, 'm1.micro', ram=128)]
+                 mock.call('m1.micro', ram=128)]
         mock_function.assert_has_calls(calls, any_order=True)
 
-    def test_create_tempest_flavors_overwrite(self):
+    def check_call_of_discover_smallest_flavor(self):
-        mock_function = mock.Mock(return_value="FakeID")
+        self.Service.flavor_list = [{'id': 'FAKE', 'name': 'Fake_flavor'},
-        self.conf.set('compute', 'flavor_ref', "FAKE_ID")
+                                    {'id': 'FAKE_1', 'name': 'Fake_flavor_1'}]
-        self.conf.set('compute', 'flavor_ref_alt', "FAKE_ID")
+        self.Service.allow_creation = False
-        self._mock_create_tempest_flavor(mock_function)
+        func2mock = 'config_tempest.flavors.Flavors.discover_smallest_flavor'
-        calls = [mock.call("FAKE_ID", 'm1.nano', ram=64),
+        mock_function = mock.Mock()
-                 mock.call("FAKE_ID", 'm1.micro', ram=128)]
+        self.useFixture(MonkeyPatch(func2mock, mock_function))
+        self.Service.create_flavor('nano')
+        calls = [mock.call('nano')]
         mock_function.assert_has_calls(calls, any_order=True)
 
+    def test_create_tempest_flavors_overwrite_flavor_ref_not_exist(self):
+        self.conf.set('compute', 'flavor_ref', "FAKE_ID")
+        try:
+            self.Service.create_tempest_flavors()
+        except Exception:
+            return
+        # it should have ended in the except block above
+        self.assertTrue(False)
+
+    def test_create_tempest_flavors_overwrite_flavor_ref_alt_not_exist(self):
+        self.Service.flavor_list = [{'id': 'FAKE', 'name': 'Fake_flavor'}]
+        self.conf.set('compute', 'flavor_ref', 'FAKE')
+        self.conf.set('compute', 'flavor_ref_alt', 'FAKE_ID')
+        try:
+            self.Service.create_tempest_flavors()
+        except Exception:
+            self.assertEqual(self.conf.get('compute', 'flavor_ref'), 'FAKE')
+            return
+        # it should have ended in the except block above
+        self.assertTrue(False)
+
     def test_create_flavor_not_allowed(self):
         # mock list_flavors() to return empty list
         self.Service.allow_creation = False
-        mock_function = mock.Mock(return_value={"flavors": []})
+        self.Service.flavor_list = []
-        self.useFixture(MonkeyPatch(self.CLIENT_MOCK + '.list_flavors',
+        try:
-                                    mock_function))
+            self.Service.create_flavor('name')
-        exc = Exception
+        except Exception:
-        self.assertRaises(exc,
+            return
-                          self.Service.find_or_create_flavor,
+        # it should have ended in the except block above
-                          flavor_id="id",
+        self.assertTrue(False)
-                          flavor_name="name")
+
+        # not enough flavors found
+        self.Service.flavor_list = [{'id': 'FAKE', 'name': 'fake_name'}]
+        try:
+            self.Service.create_flavor('name')
+        except Exception:
+            return
+        # it should have ended in the except block above
+        self.assertTrue(False)
 
     def test_create_flavor(self):
         return_value = {"flavor": {"id": "MyFakeID", "name": "MyID"}}
-        # mock list_flavors() to return empty list
+        self.Service.flavor_list = []
-        mock_function = mock.Mock(return_value={"flavors": []})
+        mock_function = mock.Mock(return_value=return_value)
-        self.useFixture(MonkeyPatch(self.CLIENT_MOCK + '.list_flavors',
+        self.useFixture(MonkeyPatch(self.CLIENT_MOCK + '.create_flavor',
                                     mock_function))
-        self._mock_find_or_create_flavor(return_value=return_value,
+        resp = self.Service.create_flavor(flavor_name="MyID")
-                                         func2mock='.create_flavor',
+        self.assertEqual(resp, return_value['flavor']['id'])
-                                         flavor_name="MyID",
-                                         expected_resp="MyFakeID",
-                                         allow_creation=True)
 
     def test_find_flavor_by_id(self):
         return_value = {"flavors": self.FLAVORS_LIST}
-        self._mock_find_or_create_flavor(return_value=return_value,
+        mock_function = mock.Mock(return_value=return_value)
-                                         func2mock='.list_flavors',
+        self.useFixture(MonkeyPatch(self.CLIENT_MOCK + '.list_flavors',
-                                         flavor_id="MyFakeID",
+                                    mock_function))
-                                         flavor_name=None,
+        resp = self.Service.find_flavor_by_id("MyFakeID")
-                                         expected_resp="MyFakeID")
+        self.assertEqual(resp, "MyFakeID")
+        # test no flavor found case
+        resp = self.Service.find_flavor_by_id("NotExist")
+        self.assertEqual(resp, None)
 
     def test_find_flavor_by_name(self):
         return_value = {"flavors": self.FLAVORS_LIST}
-        self._mock_find_or_create_flavor(return_value=return_value,
+        mock_function = mock.Mock(return_value=return_value)
-                                         func2mock='.list_flavors',
+        self.useFixture(MonkeyPatch(self.CLIENT_MOCK + '.list_flavors',
-                                         flavor_name="MyID",
+                                    mock_function))
-                                         expected_resp="MyFakeID")
+        resp = self.Service.find_flavor_by_name("MyID")
+        self.assertEqual(resp, "MyFakeID")
+        # test no flavor found case
+        resp = self.Service.find_flavor_by_name("NotExist")
+        self.assertEqual(resp, None)

roles/generate-tempestconf-file/tasks/test-demo-user.yaml

@@ -1,11 +1,10 @@
-  - name: Generate tempest configuration file as demo user (expected to fail)
+  - name: Generate tempest.conf as demo user (tool will autodiscover flavors)
     shell: |
         ./generate-tempestconf.sh
     args:
       chdir: "{{ tempestconf_src_relative_path }}"
       executable: /bin/bash
     register: result
-    failed_when: result.rc == 0
 
   - name: Create m1.nano and m1.micro flavors for demo user
     shell: |