Improve and fix images & flavors handling

Fix images matching algo (exact match favored compared to partial match)
Cache images & flavors list
Fix OpenStack session init logic (was called too late and only on deploy)

Change-Id: I6c723f03925a8b2c4b3408fcfb6c78d5af47328d

translator/common/flavors.py

@@ -12,17 +12,17 @@
 
 import logging
 
-# NOTE(aloga): this should be safe. If we do not have the clients, we won't
-# have the session below, therefore the clients won't be ever called.
 try:
     import novaclient.client
+    client_available = True
 except ImportError:
+    client_available = False
     pass
 
 log = logging.getLogger('heat-translator')
 
 
-_FLAVORS = {
+PREDEF_FLAVORS = {
     'm1.xlarge': {'mem_size': 16384, 'disk_size': 160, 'num_cpus': 8},
     'm1.large': {'mem_size': 8192, 'disk_size': 80, 'num_cpus': 4},
     'm1.medium': {'mem_size': 4096, 'disk_size': 40, 'num_cpus': 2},
@@ -34,10 +34,16 @@ _FLAVORS = {
 
 SESSION = None
 
+FLAVORS = {}
+
 
 def get_flavors():
-    ret = {}
+    global FLAVORS
-    if SESSION is not None:
+
+    if FLAVORS:
+        return FLAVORS
+
+    if SESSION is not None and client_available:
         try:
             client = novaclient.client.Client("2", session=SESSION)
         except Exception as e:
@@ -46,9 +52,13 @@ def get_flavors():
                        'Openstack Exception: %s') % str(e))
         else:
             for flv in client.flavors.list(detailed=True):
-                ret[str(flv.name)] = {
+                FLAVORS[str(flv.name)] = {
                     "mem_size": flv.ram,
                     "disk_size": flv.disk,
                     "num_cpus": flv.vcpus
                 }
-    return ret or _FLAVORS
+
+    if not FLAVORS:
+        FLAVORS = PREDEF_FLAVORS
+
+    return FLAVORS

translator/common/images.py

@@ -12,17 +12,17 @@
 
 import logging
 
-# NOTE(aloga): this should be safe. If we do not have the clients, we won't
-# have the session below, therefore the clients won't be ever called.
 try:
     import glanceclient.client
+    client_available = True
 except ImportError:
+    client_available = False
     pass
 
 log = logging.getLogger('heat-translator')
 
 
-_IMAGES = {
+PREDEF_IMAGES = {
     'ubuntu-software-config-os-init': {'architecture': 'x86_64',
                                        'type': 'Linux',
                                        'distribution': 'Ubuntu',
@@ -59,11 +59,16 @@ _IMAGES = {
 
 SESSION = None
 
+IMAGES = {}
+
 
 def get_images():
-    ret = {}
+    global IMAGES
 
-    if SESSION is not None:
+    if IMAGES:
+        return IMAGES
+
+    if SESSION is not None and client_available:
         try:
             client = glanceclient.client.Client("2", session=SESSION)
         except Exception as e:
@@ -72,10 +77,15 @@ def get_images():
                        'Openstack Exception: %s') % str(e))
         else:
             for image in client.images.list():
+                image_name = image.name.encode('ascii', 'ignore')
                 metadata = ["architecture", "type", "distribution", "version"]
                 if any(key in image.keys() for key in metadata):
-                    ret = [image["name"]] = {}
+                    IMAGES[image_name] = {}
                     for key in metadata:
                         if key in image.keys():
-                            ret[image["name"]][key] = image[key]
+                            IMAGES[image_name][key] = image[key]
-    return ret or _IMAGES
+
+    if not IMAGES:
+        IMAGES = PREDEF_IMAGES
+
+    return IMAGES

translator/hot/tosca/tosca_compute.py

@@ -186,7 +186,10 @@ class ToscaCompute(HotResource):
             return this_list
         matching_images = []
         for image in this_list:
-            if this_dict[image][attr].lower() == str(prop).lower():
+            if attr in this_dict[image]:
+                if this_dict[image][attr].lower() == str(prop).lower():
+                    matching_images.insert(0, image)
+            else:
                 matching_images.append(image)
         return matching_images
 

translator/shell.py

@@ -22,12 +22,19 @@ import yaml
 # NOTE(aloga): As per upstream developers requirement this needs to work
 # without the clients, therefore we need to pass if we cannot import them
 try:
-    import heatclient.client
     from keystoneauth1 import loading
 except ImportError:
-    has_clients = False
+    keystone_client_avail = False
 else:
-    has_clients = True
+    keystone_client_avail = True
+
+try:
+    import heatclient.client
+except ImportError:
+    heat_client_avail = False
+else:
+    heat_client_avail = True
+
 
 from toscaparser.tosca_template import ToscaTemplate
 from toscaparser.utils.gettextutils import _
@@ -103,7 +110,7 @@ class TranslatorShell(object):
         return parser
 
     def _append_global_identity_args(self, parser, argv):
-        if not has_clients:
+        if not keystone_client_avail:
             return
 
         loading.register_session_argparse_arguments(parser)
@@ -138,24 +145,33 @@ class TranslatorShell(object):
                          'validation.') % {'template_file': template_file})
                 print(msg)
             else:
+                if keystone_client_avail:
+                    try:
+                        keystone_auth = (
+                            loading.load_auth_from_argparse_arguments(args)
+                        )
+                        keystone_session = (
+                            loading.load_session_from_argparse_arguments(
+                                args,
+                                auth=keystone_auth
+                            )
+                        )
+                        images.SESSION = keystone_session
+                        flavors.SESSION = keystone_session
+                    except Exception:
+                        keystone_session = None
+
                 hot = self._translate(template_type, template_file,
                                       parsed_params, a_file, deploy)
                 if hot and deploy:
-                    if not has_clients:
+                    if not keystone_client_avail or not heat_client_avail:
-                        raise RuntimeError(_('Could not find OpenStack '
+                        raise RuntimeError(_('Could not find Heat or Keystone'
-                                             'clients and libs, aborting '))
+                                             'client to deploy, aborting '))
+                    if not keystone_session:
+                        raise RuntimeError(_('Impossible to login with '
+                                             'Keystone to deploy on Heat, '
+                                             'please check your credentials'))
 
-                    keystone_auth = (
-                        loading.load_auth_from_argparse_arguments(args)
-                    )
-                    keystone_session = (
-                        loading.load_session_from_argparse_arguments(
-                            args,
-                            auth=keystone_auth
-                        )
-                    )
-                    images.SESSION = keystone_session
-                    flavors.SESSION = keystone_session
                     self.deploy_on_heat(keystone_session, keystone_auth,
                                         hot, parsed_params)