Normalize images

We normalize a bunch of our objects now, but image support is so old we
don't. It's also wonky and hand-picked between v1 and v2 as opposed to
being both v1 and v2 like other objects that couldn't escape epic API
breakage.

Change-Id: Ie17b9888c8f5a33231c366abebb8b505fc9592e6

releasenotes/notes/normalize-images-1331bea7bfffa36a.yaml

@@ -0,0 +1,6 @@
+---
+features:
+  - Image dicts that are returned are now normalized across glance v1
+    and glance v2. Extra key/value properties are now both in the root
+    dict and in a properties dict. Additionally, cloud and region have
+    been added like they are for server.

shade/_utils.py

@@ -30,6 +30,25 @@ from shade import exc
 from shade import meta
 
 _decorated_methods = []
+_IMAGE_FIELDS = (
+    'checksum',
+    'container_format',
+    'created_at',
+    'disk_format',
+    'file',
+    'id',
+    'min_disk',
+    'min_ram',
+    'name',
+    'owner',
+    'protected',
+    'schema',
+    'size',
+    'status',
+    'tags',
+    'updated_at',
+    'virtual_size',
+)
 
 
 def _iterate_timeout(timeout, message, wait=2):
@@ -187,6 +206,34 @@ def normalize_server(server, cloud_name, region_name):
     return server
 
 
+def normalize_images(images, cloud):
+    ret = []
+    for image in images:
+        ret.append(normalize_image(image, cloud))
+    return ret
+
+
+def normalize_image(image, cloud):
+    new_image = munch.Munch(location=cloud.current_location)
+    properties = image.pop('properties', {})
+    visibility = image.pop('visibility', None)
+    if visibility:
+        is_public = (visibility == 'public')
+    else:
+        is_public = image.pop('is_public', False)
+        visibility = 'public' if is_public else 'private'
+
+    for field in _IMAGE_FIELDS:
+        new_image[field] = image.pop(field, None)
+    for key, val in image.items():
+        properties[key] = val
+        new_image[key] = val
+    new_image['properties'] = properties
+    new_image['visibility'] = visibility
+    new_image['is_public'] = is_public
+    return new_image
+
+
 def normalize_keystone_services(services):
     """Normalize the structure of keystone services
 

shade/openstackcloud.py

@@ -1625,7 +1625,7 @@ class OpenStackCloud(object):
                 images.append(image)
             elif image.status != 'DELETED':
                 images.append(image)
-        return images
+        return _utils.normalize_images(images, cloud=self)
 
     def list_floating_ip_pools(self):
         """List all available floating IP pools.

shade/tests/fakes.py

@@ -74,6 +74,20 @@ class FakeImage(object):
         self.id = id
         self.name = name
         self.status = status
+        self.checksum = ''
+        self.container_format = 'bare'
+        self.created_at = ''
+        self.disk_format = 'raw'
+        self.file = ''
+        self.min_disk = 0
+        self.min_ram = 0
+        self.owner = ''
+        self.protected = False
+        self.schema = ''
+        self.size = 0
+        self.tags = []
+        self.updated_at = ''
+        self.virtual_size = 0
 
 
 class FakeProject(object):

shade/tests/unit/test_caching.py

@@ -101,6 +101,9 @@ class TestMemoryCache(base.TestCase):
         super(TestMemoryCache, self).setUp(
             cloud_config_fixture='clouds_cache.yaml')
 
+    def _image_dict(self, fake_image):
+        return _utils.normalize_image(meta.obj_to_dict(fake_image), self.cloud)
+
     def test_openstack_cloud(self):
         self.assertIsInstance(self.cloud, shade.OpenStackCloud)
 
@@ -302,7 +305,7 @@ class TestMemoryCache(base.TestCase):
         self.assertEqual([], self.cloud.list_images())
 
         fake_image = fakes.FakeImage('22', '22 name', 'success')
-        fake_image_dict = meta.obj_to_dict(fake_image)
+        fake_image_dict = self._image_dict(fake_image)
         glance_mock.images.list.return_value = [fake_image]
         self.cloud.list_images.invalidate(self.cloud)
         self.assertEqual([fake_image_dict], self.cloud.list_images())
@@ -310,14 +313,14 @@ class TestMemoryCache(base.TestCase):
     @mock.patch.object(shade.OpenStackCloud, 'glance_client')
     def test_list_images_ignores_unsteady_status(self, glance_mock):
         steady_image = fakes.FakeImage('68', 'Jagr', 'active')
-        steady_image_dict = meta.obj_to_dict(steady_image)
+        steady_image_dict = self._image_dict(steady_image)
         for status in ('queued', 'saving', 'pending_delete'):
             active_image = fakes.FakeImage(self.getUniqueString(),
                                            self.getUniqueString(), status)
             glance_mock.images.list.return_value = [active_image]
-            active_image_dict = meta.obj_to_dict(active_image)
+            active_image_dict = self._image_dict(active_image)
-            self.assertEqual([active_image_dict],
+
-                             self.cloud.list_images())
+            self.assertEqual([active_image_dict], self.cloud.list_images())
             glance_mock.images.list.return_value = [active_image, steady_image]
             # Should expect steady_image to appear if active wasn't cached
             self.assertEqual([active_image_dict, steady_image_dict],
@@ -330,7 +333,7 @@ class TestMemoryCache(base.TestCase):
         for status in ('active', 'deleted', 'killed'):
             active_image = fakes.FakeImage(self.getUniqueString(),
                                            self.getUniqueString(), status)
-            active_image_dict = meta.obj_to_dict(active_image)
+            active_image_dict = self._image_dict(active_image)
             if not first_image:
                 first_image = active_image_dict
             glance_mock.images.list.return_value = [active_image]
@@ -367,10 +370,10 @@ class TestMemoryCache(base.TestCase):
                     'owner_specified.shade.sha256': mock.ANY,
                     'owner_specified.shade.object': 'images/42 name',
                     'is_public': False}}
-        fake_image_dict = meta.obj_to_dict(fake_image)
+        fake_image_dict = self._image_dict(fake_image)
         glance_mock.images.create.assert_called_with(**args)
         glance_mock.images.update.assert_called_with(
-            data=mock.ANY, image=fake_image_dict)
+            data=mock.ANY, image=meta.obj_to_dict(fake_image))
         self.assertEqual([fake_image_dict], self.cloud.list_images())
 
     @mock.patch.object(occ.cloud_config.CloudConfig, 'get_api_version')
@@ -396,7 +399,7 @@ class TestMemoryCache(base.TestCase):
         glance_mock.images.create.assert_called_with(**args)
         glance_mock.images.upload.assert_called_with(
             image_data=mock.ANY, image_id=fake_image.id)
-        fake_image_dict = meta.obj_to_dict(fake_image)
+        fake_image_dict = self._image_dict(fake_image)
         self.assertEqual([fake_image_dict], self.cloud.list_images())
 
     @mock.patch.object(occ.cloud_config.CloudConfig, 'get_api_version')
@@ -440,7 +443,7 @@ class TestMemoryCache(base.TestCase):
         glance_mock.images.create.assert_called_with(**args)
         glance_mock.images.upload.assert_called_with(
             image_data=mock.ANY, image_id=fake_image.id)
-        fake_image_dict = meta.obj_to_dict(fake_image)
+        fake_image_dict = self._image_dict(fake_image)
         self.assertEqual([fake_image_dict], self.cloud.list_images())
 
     @mock.patch.object(occ.cloud_config.CloudConfig, 'get_api_version')
@@ -468,7 +471,7 @@ class TestMemoryCache(base.TestCase):
         glance_mock.images.create.assert_called_with(**args)
         glance_mock.images.upload.assert_called_with(
             image_data=mock.ANY, image_id=fake_image.id)
-        fake_image_dict = meta.obj_to_dict(fake_image)
+        fake_image_dict = self._image_dict(fake_image)
         self.assertEqual([fake_image_dict], self.cloud.list_images())
 
     @mock.patch.object(occ.cloud_config.CloudConfig, 'get_api_version')
@@ -496,7 +499,7 @@ class TestMemoryCache(base.TestCase):
         glance_mock.images.create.assert_called_with(**args)
         glance_mock.images.upload.assert_called_with(
             image_data=mock.ANY, image_id=fake_image.id)
-        fake_image_dict = meta.obj_to_dict(fake_image)
+        fake_image_dict = self._image_dict(fake_image)
         self.assertEqual([fake_image_dict], self.cloud.list_images())
 
     @mock.patch.object(occ.cloud_config.CloudConfig, 'get_api_version')
@@ -568,34 +571,30 @@ class TestMemoryCache(base.TestCase):
                 'owner_specified.shade.object': object_path,
                 'image_id': 'a35e8afc-cae9-4e38-8441-2cd465f79f7b'}
         glance_mock.images.update.assert_called_with(**args)
-        fake_image_dict = meta.obj_to_dict(fake_image)
+        fake_image_dict = self._image_dict(fake_image)
         self.assertEqual([fake_image_dict], self.cloud.list_images())
 
     @mock.patch.object(shade.OpenStackCloud, 'glance_client')
     def test_cache_no_cloud_name(self, glance_mock):
-        class FakeImage(object):
-            status = 'active'
-            name = 'None Test Image'
 
-            def __init__(self, id):
-                self.id = id
-
-        fi = FakeImage(id=1)
-        glance_mock.images.list.return_value = [fi]
         self.cloud.name = None
+        fi = fakes.FakeImage(id=1, name='None Test Image', status='active')
+        fi_dict = self._image_dict(fi)
+        glance_mock.images.list.return_value = [fi]
         self.assertEqual(
-            meta.obj_list_to_dict([fi]),
+            [fi_dict],
             self.cloud.list_images())
         # Now test that the list was cached
-        fi2 = FakeImage(id=2)
+        fi2 = fakes.FakeImage(id=2, name='None Test Image', status='active')
+        fi2_dict = self._image_dict(fi2)
         glance_mock.images.list.return_value = [fi, fi2]
         self.assertEqual(
-            meta.obj_list_to_dict([fi]),
+            [fi_dict],
             self.cloud.list_images())
         # Invalidation too
         self.cloud.list_images.invalidate(self.cloud)
         self.assertEqual(
-            meta.obj_list_to_dict([fi, fi2]),
+            [fi_dict, fi2_dict],
             self.cloud.list_images())