Add method to cleanup autocreated image objects

This shouldn't really be needed, as the objects should get cleaned up
automatically. BUT - if things leak, this method can be used to delete
any objects shade has uploaded on behalf of the user for deleting
images.

While in there, clean up test_image to use a few more good practices.

Change-Id: Ifb697944856e1922517074d84a7c00a4af75b1e6

releasenotes/notes/delete-autocreated-1839187b0aa35022.yaml

@@ -0,0 +1,5 @@
+---
+features:
+  - Added new method, delete_autocreated_image_objects
+    that can be used to delete any leaked objects shade
+    may have created on behalf of the user.

shade/openstackcloud.py

@@ -50,6 +50,8 @@ from shade import _utils
 
 OBJECT_MD5_KEY = 'x-object-meta-x-shade-md5'
 OBJECT_SHA256_KEY = 'x-object-meta-x-shade-sha256'
+OBJECT_AUTOCREATE_KEY = 'x-object-meta-x-shade-autocreated'
+OBJECT_AUTOCREATE_CONTAINER = 'images'
 IMAGE_MD5_KEY = 'owner_specified.shade.md5'
 IMAGE_SHA256_KEY = 'owner_specified.shade.sha256'
 IMAGE_OBJECT_KEY = 'owner_specified.shade.object'
@@ -4526,7 +4528,7 @@ class OpenStackCloud(
         return up_to_date
 
     def create_image(
-            self, name, filename=None, container='images',
+            self, name, filename=None, container=OBJECT_AUTOCREATE_CONTAINER,
             md5=None, sha256=None,
             disk_format=None, container_format=None,
             disable_vendor_agent=True,
@@ -4823,6 +4825,7 @@ class OpenStackCloud(
         self.create_object(
             container, name, filename,
             md5=md5, sha256=sha256,
+            metadata={OBJECT_AUTOCREATE_KEY: 'true'},
             **{'content-type': 'application/octet-stream'})
         if not current_image:
             current_image = self.get_image(name)
@@ -7561,11 +7564,13 @@ class OpenStackCloud(
         return self._object_store_client.get(
             container, params=dict(format='json'))
 
-    def delete_object(self, container, name):
+    def delete_object(self, container, name, meta=None):
         """Delete an object from a container.
 
         :param string container: Name of the container holding the object.
         :param string name: Name of the object to delete.
+        :param dict meta: Metadata for the object in question. (optional, will
+                          be fetched if not provided)
 
         :returns: True if delete succeeded, False if the object was not found.
 
@@ -7580,7 +7585,8 @@ class OpenStackCloud(
         #   Errors:
         # We should ultimately do something with that
         try:
-            meta = self.get_object_metadata(container, name)
+            if not meta:
+                meta = self.get_object_metadata(container, name)
             if not meta:
                 return False
             params = {}
@@ -7594,6 +7600,28 @@ class OpenStackCloud(
         except OpenStackCloudHTTPError:
             return False
 
+    def delete_autocreated_image_objects(
+            self, container=OBJECT_AUTOCREATE_CONTAINER):
+        """Delete all objects autocreated for image uploads.
+
+        This method should generally not be needed, as shade should clean up
+        the objects it uses for object-based image creation. If something
+        goes wrong and it is found that there are leaked objects, this method
+        can be used to delete any objects that shade has created on the user's
+        behalf in service of image uploads.
+        """
+        # This method only makes sense on clouds that use tasks
+        if not self.image_api_use_tasks:
+            return False
+
+        deleted = False
+        for obj in self.list_objects(container):
+            meta = self.get_object_metadata(container, obj['name'])
+            if meta.get(OBJECT_AUTOCREATE_KEY) == 'true':
+                if self.delete_object(container, obj['name'], meta):
+                    deleted = True
+        return deleted
+
     def get_object_metadata(self, container, name):
         try:
             return self._object_store_client.head(

shade/tests/unit/base.py

@@ -408,6 +408,10 @@ class RequestsMockTestCase(BaseTestCase):
         return _RoleData(role_id, role_name, {'role': response},
                          {'role': request})
 
+    def use_nothing(self):
+        self.calls = []
+        self._uri_registry.clear()
+
     def use_keystone_v3(self, catalog='catalog-v3.json'):
         self.adapter = self.useFixture(rm_fixture.Fixture())
         self.calls = []

shade/tests/unit/test_image.py

@@ -24,6 +24,7 @@ import munch
 import six
 
 import shade
+import shade.openstackcloud
 from shade import exc
 from shade import meta
 from shade.tests import fakes
@@ -44,6 +45,8 @@ class BaseTestImage(base.RequestsMockTestCase):
         self.fake_image_dict = fakes.make_fake_image(image_id=self.image_id)
         self.fake_search_return = {'images': [self.fake_image_dict]}
         self.output = uuid.uuid4().bytes
+        self.image_name = self.getUniqueString('image')
+        self.container_name = self.getUniqueString('container')
 
 
 class TestImage(BaseTestImage):
@@ -258,8 +261,6 @@ class TestImage(BaseTestImage):
 
     def test_create_image_task(self):
         self.cloud.image_api_use_tasks = True
-        image_name = 'name-99'
-        container_name = 'image_upload_v2_test_container'
         endpoint = self.cloud._object_store_client.get_endpoint()
 
         task_id = str(uuid.uuid4())
@@ -286,18 +287,18 @@ class TestImage(BaseTestImage):
                      slo={'min_segment_size': 500})),
             dict(method='HEAD',
                  uri='{endpoint}/{container}'.format(
-                     endpoint=endpoint, container=container_name),
+                     endpoint=endpoint, container=self.container_name),
                  status_code=404),
             dict(method='PUT',
                  uri='{endpoint}/{container}'.format(
-                     endpoint=endpoint, container=container_name),
+                     endpoint=endpoint, container=self.container_name),
                  status_code=201,
                  headers={'Date': 'Fri, 16 Dec 2016 18:21:20 GMT',
                           'Content-Length': '0',
                           'Content-Type': 'text/html; charset=UTF-8'}),
             dict(method='HEAD',
                  uri='{endpoint}/{container}'.format(
-                     endpoint=endpoint, container=container_name),
+                     endpoint=endpoint, container=self.container_name),
                  headers={'Content-Length': '0',
                           'X-Container-Object-Count': '0',
                           'Accept-Ranges': 'bytes',
@@ -309,13 +310,13 @@ class TestImage(BaseTestImage):
                          'Content-Type': 'text/plain; charset=utf-8'}),
             dict(method='HEAD',
                  uri='{endpoint}/{container}/{object}'.format(
-                     endpoint=endpoint, container=container_name,
+                     endpoint=endpoint, container=self.container_name,
-                     object=image_name),
+                     object=self.image_name),
                  status_code=404),
             dict(method='PUT',
                  uri='{endpoint}/{container}/{object}'.format(
-                     endpoint=endpoint, container=container_name,
+                     endpoint=endpoint, container=self.container_name,
-                     object=image_name),
+                     object=self.image_name),
                  status_code=201,
                  validate=dict(
                      headers={'x-object-meta-x-shade-md5': fakes.NO_MD5,
@@ -329,8 +330,9 @@ class TestImage(BaseTestImage):
                      json=dict(
                          type='import', input={
                              'import_from': '{container}/{object}'.format(
-                                 container=container_name, object=image_name),
+                                 container=self.container_name,
-                             'image_properties': {'name': image_name}}))
+                                 object=self.image_name),
+                             'image_properties': {'name': self.image_name}}))
                  ),
             dict(method='GET',
                  uri='https://image.example.com/v2/tasks/{id}'.format(
@@ -348,8 +350,8 @@ class TestImage(BaseTestImage):
                  validate=dict(
                      json=sorted([{u'op': u'add',
                                    u'value': '{container}/{object}'.format(
-                                       container=container_name,
+                                       container=self.container_name,
-                                       object=image_name),
+                                       object=self.image_name),
                                    u'path': u'/owner_specified.shade.object'},
                                   {u'op': u'add', u'value': fakes.NO_MD5,
                                    u'path': u'/owner_specified.shade.md5'},
@@ -362,8 +364,8 @@ class TestImage(BaseTestImage):
                  ),
             dict(method='HEAD',
                  uri='{endpoint}/{container}/{object}'.format(
-                     endpoint=endpoint, container=container_name,
+                     endpoint=endpoint, container=self.container_name,
-                     object=image_name),
+                     object=self.image_name),
                  headers={
                      'X-Timestamp': '1429036140.50253',
                      'X-Trans-Id': 'txbbb825960a3243b49a36f-005a0dadaedfw1',
@@ -377,15 +379,94 @@ class TestImage(BaseTestImage):
                      'Etag': fakes.NO_MD5}),
             dict(method='DELETE',
                  uri='{endpoint}/{container}/{object}'.format(
-                     endpoint=endpoint, container=container_name,
+                     endpoint=endpoint, container=self.container_name,
-                     object=image_name)),
+                     object=self.image_name)),
             dict(method='GET', uri='https://image.example.com/v2/images',
                  json=self.fake_search_return)
         ])
 
         self.cloud.create_image(
-            image_name, self.imagefile.name, wait=True, timeout=1,
+            self.image_name, self.imagefile.name, wait=True, timeout=1,
-            is_public=False, container=container_name)
+            is_public=False, container=self.container_name)
+
+        self.assert_calls()
+
+    def test_delete_autocreated_no_tasks(self):
+        self.use_nothing()
+        self.cloud.image_api_use_tasks = False
+        deleted = self.cloud.delete_autocreated_image_objects(
+            container=self.container_name)
+        self.assertFalse(deleted)
+        self.assert_calls()
+
+    def test_delete_autocreated_image_objects(self):
+        self.use_keystone_v3()
+        self.cloud.image_api_use_tasks = True
+        endpoint = self.cloud._object_store_client.get_endpoint()
+        other_image = self.getUniqueString('no-delete')
+
+        self.register_uris([
+            dict(method='GET',
+                 uri=self.get_mock_url(
+                     service_type='object-store',
+                     resource=self.container_name,
+                     qs_elements=['format=json']),
+                 json=[{
+                     'content_type': 'application/octet-stream',
+                     'bytes': 1437258240,
+                     'hash': '249219347276c331b87bf1ac2152d9af',
+                     'last_modified': '2015-02-16T17:50:05.289600',
+                     'name': other_image,
+                 }, {
+                     'content_type': 'application/octet-stream',
+                     'bytes': 1290170880,
+                     'hash': fakes.NO_MD5,
+                     'last_modified': '2015-04-14T18:29:00.502530',
+                     'name': self.image_name,
+                 }]),
+            dict(method='HEAD',
+                 uri=self.get_mock_url(
+                     service_type='object-store',
+                     resource=self.container_name,
+                     append=[other_image]),
+                 headers={
+                     'X-Timestamp': '1429036140.50253',
+                     'X-Trans-Id': 'txbbb825960a3243b49a36f-005a0dadaedfw1',
+                     'Content-Length': '1290170880',
+                     'Last-Modified': 'Tue, 14 Apr 2015 18:29:01 GMT',
+                     'X-Object-Meta-X-Shade-Sha256': 'does not matter',
+                     'X-Object-Meta-X-Shade-Md5': 'does not matter',
+                     'Date': 'Thu, 16 Nov 2017 15:24:30 GMT',
+                     'Accept-Ranges': 'bytes',
+                     'Content-Type': 'application/octet-stream',
+                     'Etag': '249219347276c331b87bf1ac2152d9af',
+                 }),
+            dict(method='HEAD',
+                 uri=self.get_mock_url(
+                     service_type='object-store',
+                     resource=self.container_name,
+                     append=[self.image_name]),
+                 headers={
+                     'X-Timestamp': '1429036140.50253',
+                     'X-Trans-Id': 'txbbb825960a3243b49a36f-005a0dadaedfw1',
+                     'Content-Length': '1290170880',
+                     'Last-Modified': 'Tue, 14 Apr 2015 18:29:01 GMT',
+                     'X-Object-Meta-X-Shade-Sha256': fakes.NO_SHA256,
+                     'X-Object-Meta-X-Shade-Md5': fakes.NO_MD5,
+                     'Date': 'Thu, 16 Nov 2017 15:24:30 GMT',
+                     'Accept-Ranges': 'bytes',
+                     'Content-Type': 'application/octet-stream',
+                     shade.openstackcloud.OBJECT_AUTOCREATE_KEY: 'true',
+                     'Etag': fakes.NO_MD5}),
+            dict(method='DELETE',
+                 uri='{endpoint}/{container}/{object}'.format(
+                     endpoint=endpoint, container=self.container_name,
+                     object=self.image_name)),
+        ])
+
+        deleted = self.cloud.delete_autocreated_image_objects(
+            container=self.container_name)
+        self.assertTrue(deleted)
 
         self.assert_calls()