Add method to cleanup autocreated image objects
This shouldn't really be needed, as the objects should get cleaned up automatically. BUT - if things leak, this method can be used to delete any objects shade has uploaded on behalf of the user for deleting images. While in there, clean up test_image to use a few more good practices. Change-Id: Ifb697944856e1922517074d84a7c00a4af75b1e6
This commit is contained in:
Monty Taylor
parent
371e0b667e
commit
5996a0313b
@ -0,0 +1,5 @@
|
||||
---
|
||||
features:
|
||||
- Added new method, delete_autocreated_image_objects
|
||||
that can be used to delete any leaked objects shade
|
||||
may have created on behalf of the user.
|
||||
@ -50,6 +50,8 @@ from shade import _utils
|
||||
|
||||
OBJECT_MD5_KEY = 'x-object-meta-x-shade-md5'
|
||||
OBJECT_SHA256_KEY = 'x-object-meta-x-shade-sha256'
|
||||
OBJECT_AUTOCREATE_KEY = 'x-object-meta-x-shade-autocreated'
|
||||
OBJECT_AUTOCREATE_CONTAINER = 'images'
|
||||
IMAGE_MD5_KEY = 'owner_specified.shade.md5'
|
||||
IMAGE_SHA256_KEY = 'owner_specified.shade.sha256'
|
||||
IMAGE_OBJECT_KEY = 'owner_specified.shade.object'
|
||||
@ -4526,7 +4528,7 @@ class OpenStackCloud(
|
||||
return up_to_date
|
||||
|
||||
def create_image(
|
||||
self, name, filename=None, container='images',
|
||||
self, name, filename=None, container=OBJECT_AUTOCREATE_CONTAINER,
|
||||
md5=None, sha256=None,
|
||||
disk_format=None, container_format=None,
|
||||
disable_vendor_agent=True,
|
||||
@ -4823,6 +4825,7 @@ class OpenStackCloud(
|
||||
self.create_object(
|
||||
container, name, filename,
|
||||
md5=md5, sha256=sha256,
|
||||
metadata={OBJECT_AUTOCREATE_KEY: 'true'},
|
||||
**{'content-type': 'application/octet-stream'})
|
||||
if not current_image:
|
||||
current_image = self.get_image(name)
|
||||
@ -7561,11 +7564,13 @@ class OpenStackCloud(
|
||||
return self._object_store_client.get(
|
||||
container, params=dict(format='json'))
|
||||
|
||||
def delete_object(self, container, name):
|
||||
def delete_object(self, container, name, meta=None):
|
||||
"""Delete an object from a container.
|
||||
|
||||
:param string container: Name of the container holding the object.
|
||||
:param string name: Name of the object to delete.
|
||||
:param dict meta: Metadata for the object in question. (optional, will
|
||||
be fetched if not provided)
|
||||
|
||||
:returns: True if delete succeeded, False if the object was not found.
|
||||
|
||||
@ -7580,7 +7585,8 @@ class OpenStackCloud(
|
||||
# Errors:
|
||||
# We should ultimately do something with that
|
||||
try:
|
||||
meta = self.get_object_metadata(container, name)
|
||||
if not meta:
|
||||
meta = self.get_object_metadata(container, name)
|
||||
if not meta:
|
||||
return False
|
||||
params = {}
|
||||
@ -7594,6 +7600,28 @@ class OpenStackCloud(
|
||||
except OpenStackCloudHTTPError:
|
||||
return False
|
||||
|
||||
def delete_autocreated_image_objects(
|
||||
self, container=OBJECT_AUTOCREATE_CONTAINER):
|
||||
"""Delete all objects autocreated for image uploads.
|
||||
|
||||
This method should generally not be needed, as shade should clean up
|
||||
the objects it uses for object-based image creation. If something
|
||||
goes wrong and it is found that there are leaked objects, this method
|
||||
can be used to delete any objects that shade has created on the user's
|
||||
behalf in service of image uploads.
|
||||
"""
|
||||
# This method only makes sense on clouds that use tasks
|
||||
if not self.image_api_use_tasks:
|
||||
return False
|
||||
|
||||
deleted = False
|
||||
for obj in self.list_objects(container):
|
||||
meta = self.get_object_metadata(container, obj['name'])
|
||||
if meta.get(OBJECT_AUTOCREATE_KEY) == 'true':
|
||||
if self.delete_object(container, obj['name'], meta):
|
||||
deleted = True
|
||||
return deleted
|
||||
|
||||
def get_object_metadata(self, container, name):
|
||||
try:
|
||||
return self._object_store_client.head(
|
||||
|
||||
@ -408,6 +408,10 @@ class RequestsMockTestCase(BaseTestCase):
|
||||
return _RoleData(role_id, role_name, {'role': response},
|
||||
{'role': request})
|
||||
|
||||
def use_nothing(self):
|
||||
self.calls = []
|
||||
self._uri_registry.clear()
|
||||
|
||||
def use_keystone_v3(self, catalog='catalog-v3.json'):
|
||||
self.adapter = self.useFixture(rm_fixture.Fixture())
|
||||
self.calls = []
|
||||
|
||||
@ -24,6 +24,7 @@ import munch
|
||||
import six
|
||||
|
||||
import shade
|
||||
import shade.openstackcloud
|
||||
from shade import exc
|
||||
from shade import meta
|
||||
from shade.tests import fakes
|
||||
@ -44,6 +45,8 @@ class BaseTestImage(base.RequestsMockTestCase):
|
||||
self.fake_image_dict = fakes.make_fake_image(image_id=self.image_id)
|
||||
self.fake_search_return = {'images': [self.fake_image_dict]}
|
||||
self.output = uuid.uuid4().bytes
|
||||
self.image_name = self.getUniqueString('image')
|
||||
self.container_name = self.getUniqueString('container')
|
||||
|
||||
|
||||
class TestImage(BaseTestImage):
|
||||
@ -258,8 +261,6 @@ class TestImage(BaseTestImage):
|
||||
|
||||
def test_create_image_task(self):
|
||||
self.cloud.image_api_use_tasks = True
|
||||
image_name = 'name-99'
|
||||
container_name = 'image_upload_v2_test_container'
|
||||
endpoint = self.cloud._object_store_client.get_endpoint()
|
||||
|
||||
task_id = str(uuid.uuid4())
|
||||
@ -286,18 +287,18 @@ class TestImage(BaseTestImage):
|
||||
slo={'min_segment_size': 500})),
|
||||
dict(method='HEAD',
|
||||
uri='{endpoint}/{container}'.format(
|
||||
endpoint=endpoint, container=container_name),
|
||||
endpoint=endpoint, container=self.container_name),
|
||||
status_code=404),
|
||||
dict(method='PUT',
|
||||
uri='{endpoint}/{container}'.format(
|
||||
endpoint=endpoint, container=container_name),
|
||||
endpoint=endpoint, container=self.container_name),
|
||||
status_code=201,
|
||||
headers={'Date': 'Fri, 16 Dec 2016 18:21:20 GMT',
|
||||
'Content-Length': '0',
|
||||
'Content-Type': 'text/html; charset=UTF-8'}),
|
||||
dict(method='HEAD',
|
||||
uri='{endpoint}/{container}'.format(
|
||||
endpoint=endpoint, container=container_name),
|
||||
endpoint=endpoint, container=self.container_name),
|
||||
headers={'Content-Length': '0',
|
||||
'X-Container-Object-Count': '0',
|
||||
'Accept-Ranges': 'bytes',
|
||||
@ -309,13 +310,13 @@ class TestImage(BaseTestImage):
|
||||
'Content-Type': 'text/plain; charset=utf-8'}),
|
||||
dict(method='HEAD',
|
||||
uri='{endpoint}/{container}/{object}'.format(
|
||||
endpoint=endpoint, container=container_name,
|
||||
object=image_name),
|
||||
endpoint=endpoint, container=self.container_name,
|
||||
object=self.image_name),
|
||||
status_code=404),
|
||||
dict(method='PUT',
|
||||
uri='{endpoint}/{container}/{object}'.format(
|
||||
endpoint=endpoint, container=container_name,
|
||||
object=image_name),
|
||||
endpoint=endpoint, container=self.container_name,
|
||||
object=self.image_name),
|
||||
status_code=201,
|
||||
validate=dict(
|
||||
headers={'x-object-meta-x-shade-md5': fakes.NO_MD5,
|
||||
@ -329,8 +330,9 @@ class TestImage(BaseTestImage):
|
||||
json=dict(
|
||||
type='import', input={
|
||||
'import_from': '{container}/{object}'.format(
|
||||
container=container_name, object=image_name),
|
||||
'image_properties': {'name': image_name}}))
|
||||
container=self.container_name,
|
||||
object=self.image_name),
|
||||
'image_properties': {'name': self.image_name}}))
|
||||
),
|
||||
dict(method='GET',
|
||||
uri='https://image.example.com/v2/tasks/{id}'.format(
|
||||
@ -348,8 +350,8 @@ class TestImage(BaseTestImage):
|
||||
validate=dict(
|
||||
json=sorted([{u'op': u'add',
|
||||
u'value': '{container}/{object}'.format(
|
||||
container=container_name,
|
||||
object=image_name),
|
||||
container=self.container_name,
|
||||
object=self.image_name),
|
||||
u'path': u'/owner_specified.shade.object'},
|
||||
{u'op': u'add', u'value': fakes.NO_MD5,
|
||||
u'path': u'/owner_specified.shade.md5'},
|
||||
@ -362,8 +364,8 @@ class TestImage(BaseTestImage):
|
||||
),
|
||||
dict(method='HEAD',
|
||||
uri='{endpoint}/{container}/{object}'.format(
|
||||
endpoint=endpoint, container=container_name,
|
||||
object=image_name),
|
||||
endpoint=endpoint, container=self.container_name,
|
||||
object=self.image_name),
|
||||
headers={
|
||||
'X-Timestamp': '1429036140.50253',
|
||||
'X-Trans-Id': 'txbbb825960a3243b49a36f-005a0dadaedfw1',
|
||||
@ -377,15 +379,94 @@ class TestImage(BaseTestImage):
|
||||
'Etag': fakes.NO_MD5}),
|
||||
dict(method='DELETE',
|
||||
uri='{endpoint}/{container}/{object}'.format(
|
||||
endpoint=endpoint, container=container_name,
|
||||
object=image_name)),
|
||||
endpoint=endpoint, container=self.container_name,
|
||||
object=self.image_name)),
|
||||
dict(method='GET', uri='https://image.example.com/v2/images',
|
||||
json=self.fake_search_return)
|
||||
])
|
||||
|
||||
self.cloud.create_image(
|
||||
image_name, self.imagefile.name, wait=True, timeout=1,
|
||||
is_public=False, container=container_name)
|
||||
self.image_name, self.imagefile.name, wait=True, timeout=1,
|
||||
is_public=False, container=self.container_name)
|
||||
|
||||
self.assert_calls()
|
||||
|
||||
def test_delete_autocreated_no_tasks(self):
|
||||
self.use_nothing()
|
||||
self.cloud.image_api_use_tasks = False
|
||||
deleted = self.cloud.delete_autocreated_image_objects(
|
||||
container=self.container_name)
|
||||
self.assertFalse(deleted)
|
||||
self.assert_calls()
|
||||
|
||||
def test_delete_autocreated_image_objects(self):
|
||||
self.use_keystone_v3()
|
||||
self.cloud.image_api_use_tasks = True
|
||||
endpoint = self.cloud._object_store_client.get_endpoint()
|
||||
other_image = self.getUniqueString('no-delete')
|
||||
|
||||
self.register_uris([
|
||||
dict(method='GET',
|
||||
uri=self.get_mock_url(
|
||||
service_type='object-store',
|
||||
resource=self.container_name,
|
||||
qs_elements=['format=json']),
|
||||
json=[{
|
||||
'content_type': 'application/octet-stream',
|
||||
'bytes': 1437258240,
|
||||
'hash': '249219347276c331b87bf1ac2152d9af',
|
||||
'last_modified': '2015-02-16T17:50:05.289600',
|
||||
'name': other_image,
|
||||
}, {
|
||||
'content_type': 'application/octet-stream',
|
||||
'bytes': 1290170880,
|
||||
'hash': fakes.NO_MD5,
|
||||
'last_modified': '2015-04-14T18:29:00.502530',
|
||||
'name': self.image_name,
|
||||
}]),
|
||||
dict(method='HEAD',
|
||||
uri=self.get_mock_url(
|
||||
service_type='object-store',
|
||||
resource=self.container_name,
|
||||
append=[other_image]),
|
||||
headers={
|
||||
'X-Timestamp': '1429036140.50253',
|
||||
'X-Trans-Id': 'txbbb825960a3243b49a36f-005a0dadaedfw1',
|
||||
'Content-Length': '1290170880',
|
||||
'Last-Modified': 'Tue, 14 Apr 2015 18:29:01 GMT',
|
||||
'X-Object-Meta-X-Shade-Sha256': 'does not matter',
|
||||
'X-Object-Meta-X-Shade-Md5': 'does not matter',
|
||||
'Date': 'Thu, 16 Nov 2017 15:24:30 GMT',
|
||||
'Accept-Ranges': 'bytes',
|
||||
'Content-Type': 'application/octet-stream',
|
||||
'Etag': '249219347276c331b87bf1ac2152d9af',
|
||||
}),
|
||||
dict(method='HEAD',
|
||||
uri=self.get_mock_url(
|
||||
service_type='object-store',
|
||||
resource=self.container_name,
|
||||
append=[self.image_name]),
|
||||
headers={
|
||||
'X-Timestamp': '1429036140.50253',
|
||||
'X-Trans-Id': 'txbbb825960a3243b49a36f-005a0dadaedfw1',
|
||||
'Content-Length': '1290170880',
|
||||
'Last-Modified': 'Tue, 14 Apr 2015 18:29:01 GMT',
|
||||
'X-Object-Meta-X-Shade-Sha256': fakes.NO_SHA256,
|
||||
'X-Object-Meta-X-Shade-Md5': fakes.NO_MD5,
|
||||
'Date': 'Thu, 16 Nov 2017 15:24:30 GMT',
|
||||
'Accept-Ranges': 'bytes',
|
||||
'Content-Type': 'application/octet-stream',
|
||||
shade.openstackcloud.OBJECT_AUTOCREATE_KEY: 'true',
|
||||
'Etag': fakes.NO_MD5}),
|
||||
dict(method='DELETE',
|
||||
uri='{endpoint}/{container}/{object}'.format(
|
||||
endpoint=endpoint, container=self.container_name,
|
||||
object=self.image_name)),
|
||||
])
|
||||
|
||||
deleted = self.cloud.delete_autocreated_image_objects(
|
||||
container=self.container_name)
|
||||
self.assertTrue(deleted)
|
||||
|
||||
self.assert_calls()
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user