dd64a81e65
Co-Authored-By: Florent Vennetier <florent.vennetier@ovhcloud.com> Change-Id: I635bc91faa7709f9df9cdf3aec157a21c08923ca
270 lines
11 KiB
Python
270 lines
11 KiB
Python
# Copyright (c) 2015 OpenStack Foundation
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
# implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
import six
|
|
import unittest
|
|
import os
|
|
import test.functional as tf
|
|
from swift.common.middleware.s3api.etree import fromstring, tostring, \
|
|
Element, SubElement
|
|
|
|
from test.functional.s3api import S3ApiBase
|
|
from test.functional.s3api.s3_test_client import Connection
|
|
from test.functional.s3api.utils import get_error_code, calculate_md5
|
|
|
|
|
|
def setUpModule():
|
|
tf.setup_package()
|
|
|
|
|
|
def tearDownModule():
|
|
tf.teardown_package()
|
|
|
|
|
|
class TestS3ApiMultiDelete(S3ApiBase):
|
|
def _prepare_test_delete_multi_objects(self, bucket, objects):
|
|
self.conn.make_request('PUT', bucket)
|
|
for obj in objects:
|
|
self.conn.make_request('PUT', bucket, obj)
|
|
|
|
def _gen_multi_delete_xml(self, objects, quiet=None):
|
|
elem = Element('Delete')
|
|
if quiet:
|
|
SubElement(elem, 'Quiet').text = quiet
|
|
for key in objects:
|
|
obj = SubElement(elem, 'Object')
|
|
SubElement(obj, 'Key').text = key
|
|
|
|
return tostring(elem, use_s3ns=False)
|
|
|
|
def _gen_invalid_multi_delete_xml(self, hasObjectTag=False):
|
|
elem = Element('Delete')
|
|
if hasObjectTag:
|
|
obj = SubElement(elem, 'Object')
|
|
SubElement(obj, 'Key').text = ''
|
|
|
|
return tostring(elem, use_s3ns=False)
|
|
|
|
def _test_delete_multi_objects(self, with_non_ascii=False):
|
|
bucket = 'bucket'
|
|
if with_non_ascii:
|
|
put_objects = [u'\N{SNOWMAN}obj%s' % var for var in range(4)]
|
|
else:
|
|
put_objects = ['obj%s' % var for var in range(4)]
|
|
self._prepare_test_delete_multi_objects(bucket, put_objects)
|
|
query = 'delete'
|
|
|
|
# Delete an object via MultiDelete API
|
|
req_objects = put_objects[:1]
|
|
xml = self._gen_multi_delete_xml(req_objects)
|
|
content_md5 = calculate_md5(xml)
|
|
status, headers, body = \
|
|
self.conn.make_request('POST', bucket, body=xml,
|
|
headers={'Content-MD5': content_md5},
|
|
query=query)
|
|
self.assertEqual(status, 200)
|
|
self.assertCommonResponseHeaders(headers)
|
|
self.assertTrue(headers['content-type'] is not None)
|
|
self.assertEqual(headers['content-length'], str(len(body)))
|
|
elem = fromstring(body)
|
|
resp_objects = elem.findall('Deleted')
|
|
self.assertEqual(len(resp_objects), len(req_objects))
|
|
for o in resp_objects:
|
|
key = o.find('Key').text
|
|
if six.PY2:
|
|
key = key.decode('utf-8')
|
|
self.assertTrue(key in req_objects)
|
|
|
|
# Delete 2 objects via MultiDelete API
|
|
req_objects = put_objects[1:3]
|
|
xml = self._gen_multi_delete_xml(req_objects)
|
|
content_md5 = calculate_md5(xml)
|
|
status, headers, body = \
|
|
self.conn.make_request('POST', bucket, body=xml,
|
|
headers={'Content-MD5': content_md5},
|
|
query=query)
|
|
self.assertEqual(status, 200)
|
|
elem = fromstring(body, 'DeleteResult')
|
|
resp_objects = elem.findall('Deleted')
|
|
self.assertEqual(len(resp_objects), len(req_objects))
|
|
for o in resp_objects:
|
|
key = o.find('Key').text
|
|
if six.PY2:
|
|
key = key.decode('utf-8')
|
|
self.assertTrue(key in req_objects)
|
|
|
|
if with_non_ascii:
|
|
fake_objs = [u'\N{SNOWMAN}obj%s' % var for var in range(4, 6)]
|
|
else:
|
|
fake_objs = ['obj%s' % var for var in range(4, 6)]
|
|
# Delete 2 objects via MultiDelete API but one (obj4) doesn't exist.
|
|
req_objects = [put_objects[-1], fake_objs[0]]
|
|
xml = self._gen_multi_delete_xml(req_objects)
|
|
content_md5 = calculate_md5(xml)
|
|
status, headers, body = \
|
|
self.conn.make_request('POST', bucket, body=xml,
|
|
headers={'Content-MD5': content_md5},
|
|
query=query)
|
|
self.assertEqual(status, 200)
|
|
elem = fromstring(body, 'DeleteResult')
|
|
resp_objects = elem.findall('Deleted')
|
|
# S3 assumes a NoSuchKey object as deleted.
|
|
self.assertEqual(len(resp_objects), len(req_objects))
|
|
for o in resp_objects:
|
|
key = o.find('Key').text
|
|
if six.PY2:
|
|
key = key.decode('utf-8')
|
|
self.assertTrue(key in req_objects)
|
|
|
|
# Delete 2 objects via MultiDelete API but no objects exist
|
|
req_objects = fake_objs[:2]
|
|
xml = self._gen_multi_delete_xml(req_objects)
|
|
content_md5 = calculate_md5(xml)
|
|
status, headers, body = \
|
|
self.conn.make_request('POST', bucket, body=xml,
|
|
headers={'Content-MD5': content_md5},
|
|
query=query)
|
|
self.assertEqual(status, 200)
|
|
elem = fromstring(body, 'DeleteResult')
|
|
resp_objects = elem.findall('Deleted')
|
|
self.assertEqual(len(resp_objects), len(req_objects))
|
|
for o in resp_objects:
|
|
key = o.find('Key').text
|
|
if six.PY2:
|
|
key = key.decode('utf-8')
|
|
self.assertTrue(key in req_objects)
|
|
|
|
def test_delete_multi_objects(self):
|
|
self._test_delete_multi_objects()
|
|
|
|
def test_delete_multi_objects_with_non_ascii(self):
|
|
self._test_delete_multi_objects(with_non_ascii=True)
|
|
|
|
def test_delete_multi_objects_error(self):
|
|
bucket = 'bucket'
|
|
put_objects = ['obj']
|
|
self._prepare_test_delete_multi_objects(bucket, put_objects)
|
|
xml = self._gen_multi_delete_xml(put_objects)
|
|
content_md5 = calculate_md5(xml)
|
|
query = 'delete'
|
|
|
|
auth_error_conn = Connection(tf.config['s3_access_key'], 'invalid')
|
|
status, headers, body = \
|
|
auth_error_conn.make_request('POST', bucket, body=xml,
|
|
headers={
|
|
'Content-MD5': content_md5
|
|
},
|
|
query=query)
|
|
self.assertEqual(get_error_code(body), 'SignatureDoesNotMatch')
|
|
|
|
status, headers, body = \
|
|
self.conn.make_request('POST', 'nothing', body=xml,
|
|
headers={'Content-MD5': content_md5},
|
|
query=query)
|
|
self.assertEqual(get_error_code(body), 'NoSuchBucket')
|
|
|
|
# without Object tag
|
|
xml = self._gen_invalid_multi_delete_xml()
|
|
content_md5 = calculate_md5(xml)
|
|
status, headers, body = \
|
|
self.conn.make_request('POST', bucket, body=xml,
|
|
headers={'Content-MD5': content_md5},
|
|
query=query)
|
|
self.assertEqual(get_error_code(body), 'MalformedXML')
|
|
|
|
# without value of Key tag
|
|
xml = self._gen_invalid_multi_delete_xml(hasObjectTag=True)
|
|
content_md5 = calculate_md5(xml)
|
|
status, headers, body = \
|
|
self.conn.make_request('POST', bucket, body=xml,
|
|
headers={'Content-MD5': content_md5},
|
|
query=query)
|
|
self.assertEqual(get_error_code(body), 'UserKeyMustBeSpecified')
|
|
|
|
max_deletes = int(tf.cluster_info.get('s3api', {}).get(
|
|
'max_multi_delete_objects', 1000))
|
|
# specified number of objects are over max_multi_delete_objects
|
|
# (Default 1000), but xml size is relatively small
|
|
req_objects = ['obj%s' for var in range(max_deletes + 1)]
|
|
xml = self._gen_multi_delete_xml(req_objects)
|
|
content_md5 = calculate_md5(xml)
|
|
status, headers, body = \
|
|
self.conn.make_request('POST', bucket, body=xml,
|
|
headers={'Content-MD5': content_md5},
|
|
query=query)
|
|
self.assertEqual(get_error_code(body), 'MalformedXML')
|
|
|
|
# specified xml size is large, but number of objects are
|
|
# smaller than max_multi_delete_objects.
|
|
obj = 'a' * 102400
|
|
req_objects = [obj + str(var) for var in range(max_deletes - 1)]
|
|
xml = self._gen_multi_delete_xml(req_objects)
|
|
content_md5 = calculate_md5(xml)
|
|
status, headers, body = \
|
|
self.conn.make_request('POST', bucket, body=xml,
|
|
headers={'Content-MD5': content_md5},
|
|
query=query)
|
|
self.assertEqual(get_error_code(body), 'MalformedXML')
|
|
|
|
def test_delete_multi_objects_with_quiet(self):
|
|
bucket = 'bucket'
|
|
put_objects = ['obj']
|
|
query = 'delete'
|
|
|
|
# with Quiet true
|
|
quiet = 'true'
|
|
self._prepare_test_delete_multi_objects(bucket, put_objects)
|
|
xml = self._gen_multi_delete_xml(put_objects, quiet)
|
|
content_md5 = calculate_md5(xml)
|
|
status, headers, body = \
|
|
self.conn.make_request('POST', bucket, body=xml,
|
|
headers={'Content-MD5': content_md5},
|
|
query=query)
|
|
self.assertEqual(status, 200)
|
|
elem = fromstring(body, 'DeleteResult')
|
|
resp_objects = elem.findall('Deleted')
|
|
self.assertEqual(len(resp_objects), 0)
|
|
|
|
# with Quiet false
|
|
quiet = 'false'
|
|
self._prepare_test_delete_multi_objects(bucket, put_objects)
|
|
xml = self._gen_multi_delete_xml(put_objects, quiet)
|
|
content_md5 = calculate_md5(xml)
|
|
status, headers, body = \
|
|
self.conn.make_request('POST', bucket, body=xml,
|
|
headers={'Content-MD5': content_md5},
|
|
query=query)
|
|
self.assertEqual(status, 200)
|
|
elem = fromstring(body, 'DeleteResult')
|
|
resp_objects = elem.findall('Deleted')
|
|
self.assertEqual(len(resp_objects), 1)
|
|
|
|
|
|
class TestS3ApiMultiDeleteSigV4(TestS3ApiMultiDelete):
|
|
@classmethod
|
|
def setUpClass(cls):
|
|
os.environ['S3_USE_SIGV4'] = "True"
|
|
|
|
@classmethod
|
|
def tearDownClass(cls):
|
|
del os.environ['S3_USE_SIGV4']
|
|
|
|
def setUp(self):
|
|
super(TestS3ApiMultiDeleteSigV4, self).setUp()
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|