openstack/swift
Code Issues Proposed changes

Add per-container storage policy to account listing

Browse Source 
Add the storage_policy attribute to the metadata returned
when listing containers using the GET account API function.

The storage policy of a container is a very useful attribute
for telemetry and billing purposes, as it determines the location
and method/redundancy of on-disk storage for the objects in the
container. Ceilometer currently cannot define the storage policy as a
metadata attribute in Gnocchi as GET account, the most efficient way
of discovering all containers in an account, does not return the
storage policy for each container.

Returning the storage policy for each container in GET account
is the ideal way of resolving this issue, as it allows Ceilometer
to find all containers' storage policies without performing additional
costly API calls.

Special care has been taken to ensure the change is backwards
compatible when migrating from pre-storage policy versions
of Swift, even though those versions are quite old now.
This special handling can be removed if support for migrating
from older versions is discontinued.

Closes-bug: #2097074
Change-Id: I52b37cfa49cac8675f5087bcbcfe18db0b46d887
This commit is contained in:
Callum Dickinson 2025-01-31 19:09:05 +13:00
parent 0dfa38d025
commit 965cc2fcbc
10 changed files with 310 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
swift/account/backend.py
View File

@ -367,7 +367,7 @@ class AccountBroker(DatabaseBroker):
:param allow_reserved: exclude names with reserved-byte by default
:returns: list of tuples of (name, object_count, bytes_used,
put_timestamp, 0)
put_timestamp, storage_policy_index, 0)
"""
delim_force_gte = False
if reverse:
@ -383,7 +383,8 @@ class AccountBroker(DatabaseBroker):
results = []
while len(results) < limit:
query = """
SELECT name, object_count, bytes_used, put_timestamp, 0
SELECT name, object_count, bytes_used, put_timestamp,
{storage_policy_index}, 0
FROM container
WHERE """
query_args = []
@ -415,7 +416,27 @@ class AccountBroker(DatabaseBroker):
query += ' ORDER BY name %s LIMIT ?' % \
('DESC' if reverse else '')
query_args.append(limit - len(results))
curs = conn.execute(query, query_args)
try:
# First, try querying with the storage policy index.
curs = conn.execute(
query.format(
storage_policy_index="storage_policy_index"),
query_args)
except sqlite3.OperationalError as err:
# If the storage policy column is not available,
# the database has not been migrated to the new schema
# with storage_policy_index. Re-run the query with
# storage_policy_index set to 0, which is what
# would be set once the database is migrated.
# TODO(callumdickinson): If support for migrating
# pre-storage policy versions of Swift is dropped,
# then this special handling can be removed.
if "no such column: storage_policy_index" in str(err):
curs = conn.execute(
query.format(storage_policy_index="0"),
query_args)
else:
raise
curs.row_factory = None
# Delimiters without a prefix is ignored, further if there
@ -452,7 +473,7 @@ class AccountBroker(DatabaseBroker):
delim_force_gte = True
dir_name = name[:end + len(delimiter)]
if dir_name != orig_marker:
results.append([dir_name, 0, 0, '0', 1])
results.append([dir_name, 0, 0, '0', -1, 1])
curs.close()
break
results.append(row)

3
swift/account/reaper.py
View File

@ -265,7 +265,8 @@ class AccountReaper(Daemon):
container_limit, '', None, None, None, allow_reserved=True))
while containers:
try:
for (container, _junk, _junk, _junk, _junk) in containers:
for (container, _junk, _junk, _junk, _junk,
_junk) in containers:
this_shard = (
int(md5(container.encode('utf-8'),
usedforsecurity=False)

28
swift/account/utils.py
View File

@ -82,14 +82,34 @@ def account_listing_response(account, req, response_content_type, broker=None,
prefix, delimiter, reverse,
req.allow_reserved_names)
data = []
for (name, object_count, bytes_used, put_timestamp, is_subdir) \
for (name, object_count, bytes_used, put_timestamp,
storage_policy_index, is_subdir) \
in account_list:
if is_subdir:
data.append({'subdir': name})
else:
data.append(
{'name': name, 'count': object_count, 'bytes': bytes_used,
'last_modified': Timestamp(put_timestamp).isoformat})
container = {
'name': name,
'count': object_count,
'bytes': bytes_used,
'last_modified': Timestamp(put_timestamp).isoformat}
# Add the container's storage policy to the response, unless:
# * storage_policy_index < 0, which means that
# the storage policy could not be determined
# * storage_policy_index was not found in POLICIES,
# which means the storage policy is missing from
# the Swift configuration.
# The storage policy should always be returned when
# everything is configured correctly, but clients are
# expected to be able to handle this case regardless.
if (
storage_policy_index >= 0
and storage_policy_index in POLICIES
):
container['storage_policy'] = (
POLICIES[storage_policy_index].name
)
data.append(container)
if response_content_type.endswith('/xml'):
account_list = listing_formats.account_to_xml(data, account)
ret = HTTPOk(body=account_list, request=req, headers=resp_headers)

3
swift/common/middleware/listing_formats.py
View File

@ -84,6 +84,9 @@ def account_to_xml(listing, account_name):
sub = SubElement(doc, 'container')
for field in ('name', 'count', 'bytes', 'last_modified'):
SubElement(sub, field).text = str(record.pop(field))
for field in ('storage_policy',):
if field in record:
SubElement(sub, field).text = str(record.pop(field))
sub.tail = '\n'
return to_xml(doc)

3
test/functional/swift_test_client.py
View File

@ -570,7 +570,8 @@ class Account(Base):
tree = minidom.parseString(self.conn.response.read())
for x in tree.getElementsByTagName('container'):
cont = {}
for key in ['name', 'count', 'bytes', 'last_modified']:
for key in ['name', 'count', 'bytes', 'last_modified',
'storage_policy']:
cont[key] = x.getElementsByTagName(key)[0].\
childNodes[0].nodeValue
conts.append(cont)

4
test/unit/account/test_backend.py
View File

@ -1448,7 +1448,7 @@ class TestAccountBrokerBeforeSPI(TestAccountBroker):
# make sure we can iter containers without the migration
for c in broker.list_containers_iter(1, None, None, None, None):
self.assertEqual(c, ('test_name', 1, 2, timestamp, 0))
self.assertEqual(c, ('test_name', 1, 2, timestamp, 0, 0))
# stats table is mysteriously empty...
stats = broker.get_policy_stats()
@ -1607,7 +1607,7 @@ class TestAccountBrokerBeforeSPI(TestAccountBroker):
# make sure "test_name" container in new database
self.assertEqual(new_broker.get_info()['container_count'], 1)
for c in new_broker.list_containers_iter(1, None, None, None, None):
self.assertEqual(c, ('test_name', 1, 2, timestamp, 0))
self.assertEqual(c, ('test_name', 1, 2, timestamp, 0, 0))
# full migration successful
with new_broker.get() as conn:

4
test/unit/account/test_reaper.py
View File

@ -59,7 +59,7 @@ class FakeAccountBroker(object):
kwargs, ))
for cont in self.containers:
if cont > marker:
yield cont, None, None, None, None
yield cont, None, None, None, None, None
limit -= 1
if limit <= 0:
break
@ -735,7 +735,7 @@ class TestReaper(unittest.TestCase):
if container in self.containers_yielded:
continue
yield container, None, None, None, None
yield container, None, None, None, None, None
self.containers_yielded.append(container)
def fake_reap_container(self, account, account_partition,

134
test/unit/account/test_server.py
View File

@ -1089,6 +1089,8 @@ class TestAccountController(unittest.TestCase):
self.assertEqual(resp.content_type, 'text/plain')
self.assertEqual(resp.charset, 'utf-8')
@patch_policies([StoragePolicy(0, 'zero', is_default=True),
StoragePolicy(1, 'one', is_default=False)])
def test_GET_with_containers_json(self):
put_timestamps = {}
req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'PUT',
@ -1108,7 +1110,8 @@ class TestAccountController(unittest.TestCase):
'X-Delete-Timestamp': '0',
'X-Object-Count': '0',
'X-Bytes-Used': '0',
'X-Timestamp': normalize_timestamp(0)})
'X-Timestamp': normalize_timestamp(0),
'X-Backend-Storage-Policy-Index': 1})
req.get_response(self.controller)
req = Request.blank('/sda1/p/a?format=json',
environ={'REQUEST_METHOD': 'GET'})
@ -1117,9 +1120,11 @@ class TestAccountController(unittest.TestCase):
self.assertEqual(
json.loads(resp.body),
[{'count': 0, 'bytes': 0, 'name': 'c1',
'last_modified': Timestamp(put_timestamps['c1']).isoformat},
'last_modified': Timestamp(put_timestamps['c1']).isoformat,
'storage_policy': POLICIES[0].name},
{'count': 0, 'bytes': 0, 'name': 'c2',
'last_modified': Timestamp(put_timestamps['c2']).isoformat}])
'last_modified': Timestamp(put_timestamps['c2']).isoformat,
'storage_policy': POLICIES[1].name}])
put_timestamps['c1'] = normalize_timestamp(3)
req = Request.blank('/sda1/p/a/c1', environ={'REQUEST_METHOD': 'PUT'},
headers={'X-Put-Timestamp': put_timestamps['c1'],
@ -1134,7 +1139,8 @@ class TestAccountController(unittest.TestCase):
'X-Delete-Timestamp': '0',
'X-Object-Count': '3',
'X-Bytes-Used': '4',
'X-Timestamp': normalize_timestamp(0)})
'X-Timestamp': normalize_timestamp(0),
'X-Backend-Storage-Policy-Index': 1})
req.get_response(self.controller)
req = Request.blank('/sda1/p/a?format=json',
environ={'REQUEST_METHOD': 'GET'})
@ -1143,12 +1149,16 @@ class TestAccountController(unittest.TestCase):
self.assertEqual(
json.loads(resp.body),
[{'count': 1, 'bytes': 2, 'name': 'c1',
'last_modified': Timestamp(put_timestamps['c1']).isoformat},
'last_modified': Timestamp(put_timestamps['c1']).isoformat,
'storage_policy': POLICIES[0].name},
{'count': 3, 'bytes': 4, 'name': 'c2',
'last_modified': Timestamp(put_timestamps['c2']).isoformat}])
'last_modified': Timestamp(put_timestamps['c2']).isoformat,
'storage_policy': POLICIES[1].name}])
self.assertEqual(resp.content_type, 'application/json')
self.assertEqual(resp.charset, 'utf-8')
@patch_policies([StoragePolicy(0, 'zero', is_default=True),
StoragePolicy(1, 'one', is_default=False)])
def test_GET_with_containers_xml(self):
put_timestamps = {}
req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'PUT',
@ -1168,7 +1178,8 @@ class TestAccountController(unittest.TestCase):
'X-Delete-Timestamp': '0',
'X-Object-Count': '0',
'X-Bytes-Used': '0',
'X-Timestamp': normalize_timestamp(0)})
'X-Timestamp': normalize_timestamp(0),
'X-Backend-Storage-Policy-Index': 1})
req.get_response(self.controller)
req = Request.blank('/sda1/p/a?format=xml',
environ={'REQUEST_METHOD': 'GET'})
@ -1183,7 +1194,8 @@ class TestAccountController(unittest.TestCase):
self.assertEqual(listing[0].nodeName, 'container')
container = [n for n in listing[0].childNodes if n.nodeName != '#text']
self.assertEqual(sorted([n.nodeName for n in container]),
['bytes', 'count', 'last_modified', 'name'])
['bytes', 'count', 'last_modified', 'name',
'storage_policy'])
node = [n for n in container if n.nodeName == 'name'][0]
self.assertEqual(node.firstChild.nodeValue, 'c1')
node = [n for n in container if n.nodeName == 'count'][0]
@ -1193,11 +1205,14 @@ class TestAccountController(unittest.TestCase):
node = [n for n in container if n.nodeName == 'last_modified'][0]
self.assertEqual(node.firstChild.nodeValue,
Timestamp(put_timestamps['c1']).isoformat)
node = [n for n in container if n.nodeName == 'storage_policy'][0]
self.assertEqual(node.firstChild.nodeValue, POLICIES[0].name)
self.assertEqual(listing[-1].nodeName, 'container')
container = \
[n for n in listing[-1].childNodes if n.nodeName != '#text']
self.assertEqual(sorted([n.nodeName for n in container]),
['bytes', 'count', 'last_modified', 'name'])
['bytes', 'count', 'last_modified', 'name',
'storage_policy'])
node = [n for n in container if n.nodeName == 'name'][0]
self.assertEqual(node.firstChild.nodeValue, 'c2')
node = [n for n in container if n.nodeName == 'count'][0]
@ -1207,6 +1222,8 @@ class TestAccountController(unittest.TestCase):
node = [n for n in container if n.nodeName == 'last_modified'][0]
self.assertEqual(node.firstChild.nodeValue,
Timestamp(put_timestamps['c2']).isoformat)
node = [n for n in container if n.nodeName == 'storage_policy'][0]
self.assertEqual(node.firstChild.nodeValue, POLICIES[1].name)
req = Request.blank('/sda1/p/a/c1', environ={'REQUEST_METHOD': 'PUT'},
headers={'X-Put-Timestamp': '1',
'X-Delete-Timestamp': '0',
@ -1219,7 +1236,8 @@ class TestAccountController(unittest.TestCase):
'X-Delete-Timestamp': '0',
'X-Object-Count': '3',
'X-Bytes-Used': '4',
'X-Timestamp': normalize_timestamp(0)})
'X-Timestamp': normalize_timestamp(0),
'X-Backend-Storage-Policy-Index': 1})
req.get_response(self.controller)
req = Request.blank('/sda1/p/a?format=xml',
environ={'REQUEST_METHOD': 'GET'})
@ -1233,7 +1251,8 @@ class TestAccountController(unittest.TestCase):
self.assertEqual(listing[0].nodeName, 'container')
container = [n for n in listing[0].childNodes if n.nodeName != '#text']
self.assertEqual(sorted([n.nodeName for n in container]),
['bytes', 'count', 'last_modified', 'name'])
['bytes', 'count', 'last_modified', 'name',
'storage_policy'])
node = [n for n in container if n.nodeName == 'name'][0]
self.assertEqual(node.firstChild.nodeValue, 'c1')
node = [n for n in container if n.nodeName == 'count'][0]
@ -1243,11 +1262,14 @@ class TestAccountController(unittest.TestCase):
node = [n for n in container if n.nodeName == 'last_modified'][0]
self.assertEqual(node.firstChild.nodeValue,
Timestamp(put_timestamps['c1']).isoformat)
node = [n for n in container if n.nodeName == 'storage_policy'][0]
self.assertEqual(node.firstChild.nodeValue, POLICIES[0].name)
self.assertEqual(listing[-1].nodeName, 'container')
container = [
n for n in listing[-1].childNodes if n.nodeName != '#text']
self.assertEqual(sorted([n.nodeName for n in container]),
['bytes', 'count', 'last_modified', 'name'])
['bytes', 'count', 'last_modified', 'name',
'storage_policy'])
node = [n for n in container if n.nodeName == 'name'][0]
self.assertEqual(node.firstChild.nodeValue, 'c2')
node = [n for n in container if n.nodeName == 'count'][0]
@ -1257,6 +1279,8 @@ class TestAccountController(unittest.TestCase):
node = [n for n in container if n.nodeName == 'last_modified'][0]
self.assertEqual(node.firstChild.nodeValue,
Timestamp(put_timestamps['c2']).isoformat)
node = [n for n in container if n.nodeName == 'storage_policy'][0]
self.assertEqual(node.firstChild.nodeValue, POLICIES[1].name)
self.assertEqual(resp.charset, 'utf-8')
def test_GET_xml_escapes_account_name(self):
@ -1347,6 +1371,8 @@ class TestAccountController(unittest.TestCase):
self.assertEqual(resp.body.strip().split(b'\n'),
[b'c3', b'c4'])
@patch_policies([StoragePolicy(0, 'zero', is_default=True),
StoragePolicy(1, 'one', is_default=False)])
def test_GET_limit_marker_json(self):
req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'PUT',
'HTTP_X_TIMESTAMP': '0'})
@ -1360,29 +1386,37 @@ class TestAccountController(unittest.TestCase):
'X-Delete-Timestamp': '0',
'X-Object-Count': '2',
'X-Bytes-Used': '3',
'X-Timestamp': put_timestamp})
'X-Timestamp': put_timestamp,
'X-Backend-Storage-Policy-Index': c % 2})
req.get_response(self.controller)
req = Request.blank('/sda1/p/a?limit=3&format=json',
environ={'REQUEST_METHOD': 'GET'})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 200)
expected = [{'count': 2, 'bytes': 3, 'name': 'c0',
'last_modified': Timestamp('1').isoformat},
'last_modified': Timestamp('1').isoformat,
'storage_policy': POLICIES[0].name},
{'count': 2, 'bytes': 3, 'name': 'c1',
'last_modified': Timestamp('2').isoformat},
'last_modified': Timestamp('2').isoformat,
'storage_policy': POLICIES[1].name},
{'count': 2, 'bytes': 3, 'name': 'c2',
'last_modified': Timestamp('3').isoformat}]
'last_modified': Timestamp('3').isoformat,
'storage_policy': POLICIES[0].name}]
self.assertEqual(json.loads(resp.body), expected)
req = Request.blank('/sda1/p/a?limit=3&marker=c2&format=json',
environ={'REQUEST_METHOD': 'GET'})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 200)
expected = [{'count': 2, 'bytes': 3, 'name': 'c3',
'last_modified': Timestamp('4').isoformat},
'last_modified': Timestamp('4').isoformat,
'storage_policy': POLICIES[1].name},
{'count': 2, 'bytes': 3, 'name': 'c4',
'last_modified': Timestamp('5').isoformat}]
'last_modified': Timestamp('5').isoformat,
'storage_policy': POLICIES[0].name}]
self.assertEqual(json.loads(resp.body), expected)
@patch_policies([StoragePolicy(0, 'zero', is_default=True),
StoragePolicy(1, 'one', is_default=False)])
def test_GET_limit_marker_xml(self):
req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'PUT',
'HTTP_X_TIMESTAMP': '0'})
@ -1396,7 +1430,8 @@ class TestAccountController(unittest.TestCase):
'X-Delete-Timestamp': '0',
'X-Object-Count': '2',
'X-Bytes-Used': '3',
'X-Timestamp': put_timestamp})
'X-Timestamp': put_timestamp,
'X-Backend-Storage-Policy-Index': c % 2})
req.get_response(self.controller)
req = Request.blank('/sda1/p/a?limit=3&format=xml',
environ={'REQUEST_METHOD': 'GET'})
@ -1410,7 +1445,8 @@ class TestAccountController(unittest.TestCase):
self.assertEqual(listing[0].nodeName, 'container')
container = [n for n in listing[0].childNodes if n.nodeName != '#text']
self.assertEqual(sorted([n.nodeName for n in container]),
['bytes', 'count', 'last_modified', 'name'])
['bytes', 'count', 'last_modified', 'name',
'storage_policy'])
node = [n for n in container if n.nodeName == 'name'][0]
self.assertEqual(node.firstChild.nodeValue, 'c0')
node = [n for n in container if n.nodeName == 'count'][0]
@ -1420,11 +1456,14 @@ class TestAccountController(unittest.TestCase):
node = [n for n in container if n.nodeName == 'last_modified'][0]
self.assertEqual(node.firstChild.nodeValue,
Timestamp('1').isoformat)
node = [n for n in container if n.nodeName == 'storage_policy'][0]
self.assertEqual(node.firstChild.nodeValue, POLICIES[0].name)
self.assertEqual(listing[-1].nodeName, 'container')
container = [
n for n in listing[-1].childNodes if n.nodeName != '#text']
self.assertEqual(sorted([n.nodeName for n in container]),
['bytes', 'count', 'last_modified', 'name'])
['bytes', 'count', 'last_modified', 'name',
'storage_policy'])
node = [n for n in container if n.nodeName == 'name'][0]
self.assertEqual(node.firstChild.nodeValue, 'c2')
node = [n for n in container if n.nodeName == 'count'][0]
@ -1434,6 +1473,8 @@ class TestAccountController(unittest.TestCase):
node = [n for n in container if n.nodeName == 'last_modified'][0]
self.assertEqual(node.firstChild.nodeValue,
Timestamp('3').isoformat)
node = [n for n in container if n.nodeName == 'storage_policy'][0]
self.assertEqual(node.firstChild.nodeValue, POLICIES[0].name)
req = Request.blank('/sda1/p/a?limit=3&marker=c2&format=xml',
environ={'REQUEST_METHOD': 'GET'})
resp = req.get_response(self.controller)
@ -1446,7 +1487,8 @@ class TestAccountController(unittest.TestCase):
self.assertEqual(listing[0].nodeName, 'container')
container = [n for n in listing[0].childNodes if n.nodeName != '#text']
self.assertEqual(sorted([n.nodeName for n in container]),
['bytes', 'count', 'last_modified', 'name'])
['bytes', 'count', 'last_modified', 'name',
'storage_policy'])
node = [n for n in container if n.nodeName == 'name'][0]
self.assertEqual(node.firstChild.nodeValue, 'c3')
node = [n for n in container if n.nodeName == 'count'][0]
@ -1456,11 +1498,14 @@ class TestAccountController(unittest.TestCase):
node = [n for n in container if n.nodeName == 'last_modified'][0]
self.assertEqual(node.firstChild.nodeValue,
Timestamp('4').isoformat)
node = [n for n in container if n.nodeName == 'storage_policy'][0]
self.assertEqual(node.firstChild.nodeValue, POLICIES[1].name)
self.assertEqual(listing[-1].nodeName, 'container')
container = [
n for n in listing[-1].childNodes if n.nodeName != '#text']
self.assertEqual(sorted([n.nodeName for n in container]),
['bytes', 'count', 'last_modified', 'name'])
['bytes', 'count', 'last_modified', 'name',
'storage_policy'])
node = [n for n in container if n.nodeName == 'name'][0]
self.assertEqual(node.firstChild.nodeValue, 'c4')
node = [n for n in container if n.nodeName == 'count'][0]
@ -1470,6 +1515,8 @@ class TestAccountController(unittest.TestCase):
node = [n for n in container if n.nodeName == 'last_modified'][0]
self.assertEqual(node.firstChild.nodeValue,
Timestamp('5').isoformat)
node = [n for n in container if n.nodeName == 'storage_policy'][0]
self.assertEqual(node.firstChild.nodeValue, POLICIES[0].name)
def test_GET_accept_wildcard(self):
req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'PUT',
@ -1911,13 +1958,18 @@ class TestAccountController(unittest.TestCase):
self.assertEqual(resp.status_int // 100, 2, resp.body)
for container in containers:
path = '/sda1/p/%s/%s' % (account, container['name'])
req = Request.blank(path, method='PUT', headers={
headers = {
'X-Put-Timestamp': container['timestamp'].internal,
'X-Delete-Timestamp': container.get(
'deleted', Timestamp(0)).internal,
'X-Object-Count': container['count'],
'X-Bytes-Used': container['bytes'],
})
}
if 'storage_policy' in container:
headers['X-Backend-Storage-Policy-Index'] = (
POLICIES.get_by_name(container['storage_policy']).idx
)
req = Request.blank(path, method='PUT', headers=headers)
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int // 100, 2, resp.body)
@ -1927,6 +1979,7 @@ class TestAccountController(unittest.TestCase):
'bytes': 200,
'count': 2,
'timestamp': next(self.ts),
'storage_policy': POLICIES[0].name,
}]
self._report_containers(containers)
@ -1960,17 +2013,21 @@ class TestAccountController(unittest.TestCase):
self.assertEqual(json.loads(resp.body), [{
'subdir': '%s' % get_reserved_name('null')}])
@patch_policies([StoragePolicy(0, 'zero', is_default=True),
StoragePolicy(1, 'one', is_default=False)])
def test_delimiter_with_reserved_and_public(self):
containers = [{
'name': get_reserved_name('null', 'test01'),
'bytes': 200,
'count': 2,
'timestamp': next(self.ts),
'storage_policy': POLICIES[0].name,
}, {
'name': 'nullish',
'bytes': 10,
'count': 10,
'timestamp': next(self.ts),
'storage_policy': POLICIES[1].name,
}]
self._report_containers(containers)
@ -2020,17 +2077,21 @@ class TestAccountController(unittest.TestCase):
[{'subdir': '\x00'}] +
self._expected_listing(containers)[1:])
@patch_policies([StoragePolicy(0, 'zero', is_default=True),
StoragePolicy(1, 'one', is_default=False)])
def test_markers_with_reserved(self):
containers = [{
'name': get_reserved_name('null', 'test01'),
'bytes': 200,
'count': 2,
'timestamp': next(self.ts),
'storage_policy': POLICIES[0].name,
}, {
'name': get_reserved_name('null', 'test02'),
'bytes': 10,
'count': 10,
'timestamp': next(self.ts),
'storage_policy': POLICIES[1].name,
}]
self._report_containers(containers)
@ -2064,6 +2125,7 @@ class TestAccountController(unittest.TestCase):
'bytes': 300,
'count': 30,
'timestamp': next(self.ts),
'storage_policy': POLICIES[0].name,
})
self._report_containers(containers)
@ -2085,27 +2147,33 @@ class TestAccountController(unittest.TestCase):
self.assertEqual(json.loads(resp.body),
self._expected_listing(containers)[-1:])
@patch_policies([StoragePolicy(0, 'zero', is_default=True),
StoragePolicy(1, 'one', is_default=False)])
def test_prefix_with_reserved(self):
containers = [{
'name': get_reserved_name('null', 'test01'),
'bytes': 200,
'count': 2,
'timestamp': next(self.ts),
'storage_policy': POLICIES[0].name,
}, {
'name': get_reserved_name('null', 'test02'),
'bytes': 10,
'count': 10,
'timestamp': next(self.ts),
'storage_policy': POLICIES[1].name,
}, {
'name': get_reserved_name('null', 'foo'),
'bytes': 10,
'count': 10,
'timestamp': next(self.ts),
'storage_policy': POLICIES[0].name,
}, {
'name': get_reserved_name('nullish'),
'bytes': 300,
'count': 32,
'timestamp': next(self.ts),
'storage_policy': POLICIES[1].name,
}]
self._report_containers(containers)
@ -2125,27 +2193,33 @@ class TestAccountController(unittest.TestCase):
self.assertEqual(json.loads(resp.body),
self._expected_listing(containers[:2]))
@patch_policies([StoragePolicy(0, 'zero', is_default=True),
StoragePolicy(1, 'one', is_default=False)])
def test_prefix_and_delim_with_reserved(self):
containers = [{
'name': get_reserved_name('null', 'test01'),
'bytes': 200,
'count': 2,
'timestamp': next(self.ts),
'storage_policy': POLICIES[0].name,
}, {
'name': get_reserved_name('null', 'test02'),
'bytes': 10,
'count': 10,
'timestamp': next(self.ts),
'storage_policy': POLICIES[1].name,
}, {
'name': get_reserved_name('null', 'foo'),
'bytes': 10,
'count': 10,
'timestamp': next(self.ts),
'storage_policy': POLICIES[0].name,
}, {
'name': get_reserved_name('nullish'),
'bytes': 300,
'count': 32,
'timestamp': next(self.ts),
'storage_policy': POLICIES[1].name,
}]
self._report_containers(containers)
@ -2166,22 +2240,27 @@ class TestAccountController(unittest.TestCase):
self._expected_listing(containers[-1:])
self.assertEqual(json.loads(resp.body), expected)
@patch_policies([StoragePolicy(0, 'zero', is_default=True),
StoragePolicy(1, 'one', is_default=False)])
def test_reserved_markers_with_non_reserved(self):
containers = [{
'name': get_reserved_name('null', 'test01'),
'bytes': 200,
'count': 2,
'timestamp': next(self.ts),
'storage_policy': POLICIES[0].name,
}, {
'name': get_reserved_name('null', 'test02'),
'bytes': 10,
'count': 10,
'timestamp': next(self.ts),
'storage_policy': POLICIES[1].name,
}, {
'name': 'nullish',
'bytes': 300,
'count': 32,
'timestamp': next(self.ts),
'storage_policy': POLICIES[0].name,
}]
self._report_containers(containers)
@ -2221,22 +2300,27 @@ class TestAccountController(unittest.TestCase):
self.assertEqual(json.loads(resp.body),
self._expected_listing(containers)[1:])
@patch_policies([StoragePolicy(0, 'zero', is_default=True),
StoragePolicy(1, 'one', is_default=False)])
def test_null_markers(self):
containers = [{
'name': get_reserved_name('null', ''),
'bytes': 200,
'count': 2,
'timestamp': next(self.ts),
'storage_policy': POLICIES[0].name,
}, {
'name': get_reserved_name('null', 'test01'),
'bytes': 200,
'count': 2,
'timestamp': next(self.ts),
'storage_policy': POLICIES[1].name,
}, {
'name': 'null',
'bytes': 300,
'count': 32,
'timestamp': next(self.ts),
'storage_policy': POLICIES[0].name,
}]
self._report_containers(containers)

82
test/unit/account/test_utils.py
View File

@ -214,7 +214,58 @@ class TestAccountUtils(TestDbBase):
self.assertEqual(expected, resp.headers)
self.assertEqual(b'', resp.body)
@patch_policies([StoragePolicy(0, 'zero', is_default=True)])
@patch_policies([StoragePolicy(0, 'zero', is_default=True),
StoragePolicy(1, 'one', is_default=False)])
def test_account_listing_with_containers(self):
broker = backend.AccountBroker(self.db_path, account='a')
put_timestamp = next(self.ts)
now = time.time()
with mock.patch('time.time', new=lambda: now):
broker.initialize(put_timestamp.internal)
container_timestamp = next(self.ts)
broker.put_container('foo',
container_timestamp.internal, 0, 10, 100, 0)
broker.put_container('bar',
container_timestamp.internal, 0, 10, 100, 1)
req = Request.blank('')
resp = utils.account_listing_response(
'a', req, 'application/json', broker)
self.assertEqual(resp.status_int, 200)
expected = HeaderKeyDict({
'Content-Type': 'application/json; charset=utf-8',
'Content-Length': 233,
'X-Account-Container-Count': 2,
'X-Account-Object-Count': 20,
'X-Account-Bytes-Used': 200,
'X-Timestamp': Timestamp(now).normal,
'X-PUT-Timestamp': put_timestamp.normal,
'X-Account-Storage-Policy-Zero-Container-Count': 1,
'X-Account-Storage-Policy-Zero-Object-Count': 10,
'X-Account-Storage-Policy-Zero-Bytes-Used': 100,
'X-Account-Storage-Policy-One-Container-Count': 1,
'X-Account-Storage-Policy-One-Object-Count': 10,
'X-Account-Storage-Policy-One-Bytes-Used': 100,
})
self.assertEqual(expected, resp.headers)
expected = [{
"last_modified": container_timestamp.isoformat,
"count": 10,
"bytes": 100,
"name": 'foo',
'storage_policy': POLICIES[0].name,
}, {
"last_modified": container_timestamp.isoformat,
"count": 10,
"bytes": 100,
"name": 'bar',
'storage_policy': POLICIES[1].name,
}]
self.assertEqual(sorted(json.dumps(expected).encode('ascii')),
sorted(resp.body))
@patch_policies([StoragePolicy(0, 'zero', is_default=True),
StoragePolicy(1, 'one', is_default=False)])
def test_account_listing_reserved_names(self):
broker = backend.AccountBroker(self.db_path, account='a')
put_timestamp = next(self.ts)
@ -224,6 +275,8 @@ class TestAccountUtils(TestDbBase):
container_timestamp = next(self.ts)
broker.put_container(get_reserved_name('foo'),
container_timestamp.internal, 0, 10, 100, 0)
broker.put_container(get_reserved_name('bar'),
container_timestamp.internal, 0, 10, 100, 1)
req = Request.blank('')
resp = utils.account_listing_response(
@ -232,14 +285,17 @@ class TestAccountUtils(TestDbBase):
expected = HeaderKeyDict({
'Content-Type': 'application/json; charset=utf-8',
'Content-Length': 2,
'X-Account-Container-Count': 1,
'X-Account-Object-Count': 10,
'X-Account-Bytes-Used': 100,
'X-Account-Container-Count': 2,
'X-Account-Object-Count': 20,
'X-Account-Bytes-Used': 200,
'X-Timestamp': Timestamp(now).normal,
'X-PUT-Timestamp': put_timestamp.normal,
'X-Account-Storage-Policy-Zero-Container-Count': 1,
'X-Account-Storage-Policy-Zero-Object-Count': 10,
'X-Account-Storage-Policy-Zero-Bytes-Used': 100,
'X-Account-Storage-Policy-One-Container-Count': 1,
'X-Account-Storage-Policy-One-Object-Count': 10,
'X-Account-Storage-Policy-One-Bytes-Used': 100,
})
self.assertEqual(expected, resp.headers)
self.assertEqual(b'[]', resp.body)
@ -251,15 +307,18 @@ class TestAccountUtils(TestDbBase):
self.assertEqual(resp.status_int, 200)
expected = HeaderKeyDict({
'Content-Type': 'application/json; charset=utf-8',
'Content-Length': 97,
'X-Account-Container-Count': 1,
'X-Account-Object-Count': 10,
'X-Account-Bytes-Used': 100,
'Content-Length': 245,
'X-Account-Container-Count': 2,
'X-Account-Object-Count': 20,
'X-Account-Bytes-Used': 200,
'X-Timestamp': Timestamp(now).normal,
'X-PUT-Timestamp': put_timestamp.normal,
'X-Account-Storage-Policy-Zero-Container-Count': 1,
'X-Account-Storage-Policy-Zero-Object-Count': 10,
'X-Account-Storage-Policy-Zero-Bytes-Used': 100,
'X-Account-Storage-Policy-One-Container-Count': 1,
'X-Account-Storage-Policy-One-Object-Count': 10,
'X-Account-Storage-Policy-One-Bytes-Used': 100,
})
self.assertEqual(expected, resp.headers)
expected = [{
@ -267,6 +326,13 @@ class TestAccountUtils(TestDbBase):
"count": 10,
"bytes": 100,
"name": get_reserved_name('foo'),
'storage_policy': POLICIES[0].name,
}, {
"last_modified": container_timestamp.isoformat,
"count": 10,
"bytes": 100,
"name": get_reserved_name('bar'),
'storage_policy': POLICIES[1].name,
}]
self.assertEqual(sorted(json.dumps(expected).encode('ascii')),
sorted(resp.body))

82
test/unit/common/middleware/test_listing_formats.py
View File

@ -20,10 +20,14 @@ from swift.common.header_key_dict import HeaderKeyDict
from swift.common.swob import Request, HTTPOk, HTTPNoContent
from swift.common.middleware import listing_formats
from swift.common.request_helpers import get_reserved_name
from swift.common.storage_policy import POLICIES
from test.debug_logger import debug_logger
from test.unit.common.middleware.helpers import FakeSwift
TEST_POLICIES = (POLICIES[0].name, 'Policy-1')
class TestListingFormats(unittest.TestCase):
def setUp(self):
self.fake_swift = FakeSwift()
@ -32,8 +36,14 @@ class TestListingFormats(unittest.TestCase):
logger=self.logger)
self.fake_account_listing = json.dumps([
{'name': 'bar', 'bytes': 0, 'count': 0,
'last_modified': '1970-01-01T00:00:00.000000'},
'last_modified': '1970-01-01T00:00:00.000000',
'storage_policy': TEST_POLICIES[0]},
{'subdir': 'foo_'},
{'name': 'foobar', 'bytes': 0, 'count': 0,
'last_modified': '2025-01-01T00:00:00.000000',
'storage_policy': TEST_POLICIES[1]},
{'name': 'nobar', 'bytes': 0, 'count': 0, # Unknown policy
'last_modified': '2025-02-01T00:00:00.000000'},
]).encode('ascii')
self.fake_container_listing = json.dumps([
{'name': 'bar', 'hash': 'etag', 'bytes': 0,
@ -44,11 +54,18 @@ class TestListingFormats(unittest.TestCase):
self.fake_account_listing_with_reserved = json.dumps([
{'name': 'bar', 'bytes': 0, 'count': 0,
'last_modified': '1970-01-01T00:00:00.000000'},
'last_modified': '1970-01-01T00:00:00.000000',
'storage_policy': TEST_POLICIES[0]},
{'name': get_reserved_name('bar', 'versions'), 'bytes': 0,
'count': 0, 'last_modified': '1970-01-01T00:00:00.000000'},
'count': 0, 'last_modified': '1970-01-01T00:00:00.000000',
'storage_policy': TEST_POLICIES[0]},
{'subdir': 'foo_'},
{'subdir': get_reserved_name('foo_')},
{'name': 'foobar', 'bytes': 0, 'count': 0,
'last_modified': '2025-01-01T00:00:00.000000',
'storage_policy': TEST_POLICIES[1]},
{'name': 'nobar', 'bytes': 0, 'count': 0, # Unknown policy
'last_modified': '2025-02-01T00:00:00.000000'},
]).encode('ascii')
self.fake_container_listing_with_reserved = json.dumps([
{'name': 'bar', 'hash': 'etag', 'bytes': 0,
@ -68,7 +85,7 @@ class TestListingFormats(unittest.TestCase):
req = Request.blank('/v1/a')
resp = req.get_response(self.app)
self.assertEqual(resp.body, b'bar\nfoo_\n')
self.assertEqual(resp.body, b'bar\nfoo_\nfoobar\nnobar\n')
self.assertEqual(resp.headers['Content-Type'],
'text/plain; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1], (
@ -76,7 +93,7 @@ class TestListingFormats(unittest.TestCase):
req = Request.blank('/v1/a?format=plain')
resp = req.get_response(self.app)
self.assertEqual(resp.body, b'bar\nfoo_\n')
self.assertEqual(resp.body, b'bar\nfoo_\nfoobar\nnobar\n')
self.assertEqual(resp.headers['Content-Type'],
'text/plain; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1], (
@ -98,8 +115,16 @@ class TestListingFormats(unittest.TestCase):
b'<account name="a">',
b'<container><name>bar</name><count>0</count><bytes>0</bytes>'
b'<last_modified>1970-01-01T00:00:00.000000</last_modified>'
b'</container>',
b'<storage_policy>%s</storage_policy>'
b'</container>' % TEST_POLICIES[0].encode('ascii'),
b'<subdir name="foo_" />',
b'<container><name>foobar</name><count>0</count><bytes>0</bytes>'
b'<last_modified>2025-01-01T00:00:00.000000</last_modified>'
b'<storage_policy>%s</storage_policy>'
b'</container>' % TEST_POLICIES[1].encode('ascii'),
b'<container><name>nobar</name><count>0</count><bytes>0</bytes>'
b'<last_modified>2025-02-01T00:00:00.000000</last_modified>'
b'</container>',
b'</account>',
])
self.assertEqual(resp.headers['Content-Type'],
@ -247,7 +272,7 @@ class TestListingFormats(unittest.TestCase):
req = Request.blank('/v1/a\xe2\x98\x83')
resp = req.get_response(self.app)
self.assertEqual(resp.body, b'bar\nfoo_\n')
self.assertEqual(resp.body, b'bar\nfoo_\nfoobar\nnobar\n')
self.assertEqual(resp.headers['Content-Type'],
'text/plain; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1], (
@ -262,7 +287,7 @@ class TestListingFormats(unittest.TestCase):
req = Request.blank('/v1/a\xe2\x98\x83', headers={
'X-Backend-Allow-Reserved-Names': 'true'})
resp = req.get_response(self.app)
self.assertEqual(resp.body, b'bar\n%s\nfoo_\n%s\n' % (
self.assertEqual(resp.body, b'bar\n%s\nfoo_\n%s\nfoobar\nnobar\n' % (
get_reserved_name('bar', 'versions').encode('ascii'),
get_reserved_name('foo_').encode('ascii'),
))
@ -273,7 +298,7 @@ class TestListingFormats(unittest.TestCase):
req = Request.blank('/v1/a\xe2\x98\x83?format=plain')
resp = req.get_response(self.app)
self.assertEqual(resp.body, b'bar\nfoo_\n')
self.assertEqual(resp.body, b'bar\nfoo_\nfoobar\nnobar\n')
self.assertEqual(resp.headers['Content-Type'],
'text/plain; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1], (
@ -282,7 +307,7 @@ class TestListingFormats(unittest.TestCase):
req = Request.blank('/v1/a\xe2\x98\x83?format=plain', headers={
'X-Backend-Allow-Reserved-Names': 'true'})
resp = req.get_response(self.app)
self.assertEqual(resp.body, b'bar\n%s\nfoo_\n%s\n' % (
self.assertEqual(resp.body, b'bar\n%s\nfoo_\n%s\nfoobar\nnobar\n' % (
get_reserved_name('bar', 'versions').encode('ascii'),
get_reserved_name('foo_').encode('ascii'),
))
@ -317,8 +342,16 @@ class TestListingFormats(unittest.TestCase):
b'<account name="a\xe2\x98\x83">',
b'<container><name>bar</name><count>0</count><bytes>0</bytes>'
b'<last_modified>1970-01-01T00:00:00.000000</last_modified>'
b'</container>',
b'<storage_policy>%s</storage_policy>'
b'</container>' % TEST_POLICIES[0].encode('ascii'),
b'<subdir name="foo_" />',
b'<container><name>foobar</name><count>0</count><bytes>0</bytes>'
b'<last_modified>2025-01-01T00:00:00.000000</last_modified>'
b'<storage_policy>%s</storage_policy>'
b'</container>' % TEST_POLICIES[1].encode('ascii'),
b'<container><name>nobar</name><count>0</count><bytes>0</bytes>'
b'<last_modified>2025-02-01T00:00:00.000000</last_modified>'
b'</container>',
b'</account>',
])
self.assertEqual(resp.headers['Content-Type'],
@ -334,15 +367,26 @@ class TestListingFormats(unittest.TestCase):
b'<account name="a\xe2\x98\x83">',
b'<container><name>bar</name><count>0</count><bytes>0</bytes>'
b'<last_modified>1970-01-01T00:00:00.000000</last_modified>'
b'</container>',
b'<storage_policy>%s</storage_policy>'
b'</container>' % TEST_POLICIES[0].encode('ascii'),
b'<container><name>%s</name>'
b'<count>0</count><bytes>0</bytes>'
b'<last_modified>1970-01-01T00:00:00.000000</last_modified>'
b'</container>' % get_reserved_name(
'bar', 'versions').encode('ascii'),
b'<storage_policy>%s</storage_policy>'
b'</container>' % (
get_reserved_name('bar', 'versions').encode('ascii'),
TEST_POLICIES[0].encode('ascii'),
),
b'<subdir name="foo_" />',
b'<subdir name="%s" />' % get_reserved_name(
'foo_').encode('ascii'),
b'<container><name>foobar</name><count>0</count><bytes>0</bytes>'
b'<last_modified>2025-01-01T00:00:00.000000</last_modified>'
b'<storage_policy>%s</storage_policy>'
b'</container>' % TEST_POLICIES[1].encode('ascii'),
b'<container><name>nobar</name><count>0</count><bytes>0</bytes>'
b'<last_modified>2025-02-01T00:00:00.000000</last_modified>'
b'</container>',
b'</account>',
])
self.assertEqual(resp.headers['Content-Type'],
@ -659,8 +703,16 @@ class TestListingFormats(unittest.TestCase):
body = json.dumps([
{'name': 'bar', 'hash': 'etag', 'bytes': 0,
'content_type': 'text/plain',
'last_modified': '1970-01-01T00:00:00.000000'},
'last_modified': '1970-01-01T00:00:00.000000',
'storage_policy': TEST_POLICIES[0]},
{'subdir': 'foo/'},
{'name': 'foobar', 'hash': 'etag', 'bytes': 0,
'content_type': 'text/plain',
'last_modified': '2025-01-01T00:00:00.000000',
'storage_policy': TEST_POLICIES[1]},
{'name': 'nobar', 'hash': 'etag', 'bytes': 0,
'content_type': 'text/plain',
'last_modified': '2025-02-01T00:00:00.000000'},
] * 160000).encode('ascii')
self.assertGreater( # sanity
len(body), listing_formats.MAX_CONTAINER_LISTING_CONTENT_LENGTH)