diff --git a/swift/account/backend.py b/swift/account/backend.py
index d0241a2708..dbeb1eaa0d 100644
--- a/swift/account/backend.py
+++ b/swift/account/backend.py
@@ -367,7 +367,7 @@ class AccountBroker(DatabaseBroker):
:param allow_reserved: exclude names with reserved-byte by default
:returns: list of tuples of (name, object_count, bytes_used,
- put_timestamp, 0)
+ put_timestamp, storage_policy_index, 0)
"""
delim_force_gte = False
if reverse:
@@ -383,7 +383,8 @@ class AccountBroker(DatabaseBroker):
results = []
while len(results) < limit:
query = """
- SELECT name, object_count, bytes_used, put_timestamp, 0
+ SELECT name, object_count, bytes_used, put_timestamp,
+ {storage_policy_index}, 0
FROM container
WHERE """
query_args = []
@@ -415,7 +416,27 @@ class AccountBroker(DatabaseBroker):
query += ' ORDER BY name %s LIMIT ?' % \
('DESC' if reverse else '')
query_args.append(limit - len(results))
- curs = conn.execute(query, query_args)
+ try:
+ # First, try querying with the storage policy index.
+ curs = conn.execute(
+ query.format(
+ storage_policy_index="storage_policy_index"),
+ query_args)
+ except sqlite3.OperationalError as err:
+ # If the storage policy column is not available,
+ # the database has not been migrated to the new schema
+ # with storage_policy_index. Re-run the query with
+ # storage_policy_index set to 0, which is what
+ # would be set once the database is migrated.
+ # TODO(callumdickinson): If support for migrating
+ # pre-storage policy versions of Swift is dropped,
+ # then this special handling can be removed.
+ if "no such column: storage_policy_index" in str(err):
+ curs = conn.execute(
+ query.format(storage_policy_index="0"),
+ query_args)
+ else:
+ raise
curs.row_factory = None
# Delimiters without a prefix is ignored, further if there
@@ -452,7 +473,7 @@ class AccountBroker(DatabaseBroker):
delim_force_gte = True
dir_name = name[:end + len(delimiter)]
if dir_name != orig_marker:
- results.append([dir_name, 0, 0, '0', 1])
+ results.append([dir_name, 0, 0, '0', -1, 1])
curs.close()
break
results.append(row)
diff --git a/swift/account/reaper.py b/swift/account/reaper.py
index 3323ade720..10d59ea185 100644
--- a/swift/account/reaper.py
+++ b/swift/account/reaper.py
@@ -265,7 +265,8 @@ class AccountReaper(Daemon):
container_limit, '', None, None, None, allow_reserved=True))
while containers:
try:
- for (container, _junk, _junk, _junk, _junk) in containers:
+ for (container, _junk, _junk, _junk, _junk,
+ _junk) in containers:
this_shard = (
int(md5(container.encode('utf-8'),
usedforsecurity=False)
diff --git a/swift/account/utils.py b/swift/account/utils.py
index ac7bc3a4e9..231c207448 100644
--- a/swift/account/utils.py
+++ b/swift/account/utils.py
@@ -82,14 +82,34 @@ def account_listing_response(account, req, response_content_type, broker=None,
prefix, delimiter, reverse,
req.allow_reserved_names)
data = []
- for (name, object_count, bytes_used, put_timestamp, is_subdir) \
+ for (name, object_count, bytes_used, put_timestamp,
+ storage_policy_index, is_subdir) \
in account_list:
if is_subdir:
data.append({'subdir': name})
else:
- data.append(
- {'name': name, 'count': object_count, 'bytes': bytes_used,
- 'last_modified': Timestamp(put_timestamp).isoformat})
+ container = {
+ 'name': name,
+ 'count': object_count,
+ 'bytes': bytes_used,
+ 'last_modified': Timestamp(put_timestamp).isoformat}
+ # Add the container's storage policy to the response, unless:
+ # * storage_policy_index < 0, which means that
+ # the storage policy could not be determined
+ # * storage_policy_index was not found in POLICIES,
+ # which means the storage policy is missing from
+ # the Swift configuration.
+ # The storage policy should always be returned when
+ # everything is configured correctly, but clients are
+ # expected to be able to handle this case regardless.
+ if (
+ storage_policy_index >= 0
+ and storage_policy_index in POLICIES
+ ):
+ container['storage_policy'] = (
+ POLICIES[storage_policy_index].name
+ )
+ data.append(container)
if response_content_type.endswith('/xml'):
account_list = listing_formats.account_to_xml(data, account)
ret = HTTPOk(body=account_list, request=req, headers=resp_headers)
diff --git a/swift/common/middleware/listing_formats.py b/swift/common/middleware/listing_formats.py
index 6d4ece4719..290a73152a 100644
--- a/swift/common/middleware/listing_formats.py
+++ b/swift/common/middleware/listing_formats.py
@@ -84,6 +84,9 @@ def account_to_xml(listing, account_name):
sub = SubElement(doc, 'container')
for field in ('name', 'count', 'bytes', 'last_modified'):
SubElement(sub, field).text = str(record.pop(field))
+ for field in ('storage_policy',):
+ if field in record:
+ SubElement(sub, field).text = str(record.pop(field))
sub.tail = '\n'
return to_xml(doc)
diff --git a/test/functional/swift_test_client.py b/test/functional/swift_test_client.py
index 5b01b411a0..fd72c302e2 100644
--- a/test/functional/swift_test_client.py
+++ b/test/functional/swift_test_client.py
@@ -570,7 +570,8 @@ class Account(Base):
tree = minidom.parseString(self.conn.response.read())
for x in tree.getElementsByTagName('container'):
cont = {}
- for key in ['name', 'count', 'bytes', 'last_modified']:
+ for key in ['name', 'count', 'bytes', 'last_modified',
+ 'storage_policy']:
cont[key] = x.getElementsByTagName(key)[0].\
childNodes[0].nodeValue
conts.append(cont)
diff --git a/test/unit/account/test_backend.py b/test/unit/account/test_backend.py
index 4cfde189c0..6c97a01159 100644
--- a/test/unit/account/test_backend.py
+++ b/test/unit/account/test_backend.py
@@ -1448,7 +1448,7 @@ class TestAccountBrokerBeforeSPI(TestAccountBroker):
# make sure we can iter containers without the migration
for c in broker.list_containers_iter(1, None, None, None, None):
- self.assertEqual(c, ('test_name', 1, 2, timestamp, 0))
+ self.assertEqual(c, ('test_name', 1, 2, timestamp, 0, 0))
# stats table is mysteriously empty...
stats = broker.get_policy_stats()
@@ -1607,7 +1607,7 @@ class TestAccountBrokerBeforeSPI(TestAccountBroker):
# make sure "test_name" container in new database
self.assertEqual(new_broker.get_info()['container_count'], 1)
for c in new_broker.list_containers_iter(1, None, None, None, None):
- self.assertEqual(c, ('test_name', 1, 2, timestamp, 0))
+ self.assertEqual(c, ('test_name', 1, 2, timestamp, 0, 0))
# full migration successful
with new_broker.get() as conn:
diff --git a/test/unit/account/test_reaper.py b/test/unit/account/test_reaper.py
index ae2cae5dfc..6bbc14f535 100644
--- a/test/unit/account/test_reaper.py
+++ b/test/unit/account/test_reaper.py
@@ -59,7 +59,7 @@ class FakeAccountBroker(object):
kwargs, ))
for cont in self.containers:
if cont > marker:
- yield cont, None, None, None, None
+ yield cont, None, None, None, None, None
limit -= 1
if limit <= 0:
break
@@ -735,7 +735,7 @@ class TestReaper(unittest.TestCase):
if container in self.containers_yielded:
continue
- yield container, None, None, None, None
+ yield container, None, None, None, None, None
self.containers_yielded.append(container)
def fake_reap_container(self, account, account_partition,
diff --git a/test/unit/account/test_server.py b/test/unit/account/test_server.py
index 63ee07fb69..49dc7430ed 100644
--- a/test/unit/account/test_server.py
+++ b/test/unit/account/test_server.py
@@ -1089,6 +1089,8 @@ class TestAccountController(unittest.TestCase):
self.assertEqual(resp.content_type, 'text/plain')
self.assertEqual(resp.charset, 'utf-8')
+ @patch_policies([StoragePolicy(0, 'zero', is_default=True),
+ StoragePolicy(1, 'one', is_default=False)])
def test_GET_with_containers_json(self):
put_timestamps = {}
req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'PUT',
@@ -1108,7 +1110,8 @@ class TestAccountController(unittest.TestCase):
'X-Delete-Timestamp': '0',
'X-Object-Count': '0',
'X-Bytes-Used': '0',
- 'X-Timestamp': normalize_timestamp(0)})
+ 'X-Timestamp': normalize_timestamp(0),
+ 'X-Backend-Storage-Policy-Index': 1})
req.get_response(self.controller)
req = Request.blank('/sda1/p/a?format=json',
environ={'REQUEST_METHOD': 'GET'})
@@ -1117,9 +1120,11 @@ class TestAccountController(unittest.TestCase):
self.assertEqual(
json.loads(resp.body),
[{'count': 0, 'bytes': 0, 'name': 'c1',
- 'last_modified': Timestamp(put_timestamps['c1']).isoformat},
+ 'last_modified': Timestamp(put_timestamps['c1']).isoformat,
+ 'storage_policy': POLICIES[0].name},
{'count': 0, 'bytes': 0, 'name': 'c2',
- 'last_modified': Timestamp(put_timestamps['c2']).isoformat}])
+ 'last_modified': Timestamp(put_timestamps['c2']).isoformat,
+ 'storage_policy': POLICIES[1].name}])
put_timestamps['c1'] = normalize_timestamp(3)
req = Request.blank('/sda1/p/a/c1', environ={'REQUEST_METHOD': 'PUT'},
headers={'X-Put-Timestamp': put_timestamps['c1'],
@@ -1134,7 +1139,8 @@ class TestAccountController(unittest.TestCase):
'X-Delete-Timestamp': '0',
'X-Object-Count': '3',
'X-Bytes-Used': '4',
- 'X-Timestamp': normalize_timestamp(0)})
+ 'X-Timestamp': normalize_timestamp(0),
+ 'X-Backend-Storage-Policy-Index': 1})
req.get_response(self.controller)
req = Request.blank('/sda1/p/a?format=json',
environ={'REQUEST_METHOD': 'GET'})
@@ -1143,12 +1149,16 @@ class TestAccountController(unittest.TestCase):
self.assertEqual(
json.loads(resp.body),
[{'count': 1, 'bytes': 2, 'name': 'c1',
- 'last_modified': Timestamp(put_timestamps['c1']).isoformat},
+ 'last_modified': Timestamp(put_timestamps['c1']).isoformat,
+ 'storage_policy': POLICIES[0].name},
{'count': 3, 'bytes': 4, 'name': 'c2',
- 'last_modified': Timestamp(put_timestamps['c2']).isoformat}])
+ 'last_modified': Timestamp(put_timestamps['c2']).isoformat,
+ 'storage_policy': POLICIES[1].name}])
self.assertEqual(resp.content_type, 'application/json')
self.assertEqual(resp.charset, 'utf-8')
+ @patch_policies([StoragePolicy(0, 'zero', is_default=True),
+ StoragePolicy(1, 'one', is_default=False)])
def test_GET_with_containers_xml(self):
put_timestamps = {}
req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'PUT',
@@ -1168,7 +1178,8 @@ class TestAccountController(unittest.TestCase):
'X-Delete-Timestamp': '0',
'X-Object-Count': '0',
'X-Bytes-Used': '0',
- 'X-Timestamp': normalize_timestamp(0)})
+ 'X-Timestamp': normalize_timestamp(0),
+ 'X-Backend-Storage-Policy-Index': 1})
req.get_response(self.controller)
req = Request.blank('/sda1/p/a?format=xml',
environ={'REQUEST_METHOD': 'GET'})
@@ -1183,7 +1194,8 @@ class TestAccountController(unittest.TestCase):
self.assertEqual(listing[0].nodeName, 'container')
container = [n for n in listing[0].childNodes if n.nodeName != '#text']
self.assertEqual(sorted([n.nodeName for n in container]),
- ['bytes', 'count', 'last_modified', 'name'])
+ ['bytes', 'count', 'last_modified', 'name',
+ 'storage_policy'])
node = [n for n in container if n.nodeName == 'name'][0]
self.assertEqual(node.firstChild.nodeValue, 'c1')
node = [n for n in container if n.nodeName == 'count'][0]
@@ -1193,11 +1205,14 @@ class TestAccountController(unittest.TestCase):
node = [n for n in container if n.nodeName == 'last_modified'][0]
self.assertEqual(node.firstChild.nodeValue,
Timestamp(put_timestamps['c1']).isoformat)
+ node = [n for n in container if n.nodeName == 'storage_policy'][0]
+ self.assertEqual(node.firstChild.nodeValue, POLICIES[0].name)
self.assertEqual(listing[-1].nodeName, 'container')
container = \
[n for n in listing[-1].childNodes if n.nodeName != '#text']
self.assertEqual(sorted([n.nodeName for n in container]),
- ['bytes', 'count', 'last_modified', 'name'])
+ ['bytes', 'count', 'last_modified', 'name',
+ 'storage_policy'])
node = [n for n in container if n.nodeName == 'name'][0]
self.assertEqual(node.firstChild.nodeValue, 'c2')
node = [n for n in container if n.nodeName == 'count'][0]
@@ -1207,6 +1222,8 @@ class TestAccountController(unittest.TestCase):
node = [n for n in container if n.nodeName == 'last_modified'][0]
self.assertEqual(node.firstChild.nodeValue,
Timestamp(put_timestamps['c2']).isoformat)
+ node = [n for n in container if n.nodeName == 'storage_policy'][0]
+ self.assertEqual(node.firstChild.nodeValue, POLICIES[1].name)
req = Request.blank('/sda1/p/a/c1', environ={'REQUEST_METHOD': 'PUT'},
headers={'X-Put-Timestamp': '1',
'X-Delete-Timestamp': '0',
@@ -1219,7 +1236,8 @@ class TestAccountController(unittest.TestCase):
'X-Delete-Timestamp': '0',
'X-Object-Count': '3',
'X-Bytes-Used': '4',
- 'X-Timestamp': normalize_timestamp(0)})
+ 'X-Timestamp': normalize_timestamp(0),
+ 'X-Backend-Storage-Policy-Index': 1})
req.get_response(self.controller)
req = Request.blank('/sda1/p/a?format=xml',
environ={'REQUEST_METHOD': 'GET'})
@@ -1233,7 +1251,8 @@ class TestAccountController(unittest.TestCase):
self.assertEqual(listing[0].nodeName, 'container')
container = [n for n in listing[0].childNodes if n.nodeName != '#text']
self.assertEqual(sorted([n.nodeName for n in container]),
- ['bytes', 'count', 'last_modified', 'name'])
+ ['bytes', 'count', 'last_modified', 'name',
+ 'storage_policy'])
node = [n for n in container if n.nodeName == 'name'][0]
self.assertEqual(node.firstChild.nodeValue, 'c1')
node = [n for n in container if n.nodeName == 'count'][0]
@@ -1243,11 +1262,14 @@ class TestAccountController(unittest.TestCase):
node = [n for n in container if n.nodeName == 'last_modified'][0]
self.assertEqual(node.firstChild.nodeValue,
Timestamp(put_timestamps['c1']).isoformat)
+ node = [n for n in container if n.nodeName == 'storage_policy'][0]
+ self.assertEqual(node.firstChild.nodeValue, POLICIES[0].name)
self.assertEqual(listing[-1].nodeName, 'container')
container = [
n for n in listing[-1].childNodes if n.nodeName != '#text']
self.assertEqual(sorted([n.nodeName for n in container]),
- ['bytes', 'count', 'last_modified', 'name'])
+ ['bytes', 'count', 'last_modified', 'name',
+ 'storage_policy'])
node = [n for n in container if n.nodeName == 'name'][0]
self.assertEqual(node.firstChild.nodeValue, 'c2')
node = [n for n in container if n.nodeName == 'count'][0]
@@ -1257,6 +1279,8 @@ class TestAccountController(unittest.TestCase):
node = [n for n in container if n.nodeName == 'last_modified'][0]
self.assertEqual(node.firstChild.nodeValue,
Timestamp(put_timestamps['c2']).isoformat)
+ node = [n for n in container if n.nodeName == 'storage_policy'][0]
+ self.assertEqual(node.firstChild.nodeValue, POLICIES[1].name)
self.assertEqual(resp.charset, 'utf-8')
def test_GET_xml_escapes_account_name(self):
@@ -1347,6 +1371,8 @@ class TestAccountController(unittest.TestCase):
self.assertEqual(resp.body.strip().split(b'\n'),
[b'c3', b'c4'])
+ @patch_policies([StoragePolicy(0, 'zero', is_default=True),
+ StoragePolicy(1, 'one', is_default=False)])
def test_GET_limit_marker_json(self):
req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'PUT',
'HTTP_X_TIMESTAMP': '0'})
@@ -1360,29 +1386,37 @@ class TestAccountController(unittest.TestCase):
'X-Delete-Timestamp': '0',
'X-Object-Count': '2',
'X-Bytes-Used': '3',
- 'X-Timestamp': put_timestamp})
+ 'X-Timestamp': put_timestamp,
+ 'X-Backend-Storage-Policy-Index': c % 2})
req.get_response(self.controller)
req = Request.blank('/sda1/p/a?limit=3&format=json',
environ={'REQUEST_METHOD': 'GET'})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 200)
expected = [{'count': 2, 'bytes': 3, 'name': 'c0',
- 'last_modified': Timestamp('1').isoformat},
+ 'last_modified': Timestamp('1').isoformat,
+ 'storage_policy': POLICIES[0].name},
{'count': 2, 'bytes': 3, 'name': 'c1',
- 'last_modified': Timestamp('2').isoformat},
+ 'last_modified': Timestamp('2').isoformat,
+ 'storage_policy': POLICIES[1].name},
{'count': 2, 'bytes': 3, 'name': 'c2',
- 'last_modified': Timestamp('3').isoformat}]
+ 'last_modified': Timestamp('3').isoformat,
+ 'storage_policy': POLICIES[0].name}]
self.assertEqual(json.loads(resp.body), expected)
req = Request.blank('/sda1/p/a?limit=3&marker=c2&format=json',
environ={'REQUEST_METHOD': 'GET'})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 200)
expected = [{'count': 2, 'bytes': 3, 'name': 'c3',
- 'last_modified': Timestamp('4').isoformat},
+ 'last_modified': Timestamp('4').isoformat,
+ 'storage_policy': POLICIES[1].name},
{'count': 2, 'bytes': 3, 'name': 'c4',
- 'last_modified': Timestamp('5').isoformat}]
+ 'last_modified': Timestamp('5').isoformat,
+ 'storage_policy': POLICIES[0].name}]
self.assertEqual(json.loads(resp.body), expected)
+ @patch_policies([StoragePolicy(0, 'zero', is_default=True),
+ StoragePolicy(1, 'one', is_default=False)])
def test_GET_limit_marker_xml(self):
req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'PUT',
'HTTP_X_TIMESTAMP': '0'})
@@ -1396,7 +1430,8 @@ class TestAccountController(unittest.TestCase):
'X-Delete-Timestamp': '0',
'X-Object-Count': '2',
'X-Bytes-Used': '3',
- 'X-Timestamp': put_timestamp})
+ 'X-Timestamp': put_timestamp,
+ 'X-Backend-Storage-Policy-Index': c % 2})
req.get_response(self.controller)
req = Request.blank('/sda1/p/a?limit=3&format=xml',
environ={'REQUEST_METHOD': 'GET'})
@@ -1410,7 +1445,8 @@ class TestAccountController(unittest.TestCase):
self.assertEqual(listing[0].nodeName, 'container')
container = [n for n in listing[0].childNodes if n.nodeName != '#text']
self.assertEqual(sorted([n.nodeName for n in container]),
- ['bytes', 'count', 'last_modified', 'name'])
+ ['bytes', 'count', 'last_modified', 'name',
+ 'storage_policy'])
node = [n for n in container if n.nodeName == 'name'][0]
self.assertEqual(node.firstChild.nodeValue, 'c0')
node = [n for n in container if n.nodeName == 'count'][0]
@@ -1420,11 +1456,14 @@ class TestAccountController(unittest.TestCase):
node = [n for n in container if n.nodeName == 'last_modified'][0]
self.assertEqual(node.firstChild.nodeValue,
Timestamp('1').isoformat)
+ node = [n for n in container if n.nodeName == 'storage_policy'][0]
+ self.assertEqual(node.firstChild.nodeValue, POLICIES[0].name)
self.assertEqual(listing[-1].nodeName, 'container')
container = [
n for n in listing[-1].childNodes if n.nodeName != '#text']
self.assertEqual(sorted([n.nodeName for n in container]),
- ['bytes', 'count', 'last_modified', 'name'])
+ ['bytes', 'count', 'last_modified', 'name',
+ 'storage_policy'])
node = [n for n in container if n.nodeName == 'name'][0]
self.assertEqual(node.firstChild.nodeValue, 'c2')
node = [n for n in container if n.nodeName == 'count'][0]
@@ -1434,6 +1473,8 @@ class TestAccountController(unittest.TestCase):
node = [n for n in container if n.nodeName == 'last_modified'][0]
self.assertEqual(node.firstChild.nodeValue,
Timestamp('3').isoformat)
+ node = [n for n in container if n.nodeName == 'storage_policy'][0]
+ self.assertEqual(node.firstChild.nodeValue, POLICIES[0].name)
req = Request.blank('/sda1/p/a?limit=3&marker=c2&format=xml',
environ={'REQUEST_METHOD': 'GET'})
resp = req.get_response(self.controller)
@@ -1446,7 +1487,8 @@ class TestAccountController(unittest.TestCase):
self.assertEqual(listing[0].nodeName, 'container')
container = [n for n in listing[0].childNodes if n.nodeName != '#text']
self.assertEqual(sorted([n.nodeName for n in container]),
- ['bytes', 'count', 'last_modified', 'name'])
+ ['bytes', 'count', 'last_modified', 'name',
+ 'storage_policy'])
node = [n for n in container if n.nodeName == 'name'][0]
self.assertEqual(node.firstChild.nodeValue, 'c3')
node = [n for n in container if n.nodeName == 'count'][0]
@@ -1456,11 +1498,14 @@ class TestAccountController(unittest.TestCase):
node = [n for n in container if n.nodeName == 'last_modified'][0]
self.assertEqual(node.firstChild.nodeValue,
Timestamp('4').isoformat)
+ node = [n for n in container if n.nodeName == 'storage_policy'][0]
+ self.assertEqual(node.firstChild.nodeValue, POLICIES[1].name)
self.assertEqual(listing[-1].nodeName, 'container')
container = [
n for n in listing[-1].childNodes if n.nodeName != '#text']
self.assertEqual(sorted([n.nodeName for n in container]),
- ['bytes', 'count', 'last_modified', 'name'])
+ ['bytes', 'count', 'last_modified', 'name',
+ 'storage_policy'])
node = [n for n in container if n.nodeName == 'name'][0]
self.assertEqual(node.firstChild.nodeValue, 'c4')
node = [n for n in container if n.nodeName == 'count'][0]
@@ -1470,6 +1515,8 @@ class TestAccountController(unittest.TestCase):
node = [n for n in container if n.nodeName == 'last_modified'][0]
self.assertEqual(node.firstChild.nodeValue,
Timestamp('5').isoformat)
+ node = [n for n in container if n.nodeName == 'storage_policy'][0]
+ self.assertEqual(node.firstChild.nodeValue, POLICIES[0].name)
def test_GET_accept_wildcard(self):
req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'PUT',
@@ -1911,13 +1958,18 @@ class TestAccountController(unittest.TestCase):
self.assertEqual(resp.status_int // 100, 2, resp.body)
for container in containers:
path = '/sda1/p/%s/%s' % (account, container['name'])
- req = Request.blank(path, method='PUT', headers={
+ headers = {
'X-Put-Timestamp': container['timestamp'].internal,
'X-Delete-Timestamp': container.get(
'deleted', Timestamp(0)).internal,
'X-Object-Count': container['count'],
'X-Bytes-Used': container['bytes'],
- })
+ }
+ if 'storage_policy' in container:
+ headers['X-Backend-Storage-Policy-Index'] = (
+ POLICIES.get_by_name(container['storage_policy']).idx
+ )
+ req = Request.blank(path, method='PUT', headers=headers)
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int // 100, 2, resp.body)
@@ -1927,6 +1979,7 @@ class TestAccountController(unittest.TestCase):
'bytes': 200,
'count': 2,
'timestamp': next(self.ts),
+ 'storage_policy': POLICIES[0].name,
}]
self._report_containers(containers)
@@ -1960,17 +2013,21 @@ class TestAccountController(unittest.TestCase):
self.assertEqual(json.loads(resp.body), [{
'subdir': '%s' % get_reserved_name('null')}])
+ @patch_policies([StoragePolicy(0, 'zero', is_default=True),
+ StoragePolicy(1, 'one', is_default=False)])
def test_delimiter_with_reserved_and_public(self):
containers = [{
'name': get_reserved_name('null', 'test01'),
'bytes': 200,
'count': 2,
'timestamp': next(self.ts),
+ 'storage_policy': POLICIES[0].name,
}, {
'name': 'nullish',
'bytes': 10,
'count': 10,
'timestamp': next(self.ts),
+ 'storage_policy': POLICIES[1].name,
}]
self._report_containers(containers)
@@ -2020,17 +2077,21 @@ class TestAccountController(unittest.TestCase):
[{'subdir': '\x00'}] +
self._expected_listing(containers)[1:])
+ @patch_policies([StoragePolicy(0, 'zero', is_default=True),
+ StoragePolicy(1, 'one', is_default=False)])
def test_markers_with_reserved(self):
containers = [{
'name': get_reserved_name('null', 'test01'),
'bytes': 200,
'count': 2,
'timestamp': next(self.ts),
+ 'storage_policy': POLICIES[0].name,
}, {
'name': get_reserved_name('null', 'test02'),
'bytes': 10,
'count': 10,
'timestamp': next(self.ts),
+ 'storage_policy': POLICIES[1].name,
}]
self._report_containers(containers)
@@ -2064,6 +2125,7 @@ class TestAccountController(unittest.TestCase):
'bytes': 300,
'count': 30,
'timestamp': next(self.ts),
+ 'storage_policy': POLICIES[0].name,
})
self._report_containers(containers)
@@ -2085,27 +2147,33 @@ class TestAccountController(unittest.TestCase):
self.assertEqual(json.loads(resp.body),
self._expected_listing(containers)[-1:])
+ @patch_policies([StoragePolicy(0, 'zero', is_default=True),
+ StoragePolicy(1, 'one', is_default=False)])
def test_prefix_with_reserved(self):
containers = [{
'name': get_reserved_name('null', 'test01'),
'bytes': 200,
'count': 2,
'timestamp': next(self.ts),
+ 'storage_policy': POLICIES[0].name,
}, {
'name': get_reserved_name('null', 'test02'),
'bytes': 10,
'count': 10,
'timestamp': next(self.ts),
+ 'storage_policy': POLICIES[1].name,
}, {
'name': get_reserved_name('null', 'foo'),
'bytes': 10,
'count': 10,
'timestamp': next(self.ts),
+ 'storage_policy': POLICIES[0].name,
}, {
'name': get_reserved_name('nullish'),
'bytes': 300,
'count': 32,
'timestamp': next(self.ts),
+ 'storage_policy': POLICIES[1].name,
}]
self._report_containers(containers)
@@ -2125,27 +2193,33 @@ class TestAccountController(unittest.TestCase):
self.assertEqual(json.loads(resp.body),
self._expected_listing(containers[:2]))
+ @patch_policies([StoragePolicy(0, 'zero', is_default=True),
+ StoragePolicy(1, 'one', is_default=False)])
def test_prefix_and_delim_with_reserved(self):
containers = [{
'name': get_reserved_name('null', 'test01'),
'bytes': 200,
'count': 2,
'timestamp': next(self.ts),
+ 'storage_policy': POLICIES[0].name,
}, {
'name': get_reserved_name('null', 'test02'),
'bytes': 10,
'count': 10,
'timestamp': next(self.ts),
+ 'storage_policy': POLICIES[1].name,
}, {
'name': get_reserved_name('null', 'foo'),
'bytes': 10,
'count': 10,
'timestamp': next(self.ts),
+ 'storage_policy': POLICIES[0].name,
}, {
'name': get_reserved_name('nullish'),
'bytes': 300,
'count': 32,
'timestamp': next(self.ts),
+ 'storage_policy': POLICIES[1].name,
}]
self._report_containers(containers)
@@ -2166,22 +2240,27 @@ class TestAccountController(unittest.TestCase):
self._expected_listing(containers[-1:])
self.assertEqual(json.loads(resp.body), expected)
+ @patch_policies([StoragePolicy(0, 'zero', is_default=True),
+ StoragePolicy(1, 'one', is_default=False)])
def test_reserved_markers_with_non_reserved(self):
containers = [{
'name': get_reserved_name('null', 'test01'),
'bytes': 200,
'count': 2,
'timestamp': next(self.ts),
+ 'storage_policy': POLICIES[0].name,
}, {
'name': get_reserved_name('null', 'test02'),
'bytes': 10,
'count': 10,
'timestamp': next(self.ts),
+ 'storage_policy': POLICIES[1].name,
}, {
'name': 'nullish',
'bytes': 300,
'count': 32,
'timestamp': next(self.ts),
+ 'storage_policy': POLICIES[0].name,
}]
self._report_containers(containers)
@@ -2221,22 +2300,27 @@ class TestAccountController(unittest.TestCase):
self.assertEqual(json.loads(resp.body),
self._expected_listing(containers)[1:])
+ @patch_policies([StoragePolicy(0, 'zero', is_default=True),
+ StoragePolicy(1, 'one', is_default=False)])
def test_null_markers(self):
containers = [{
'name': get_reserved_name('null', ''),
'bytes': 200,
'count': 2,
'timestamp': next(self.ts),
+ 'storage_policy': POLICIES[0].name,
}, {
'name': get_reserved_name('null', 'test01'),
'bytes': 200,
'count': 2,
'timestamp': next(self.ts),
+ 'storage_policy': POLICIES[1].name,
}, {
'name': 'null',
'bytes': 300,
'count': 32,
'timestamp': next(self.ts),
+ 'storage_policy': POLICIES[0].name,
}]
self._report_containers(containers)
diff --git a/test/unit/account/test_utils.py b/test/unit/account/test_utils.py
index 473b3b053a..4ac806fb67 100644
--- a/test/unit/account/test_utils.py
+++ b/test/unit/account/test_utils.py
@@ -214,7 +214,58 @@ class TestAccountUtils(TestDbBase):
self.assertEqual(expected, resp.headers)
self.assertEqual(b'', resp.body)
- @patch_policies([StoragePolicy(0, 'zero', is_default=True)])
+ @patch_policies([StoragePolicy(0, 'zero', is_default=True),
+ StoragePolicy(1, 'one', is_default=False)])
+ def test_account_listing_with_containers(self):
+ broker = backend.AccountBroker(self.db_path, account='a')
+ put_timestamp = next(self.ts)
+ now = time.time()
+ with mock.patch('time.time', new=lambda: now):
+ broker.initialize(put_timestamp.internal)
+ container_timestamp = next(self.ts)
+ broker.put_container('foo',
+ container_timestamp.internal, 0, 10, 100, 0)
+ broker.put_container('bar',
+ container_timestamp.internal, 0, 10, 100, 1)
+
+ req = Request.blank('')
+ resp = utils.account_listing_response(
+ 'a', req, 'application/json', broker)
+ self.assertEqual(resp.status_int, 200)
+ expected = HeaderKeyDict({
+ 'Content-Type': 'application/json; charset=utf-8',
+ 'Content-Length': 233,
+ 'X-Account-Container-Count': 2,
+ 'X-Account-Object-Count': 20,
+ 'X-Account-Bytes-Used': 200,
+ 'X-Timestamp': Timestamp(now).normal,
+ 'X-PUT-Timestamp': put_timestamp.normal,
+ 'X-Account-Storage-Policy-Zero-Container-Count': 1,
+ 'X-Account-Storage-Policy-Zero-Object-Count': 10,
+ 'X-Account-Storage-Policy-Zero-Bytes-Used': 100,
+ 'X-Account-Storage-Policy-One-Container-Count': 1,
+ 'X-Account-Storage-Policy-One-Object-Count': 10,
+ 'X-Account-Storage-Policy-One-Bytes-Used': 100,
+ })
+ self.assertEqual(expected, resp.headers)
+ expected = [{
+ "last_modified": container_timestamp.isoformat,
+ "count": 10,
+ "bytes": 100,
+ "name": 'foo',
+ 'storage_policy': POLICIES[0].name,
+ }, {
+ "last_modified": container_timestamp.isoformat,
+ "count": 10,
+ "bytes": 100,
+ "name": 'bar',
+ 'storage_policy': POLICIES[1].name,
+ }]
+ self.assertEqual(sorted(json.dumps(expected).encode('ascii')),
+ sorted(resp.body))
+
+ @patch_policies([StoragePolicy(0, 'zero', is_default=True),
+ StoragePolicy(1, 'one', is_default=False)])
def test_account_listing_reserved_names(self):
broker = backend.AccountBroker(self.db_path, account='a')
put_timestamp = next(self.ts)
@@ -224,6 +275,8 @@ class TestAccountUtils(TestDbBase):
container_timestamp = next(self.ts)
broker.put_container(get_reserved_name('foo'),
container_timestamp.internal, 0, 10, 100, 0)
+ broker.put_container(get_reserved_name('bar'),
+ container_timestamp.internal, 0, 10, 100, 1)
req = Request.blank('')
resp = utils.account_listing_response(
@@ -232,14 +285,17 @@ class TestAccountUtils(TestDbBase):
expected = HeaderKeyDict({
'Content-Type': 'application/json; charset=utf-8',
'Content-Length': 2,
- 'X-Account-Container-Count': 1,
- 'X-Account-Object-Count': 10,
- 'X-Account-Bytes-Used': 100,
+ 'X-Account-Container-Count': 2,
+ 'X-Account-Object-Count': 20,
+ 'X-Account-Bytes-Used': 200,
'X-Timestamp': Timestamp(now).normal,
'X-PUT-Timestamp': put_timestamp.normal,
'X-Account-Storage-Policy-Zero-Container-Count': 1,
'X-Account-Storage-Policy-Zero-Object-Count': 10,
'X-Account-Storage-Policy-Zero-Bytes-Used': 100,
+ 'X-Account-Storage-Policy-One-Container-Count': 1,
+ 'X-Account-Storage-Policy-One-Object-Count': 10,
+ 'X-Account-Storage-Policy-One-Bytes-Used': 100,
})
self.assertEqual(expected, resp.headers)
self.assertEqual(b'[]', resp.body)
@@ -251,15 +307,18 @@ class TestAccountUtils(TestDbBase):
self.assertEqual(resp.status_int, 200)
expected = HeaderKeyDict({
'Content-Type': 'application/json; charset=utf-8',
- 'Content-Length': 97,
- 'X-Account-Container-Count': 1,
- 'X-Account-Object-Count': 10,
- 'X-Account-Bytes-Used': 100,
+ 'Content-Length': 245,
+ 'X-Account-Container-Count': 2,
+ 'X-Account-Object-Count': 20,
+ 'X-Account-Bytes-Used': 200,
'X-Timestamp': Timestamp(now).normal,
'X-PUT-Timestamp': put_timestamp.normal,
'X-Account-Storage-Policy-Zero-Container-Count': 1,
'X-Account-Storage-Policy-Zero-Object-Count': 10,
'X-Account-Storage-Policy-Zero-Bytes-Used': 100,
+ 'X-Account-Storage-Policy-One-Container-Count': 1,
+ 'X-Account-Storage-Policy-One-Object-Count': 10,
+ 'X-Account-Storage-Policy-One-Bytes-Used': 100,
})
self.assertEqual(expected, resp.headers)
expected = [{
@@ -267,6 +326,13 @@ class TestAccountUtils(TestDbBase):
"count": 10,
"bytes": 100,
"name": get_reserved_name('foo'),
+ 'storage_policy': POLICIES[0].name,
+ }, {
+ "last_modified": container_timestamp.isoformat,
+ "count": 10,
+ "bytes": 100,
+ "name": get_reserved_name('bar'),
+ 'storage_policy': POLICIES[1].name,
}]
self.assertEqual(sorted(json.dumps(expected).encode('ascii')),
sorted(resp.body))
diff --git a/test/unit/common/middleware/test_listing_formats.py b/test/unit/common/middleware/test_listing_formats.py
index a93d95b116..315e82de31 100644
--- a/test/unit/common/middleware/test_listing_formats.py
+++ b/test/unit/common/middleware/test_listing_formats.py
@@ -20,10 +20,14 @@ from swift.common.header_key_dict import HeaderKeyDict
from swift.common.swob import Request, HTTPOk, HTTPNoContent
from swift.common.middleware import listing_formats
from swift.common.request_helpers import get_reserved_name
+from swift.common.storage_policy import POLICIES
from test.debug_logger import debug_logger
from test.unit.common.middleware.helpers import FakeSwift
+TEST_POLICIES = (POLICIES[0].name, 'Policy-1')
+
+
class TestListingFormats(unittest.TestCase):
def setUp(self):
self.fake_swift = FakeSwift()
@@ -32,8 +36,14 @@ class TestListingFormats(unittest.TestCase):
logger=self.logger)
self.fake_account_listing = json.dumps([
{'name': 'bar', 'bytes': 0, 'count': 0,
- 'last_modified': '1970-01-01T00:00:00.000000'},
+ 'last_modified': '1970-01-01T00:00:00.000000',
+ 'storage_policy': TEST_POLICIES[0]},
{'subdir': 'foo_'},
+ {'name': 'foobar', 'bytes': 0, 'count': 0,
+ 'last_modified': '2025-01-01T00:00:00.000000',
+ 'storage_policy': TEST_POLICIES[1]},
+ {'name': 'nobar', 'bytes': 0, 'count': 0, # Unknown policy
+ 'last_modified': '2025-02-01T00:00:00.000000'},
]).encode('ascii')
self.fake_container_listing = json.dumps([
{'name': 'bar', 'hash': 'etag', 'bytes': 0,
@@ -44,11 +54,18 @@ class TestListingFormats(unittest.TestCase):
self.fake_account_listing_with_reserved = json.dumps([
{'name': 'bar', 'bytes': 0, 'count': 0,
- 'last_modified': '1970-01-01T00:00:00.000000'},
+ 'last_modified': '1970-01-01T00:00:00.000000',
+ 'storage_policy': TEST_POLICIES[0]},
{'name': get_reserved_name('bar', 'versions'), 'bytes': 0,
- 'count': 0, 'last_modified': '1970-01-01T00:00:00.000000'},
+ 'count': 0, 'last_modified': '1970-01-01T00:00:00.000000',
+ 'storage_policy': TEST_POLICIES[0]},
{'subdir': 'foo_'},
{'subdir': get_reserved_name('foo_')},
+ {'name': 'foobar', 'bytes': 0, 'count': 0,
+ 'last_modified': '2025-01-01T00:00:00.000000',
+ 'storage_policy': TEST_POLICIES[1]},
+ {'name': 'nobar', 'bytes': 0, 'count': 0, # Unknown policy
+ 'last_modified': '2025-02-01T00:00:00.000000'},
]).encode('ascii')
self.fake_container_listing_with_reserved = json.dumps([
{'name': 'bar', 'hash': 'etag', 'bytes': 0,
@@ -68,7 +85,7 @@ class TestListingFormats(unittest.TestCase):
req = Request.blank('/v1/a')
resp = req.get_response(self.app)
- self.assertEqual(resp.body, b'bar\nfoo_\n')
+ self.assertEqual(resp.body, b'bar\nfoo_\nfoobar\nnobar\n')
self.assertEqual(resp.headers['Content-Type'],
'text/plain; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1], (
@@ -76,7 +93,7 @@ class TestListingFormats(unittest.TestCase):
req = Request.blank('/v1/a?format=plain')
resp = req.get_response(self.app)
- self.assertEqual(resp.body, b'bar\nfoo_\n')
+ self.assertEqual(resp.body, b'bar\nfoo_\nfoobar\nnobar\n')
self.assertEqual(resp.headers['Content-Type'],
'text/plain; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1], (
@@ -98,8 +115,16 @@ class TestListingFormats(unittest.TestCase):
b'',
b'bar00'
b'1970-01-01T00:00:00.000000'
- b'',
+ b'%s'
+ b'' % TEST_POLICIES[0].encode('ascii'),
b'',
+ b'foobar00'
+ b'2025-01-01T00:00:00.000000'
+ b'%s'
+ b'' % TEST_POLICIES[1].encode('ascii'),
+ b'nobar00'
+ b'2025-02-01T00:00:00.000000'
+ b'',
b'',
])
self.assertEqual(resp.headers['Content-Type'],
@@ -247,7 +272,7 @@ class TestListingFormats(unittest.TestCase):
req = Request.blank('/v1/a\xe2\x98\x83')
resp = req.get_response(self.app)
- self.assertEqual(resp.body, b'bar\nfoo_\n')
+ self.assertEqual(resp.body, b'bar\nfoo_\nfoobar\nnobar\n')
self.assertEqual(resp.headers['Content-Type'],
'text/plain; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1], (
@@ -262,7 +287,7 @@ class TestListingFormats(unittest.TestCase):
req = Request.blank('/v1/a\xe2\x98\x83', headers={
'X-Backend-Allow-Reserved-Names': 'true'})
resp = req.get_response(self.app)
- self.assertEqual(resp.body, b'bar\n%s\nfoo_\n%s\n' % (
+ self.assertEqual(resp.body, b'bar\n%s\nfoo_\n%s\nfoobar\nnobar\n' % (
get_reserved_name('bar', 'versions').encode('ascii'),
get_reserved_name('foo_').encode('ascii'),
))
@@ -273,7 +298,7 @@ class TestListingFormats(unittest.TestCase):
req = Request.blank('/v1/a\xe2\x98\x83?format=plain')
resp = req.get_response(self.app)
- self.assertEqual(resp.body, b'bar\nfoo_\n')
+ self.assertEqual(resp.body, b'bar\nfoo_\nfoobar\nnobar\n')
self.assertEqual(resp.headers['Content-Type'],
'text/plain; charset=utf-8')
self.assertEqual(self.fake_swift.calls[-1], (
@@ -282,7 +307,7 @@ class TestListingFormats(unittest.TestCase):
req = Request.blank('/v1/a\xe2\x98\x83?format=plain', headers={
'X-Backend-Allow-Reserved-Names': 'true'})
resp = req.get_response(self.app)
- self.assertEqual(resp.body, b'bar\n%s\nfoo_\n%s\n' % (
+ self.assertEqual(resp.body, b'bar\n%s\nfoo_\n%s\nfoobar\nnobar\n' % (
get_reserved_name('bar', 'versions').encode('ascii'),
get_reserved_name('foo_').encode('ascii'),
))
@@ -317,8 +342,16 @@ class TestListingFormats(unittest.TestCase):
b'',
b'bar00'
b'1970-01-01T00:00:00.000000'
- b'',
+ b'%s'
+ b'' % TEST_POLICIES[0].encode('ascii'),
b'',
+ b'foobar00'
+ b'2025-01-01T00:00:00.000000'
+ b'%s'
+ b'' % TEST_POLICIES[1].encode('ascii'),
+ b'nobar00'
+ b'2025-02-01T00:00:00.000000'
+ b'',
b'',
])
self.assertEqual(resp.headers['Content-Type'],
@@ -334,15 +367,26 @@ class TestListingFormats(unittest.TestCase):
b'',
b'bar00'
b'1970-01-01T00:00:00.000000'
- b'',
+ b'%s'
+ b'' % TEST_POLICIES[0].encode('ascii'),
b'%s'
b'00'
b'1970-01-01T00:00:00.000000'
- b'' % get_reserved_name(
- 'bar', 'versions').encode('ascii'),
+ b'%s'
+ b'' % (
+ get_reserved_name('bar', 'versions').encode('ascii'),
+ TEST_POLICIES[0].encode('ascii'),
+ ),
b'',
b'' % get_reserved_name(
'foo_').encode('ascii'),
+ b'foobar00'
+ b'2025-01-01T00:00:00.000000'
+ b'%s'
+ b'' % TEST_POLICIES[1].encode('ascii'),
+ b'nobar00'
+ b'2025-02-01T00:00:00.000000'
+ b'',
b'',
])
self.assertEqual(resp.headers['Content-Type'],
@@ -659,8 +703,16 @@ class TestListingFormats(unittest.TestCase):
body = json.dumps([
{'name': 'bar', 'hash': 'etag', 'bytes': 0,
'content_type': 'text/plain',
- 'last_modified': '1970-01-01T00:00:00.000000'},
+ 'last_modified': '1970-01-01T00:00:00.000000',
+ 'storage_policy': TEST_POLICIES[0]},
{'subdir': 'foo/'},
+ {'name': 'foobar', 'hash': 'etag', 'bytes': 0,
+ 'content_type': 'text/plain',
+ 'last_modified': '2025-01-01T00:00:00.000000',
+ 'storage_policy': TEST_POLICIES[1]},
+ {'name': 'nobar', 'hash': 'etag', 'bytes': 0,
+ 'content_type': 'text/plain',
+ 'last_modified': '2025-02-01T00:00:00.000000'},
] * 160000).encode('ascii')
self.assertGreater( # sanity
len(body), listing_formats.MAX_CONTAINER_LISTING_CONTENT_LENGTH)