swift/test/unit/account/test_utils.py
Clay Gerrard c1dc2fa624 Add two vector timestamps
The normalized form of the X-Timestamp header looks like a float with a fixed
width to ensure stable string sorting - normalized timestamps look like
"1402464677.04188"

To support overwrites of existing data without modifying the original
timestamp but still maintain consistency a second internal offset
vector is append to the normalized timestamp form which compares and
sorts greater than the fixed width float format but less than a newer
timestamp.  The internalized format of timestamps looks like
"1402464677.04188_0000000000000000" - the portion after the underscore
is the offset and is a formatted hexadecimal integer.

The internalized form is not exposed to clients in responses from Swift.
Normal client operations will not create a timestamp with an offset.

The Timestamp class in common.utils supports internalized and normalized
formatting of timestamps and also comparison of timestamp values.  When the
offset value of a Timestamp is 0 - it's considered insignificant and need not
be represented in the string format; to support backwards compatibility during
a Swift upgrade the internalized and normalized form of a Timestamp with an
insignificant offset are identical.  When a timestamp includes an offset it
will always be represented in the internalized form, but is still excluded
from the normalized form.  Timestamps with an equivalent timestamp portion
(the float part) will compare and order by their offset.  Timestamps with a
greater timestamp portion will always compare and order greater than a
Timestamp with a lesser timestamp regardless of it's offset.  String
comparison and ordering is guaranteed for the internalized string format, and
is backwards compatible for normalized timestamps which do not include an
offset.

The reconciler currently uses a offset bump to ensure that objects can move to
the wrong storage policy and be moved back.  This use-case is valid because
the content represented by the user-facing timestamp is not modified in way.
Future consumers of the offset vector of timestamps should be mindful of HTTP
semantics of If-Modified and take care to avoid deviation in the response from
the object server without an accompanying change to the user facing timestamp.

DocImpact
Implements: blueprint storage-policies
Change-Id: Id85c960b126ec919a481dc62469bf172b7fb8549
2014-06-19 10:18:06 -07:00

129 lines
4.8 KiB
Python

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import itertools
import time
import unittest
import mock
from swift.account import utils, backend
from swift.common.storage_policy import POLICIES
from swift.common.utils import Timestamp
from swift.common.swob import HeaderKeyDict
from test.unit import patch_policies
class TestFakeAccountBroker(unittest.TestCase):
def test_fake_broker_get_info(self):
broker = utils.FakeAccountBroker()
now = time.time()
with mock.patch('time.time', new=lambda: now):
info = broker.get_info()
timestamp = Timestamp(now)
expected = {
'container_count': 0,
'object_count': 0,
'bytes_used': 0,
'created_at': timestamp.internal,
'put_timestamp': timestamp.internal,
}
self.assertEqual(info, expected)
def test_fake_broker_list_containers_iter(self):
broker = utils.FakeAccountBroker()
self.assertEqual(broker.list_containers_iter(), [])
def test_fake_broker_metadata(self):
broker = utils.FakeAccountBroker()
self.assertEqual(broker.metadata, {})
def test_fake_broker_get_policy_stats(self):
broker = utils.FakeAccountBroker()
self.assertEqual(broker.get_policy_stats(), {})
class TestAccountUtils(unittest.TestCase):
def test_get_response_headers_fake_broker(self):
broker = utils.FakeAccountBroker()
now = time.time()
expected = {
'X-Account-Container-Count': 0,
'X-Account-Object-Count': 0,
'X-Account-Bytes-Used': 0,
'X-Timestamp': Timestamp(now).normal,
'X-PUT-Timestamp': Timestamp(now).normal,
}
with mock.patch('time.time', new=lambda: now):
resp_headers = utils.get_response_headers(broker)
self.assertEqual(resp_headers, expected)
def test_get_response_headers_empty_memory_broker(self):
broker = backend.AccountBroker(':memory:', account='a')
now = time.time()
with mock.patch('time.time', new=lambda: now):
broker.initialize(Timestamp(now).internal)
expected = {
'X-Account-Container-Count': 0,
'X-Account-Object-Count': 0,
'X-Account-Bytes-Used': 0,
'X-Timestamp': Timestamp(now).normal,
'X-PUT-Timestamp': Timestamp(now).normal,
}
resp_headers = utils.get_response_headers(broker)
self.assertEqual(resp_headers, expected)
@patch_policies
def test_get_response_headers_with_data(self):
broker = backend.AccountBroker(':memory:', account='a')
now = time.time()
with mock.patch('time.time', new=lambda: now):
broker.initialize(Timestamp(now).internal)
# add some container data
ts = (Timestamp(t).internal for t in itertools.count(int(now)))
total_containers = 0
total_objects = 0
total_bytes = 0
for policy in POLICIES:
delete_timestamp = ts.next()
put_timestamp = ts.next()
object_count = int(policy)
bytes_used = int(policy) * 10
broker.put_container('c-%s' % policy.name, put_timestamp,
delete_timestamp, object_count, bytes_used,
int(policy))
total_containers += 1
total_objects += object_count
total_bytes += bytes_used
expected = HeaderKeyDict({
'X-Account-Container-Count': total_containers,
'X-Account-Object-Count': total_objects,
'X-Account-Bytes-Used': total_bytes,
'X-Timestamp': Timestamp(now).normal,
'X-PUT-Timestamp': Timestamp(now).normal,
})
for policy in POLICIES:
prefix = 'X-Account-Storage-Policy-%s-' % policy.name
expected[prefix + 'Object-Count'] = int(policy)
expected[prefix + 'Bytes-Used'] = int(policy) * 10
resp_headers = utils.get_response_headers(broker)
for key, value in resp_headers.items():
expected_value = expected.pop(key)
self.assertEqual(expected_value, str(value),
'value for %r was %r not %r' % (
key, value, expected_value))
self.assertFalse(expected)