diff --git a/swift/account/backend.py b/swift/account/backend.py index 10cf18c134..595c4969ec 100644 --- a/swift/account/backend.py +++ b/swift/account/backend.py @@ -24,7 +24,7 @@ import errno import sqlite3 -from swift.common.utils import normalize_timestamp, lock_parent_directory +from swift.common.utils import Timestamp, lock_parent_directory from swift.common.db import DatabaseBroker, DatabaseConnectionError, \ PENDING_CAP, PICKLE_PROTOCOL, utf8encode @@ -155,7 +155,7 @@ class AccountBroker(DatabaseBroker): conn.execute(''' UPDATE account_stat SET account = ?, created_at = ?, id = ?, put_timestamp = ?, status_changed_at = ? - ''', (self.account, normalize_timestamp(time.time()), str(uuid4()), + ''', (self.account, Timestamp(time.time()).internal, str(uuid4()), put_timestamp, put_timestamp)) def create_policy_stat_table(self, conn): @@ -293,7 +293,7 @@ class AccountBroker(DatabaseBroker): """ return status == 'DELETED' or ( container_count in (None, '', 0, '0') and - float(delete_timestamp) > float(put_timestamp)) + Timestamp(delete_timestamp) > Timestamp(put_timestamp)) def _is_deleted(self, conn): """ diff --git a/swift/account/reaper.py b/swift/account/reaper.py index c83041182d..2f0bb0f8d7 100644 --- a/swift/account/reaper.py +++ b/swift/account/reaper.py @@ -18,7 +18,7 @@ import random from swift import gettext_ as _ from logging import DEBUG from math import sqrt -from time import time, ctime +from time import time from eventlet import GreenPool, sleep, Timeout @@ -29,7 +29,7 @@ from swift.common.direct_client import direct_delete_container, \ from swift.common.exceptions import ClientException from swift.common.ring import Ring from swift.common.utils import get_logger, whataremyips, ismount, \ - config_true_value + config_true_value, Timestamp from swift.common.daemon import Daemon from swift.common.storage_policy import POLICIES, POLICY_INDEX @@ -229,7 +229,8 @@ class AccountReaper(Daemon): """ begin = time() info = broker.get_info() - if time() - float(info['delete_timestamp']) <= self.delay_reaping: + if time() - float(Timestamp(info['delete_timestamp'])) <= \ + self.delay_reaping: return False account = info['account'] self.logger.info(_('Beginning pass on account %s'), account) @@ -281,10 +282,11 @@ class AccountReaper(Daemon): log += _(', elapsed: %.02fs') % (time() - begin) self.logger.info(log) self.logger.timing_since('timing', self.start_time) + delete_timestamp = Timestamp(info['delete_timestamp']) if self.stats_containers_remaining and \ - begin - float(info['delete_timestamp']) >= self.reap_not_done_after: + begin - float(delete_timestamp) >= self.reap_not_done_after: self.logger.warn(_('Account %s has not been reaped since %s') % - (account, ctime(float(info['delete_timestamp'])))) + (account, delete_timestamp.isoformat)) return True def reap_container(self, account, account_partition, account_nodes, diff --git a/swift/account/server.py b/swift/account/server.py index 6427b6b7dc..58dfd607ad 100644 --- a/swift/account/server.py +++ b/swift/account/server.py @@ -27,9 +27,9 @@ from swift.common.db import DatabaseConnectionError, DatabaseAlreadyExists from swift.common.request_helpers import get_param, get_listing_content_type, \ split_and_validate_path from swift.common.utils import get_logger, hash_path, public, \ - normalize_timestamp, storage_directory, config_true_value, \ + Timestamp, storage_directory, config_true_value, \ json, timing_stats, replication, get_log_line -from swift.common.constraints import check_mount, check_float, check_utf8 +from swift.common.constraints import check_mount, valid_timestamp, check_utf8 from swift.common import constraints from swift.common.db_replicator import ReplicatorRpc from swift.common.swob import HTTPAccepted, HTTPBadRequest, \ @@ -90,14 +90,11 @@ class AccountController(object): drive, part, account = split_and_validate_path(req, 3) if self.mount_check and not check_mount(self.root, drive): return HTTPInsufficientStorage(drive=drive, request=req) - if 'x-timestamp' not in req.headers or \ - not check_float(req.headers['x-timestamp']): - return HTTPBadRequest(body='Missing timestamp', request=req, - content_type='text/plain') + req_timestamp = valid_timestamp(req) broker = self._get_account_broker(drive, part, account) if broker.is_deleted(): return self._deleted_response(broker, req, HTTPNotFound) - broker.delete_db(req.headers['x-timestamp']) + broker.delete_db(req_timestamp.internal) return self._deleted_response(broker, req, HTTPNoContent) @public @@ -108,6 +105,10 @@ class AccountController(object): if self.mount_check and not check_mount(self.root, drive): return HTTPInsufficientStorage(drive=drive, request=req) if container: # put account container + if 'x-timestamp' not in req.headers: + timestamp = Timestamp(time.time()) + else: + timestamp = valid_timestamp(req) pending_timeout = None container_policy_index = req.headers.get(POLICY_INDEX, 0) if 'x-trans-id' in req.headers: @@ -117,8 +118,7 @@ class AccountController(object): if account.startswith(self.auto_create_account_prefix) and \ not os.path.exists(broker.db_file): try: - broker.initialize(normalize_timestamp( - req.headers.get('x-timestamp') or time.time())) + broker.initialize(timestamp.internal) except DatabaseAlreadyExists: pass if req.headers.get('x-account-override-deleted', 'no').lower() != \ @@ -135,11 +135,11 @@ class AccountController(object): else: return HTTPCreated(request=req) else: # put account + timestamp = valid_timestamp(req) broker = self._get_account_broker(drive, part, account) - timestamp = normalize_timestamp(req.headers['x-timestamp']) if not os.path.exists(broker.db_file): try: - broker.initialize(timestamp) + broker.initialize(timestamp.internal) created = True except DatabaseAlreadyExists: created = False @@ -148,11 +148,11 @@ class AccountController(object): body='Recently deleted') else: created = broker.is_deleted() - broker.update_put_timestamp(timestamp) + broker.update_put_timestamp(timestamp.internal) if broker.is_deleted(): return HTTPConflict(request=req) metadata = {} - metadata.update((key, (value, timestamp)) + metadata.update((key, (value, timestamp.internal)) for key, value in req.headers.iteritems() if is_sys_or_user_meta('account', key)) if metadata: @@ -238,19 +238,14 @@ class AccountController(object): def POST(self, req): """Handle HTTP POST request.""" drive, part, account = split_and_validate_path(req, 3) - if 'x-timestamp' not in req.headers or \ - not check_float(req.headers['x-timestamp']): - return HTTPBadRequest(body='Missing or bad timestamp', - request=req, - content_type='text/plain') + req_timestamp = valid_timestamp(req) if self.mount_check and not check_mount(self.root, drive): return HTTPInsufficientStorage(drive=drive, request=req) broker = self._get_account_broker(drive, part, account) if broker.is_deleted(): return self._deleted_response(broker, req, HTTPNotFound) - timestamp = normalize_timestamp(req.headers['x-timestamp']) metadata = {} - metadata.update((key, (value, timestamp)) + metadata.update((key, (value, req_timestamp.internal)) for key, value in req.headers.iteritems() if is_sys_or_user_meta('account', key)) if metadata: diff --git a/swift/account/utils.py b/swift/account/utils.py index 13dec505cc..6cc8700961 100644 --- a/swift/account/utils.py +++ b/swift/account/utils.py @@ -17,7 +17,7 @@ import time from xml.sax import saxutils from swift.common.swob import HTTPOk, HTTPNoContent -from swift.common.utils import json, normalize_timestamp +from swift.common.utils import json, Timestamp from swift.common.storage_policy import POLICIES @@ -27,7 +27,7 @@ class FakeAccountBroker(object): like an account broker would for a real, empty account with no metadata. """ def get_info(self): - now = normalize_timestamp(time.time()) + now = Timestamp(time.time()).internal return {'container_count': 0, 'object_count': 0, 'bytes_used': 0, @@ -51,8 +51,8 @@ def get_response_headers(broker): 'X-Account-Container-Count': info['container_count'], 'X-Account-Object-Count': info['object_count'], 'X-Account-Bytes-Used': info['bytes_used'], - 'X-Timestamp': info['created_at'], - 'X-PUT-Timestamp': info['put_timestamp']} + 'X-Timestamp': Timestamp(info['created_at']).normal, + 'X-PUT-Timestamp': Timestamp(info['put_timestamp']).normal} policy_stats = broker.get_policy_stats() for policy_idx, stats in policy_stats.items(): policy = POLICIES.get_by_index(policy_idx) diff --git a/swift/cli/info.py b/swift/cli/info.py index fd0fa53206..7d1557c08e 100644 --- a/swift/cli/info.py +++ b/swift/cli/info.py @@ -14,10 +14,10 @@ import itertools import os import sqlite3 import urllib -from datetime import datetime from hashlib import md5 -from swift.common.utils import hash_path, storage_directory +from swift.common.utils import hash_path, storage_directory, \ + Timestamp from swift.common.ring import Ring from swift.common.request_helpers import is_sys_meta, is_user_meta, \ strip_sys_meta_prefix, strip_user_meta_prefix @@ -174,16 +174,16 @@ def print_db_info_metadata(db_type, info, metadata): print 'Metadata:' print (' Created at: %s (%s)' % - (datetime.utcfromtimestamp(float(info['created_at'])), + (Timestamp(info['created_at']).isoformat, info['created_at'])) print (' Put Timestamp: %s (%s)' % - (datetime.utcfromtimestamp(float(info['put_timestamp'])), + (Timestamp(info['put_timestamp']).isoformat, info['put_timestamp'])) print (' Delete Timestamp: %s (%s)' % - (datetime.utcfromtimestamp(float(info['delete_timestamp'])), + (Timestamp(info['delete_timestamp']).isoformat, info['delete_timestamp'])) print (' Status Timestamp: %s (%s)' % - (datetime.utcfromtimestamp(float(info['status_changed_at'])), + (Timestamp(info['status_changed_at']).isoformat, info['status_changed_at'])) if db_type == 'account': print ' Container Count: %s' % info['container_count'] @@ -197,12 +197,10 @@ def print_db_info_metadata(db_type, info, metadata): print (' Storage Policy: %s (%s)' % ( policy_name, info['storage_policy_index'])) print (' Reported Put Timestamp: %s (%s)' % - (datetime.utcfromtimestamp( - float(info['reported_put_timestamp'])), + (Timestamp(info['reported_put_timestamp']).isoformat, info['reported_put_timestamp'])) print (' Reported Delete Timestamp: %s (%s)' % - (datetime.utcfromtimestamp - (float(info['reported_delete_timestamp'])), + (Timestamp(info['reported_delete_timestamp']).isoformat, info['reported_delete_timestamp'])) print ' Reported Object Count: %s' % info['reported_object_count'] print ' Reported Bytes Used: %s' % info['reported_bytes_used'] @@ -255,7 +253,7 @@ def print_obj_metadata(metadata): raise ValueError('Metadata is None') path = metadata.pop('name', '') content_type = metadata.pop('Content-Type', '') - ts = metadata.pop('X-Timestamp', 0) + ts = Timestamp(metadata.pop('X-Timestamp', 0)) account = container = obj = obj_hash = None if path: try: @@ -276,8 +274,7 @@ def print_obj_metadata(metadata): else: print 'Content-Type: Not found in metadata' if ts: - print ('Timestamp: %s (%s)' % - (datetime.utcfromtimestamp(float(ts)), ts)) + print ('Timestamp: %s (%s)' % (ts.isoformat, ts.internal)) else: print 'Timestamp: Not found in metadata' diff --git a/swift/common/constraints.py b/swift/common/constraints.py index d3cbea0863..24dcbf034c 100644 --- a/swift/common/constraints.py +++ b/swift/common/constraints.py @@ -18,7 +18,7 @@ import urllib from urllib import unquote from ConfigParser import ConfigParser, NoSectionError, NoOptionError -from swift.common import utils +from swift.common import utils, exceptions from swift.common.swob import HTTPBadRequest, HTTPLengthRequired, \ HTTPRequestEntityTooLarge, HTTPPreconditionFailed @@ -209,6 +209,22 @@ def check_float(string): return False +def valid_timestamp(request): + """ + Helper function to extract a timestamp from requests that require one. + + :param request: the swob request object + + :returns: a valid Timestamp instance + :raises: HTTPBadRequest on missing or invalid X-Timestamp + """ + try: + return request.timestamp + except exceptions.InvalidTimestamp as e: + raise HTTPBadRequest(body=str(e), request=request, + content_type='text/plain') + + def check_utf8(string): """ Validate if a string is valid UTF-8 str or unicode and that it diff --git a/swift/common/db.py b/swift/common/db.py index 10fcc1be9f..9e654e5058 100644 --- a/swift/common/db.py +++ b/swift/common/db.py @@ -29,7 +29,7 @@ from tempfile import mkstemp from eventlet import sleep, Timeout import sqlite3 -from swift.common.utils import json, normalize_timestamp, renamer, \ +from swift.common.utils import json, Timestamp, renamer, \ mkdirs, lock_parent_directory, fallocate from swift.common.exceptions import LockTimeout @@ -144,7 +144,7 @@ def chexor(old, name, timestamp): :param old: hex representation of the current DB hash :param name: name of the object or container being inserted - :param timestamp: timestamp of the new record + :param timestamp: internalized timestamp of the new record :returns: a hex representation of the new hash value """ if name is None: @@ -222,7 +222,7 @@ class DatabaseBroker(object): The storage_policy_index is passed through to the subclass's ``_initialize`` method. It is ignored by ``AccountBroker``. - :param put_timestamp: timestamp of initial PUT request + :param put_timestamp: internalized timestamp of initial PUT request :param storage_policy_index: only required for containers """ if self.db_file == ':memory:': @@ -280,7 +280,7 @@ class DatabaseBroker(object): END; """) if not put_timestamp: - put_timestamp = normalize_timestamp(0) + put_timestamp = Timestamp(0).internal self._initialize(conn, put_timestamp, storage_policy_index=storage_policy_index) conn.commit() @@ -302,9 +302,8 @@ class DatabaseBroker(object): """ Mark the DB as deleted - :param timestamp: delete timestamp + :param timestamp: internalized delete timestamp """ - timestamp = normalize_timestamp(timestamp) # first, clear the metadata cleared_meta = {} for k in self.metadata: @@ -463,8 +462,8 @@ class DatabaseBroker(object): delete_timestamp=MAX(?, delete_timestamp) ''' % self.db_type, (created_at, put_timestamp, delete_timestamp)) if old_status != self._is_deleted(conn): - timestamp = normalize_timestamp(time.time()) - self._update_status_changed_at(conn, timestamp) + timestamp = Timestamp(time.time()) + self._update_status_changed_at(conn, timestamp.internal) conn.commit() @@ -790,7 +789,7 @@ class DatabaseBroker(object): Update the put_timestamp. Only modifies it if it is greater than the current timestamp. - :param timestamp: put timestamp + :param timestamp: internalized put timestamp """ with self.get() as conn: conn.execute( @@ -804,6 +803,8 @@ class DatabaseBroker(object): Update the status_changed_at field in the stat table. Only modifies status_changed_at if the timestamp is greater than the current status_changed_at timestamp. + + :param timestamp: internalized timestamp """ with self.get() as conn: self._update_status_changed_at(conn, timestamp) diff --git a/swift/common/db_replicator.py b/swift/common/db_replicator.py index 952130ae8e..c03a13790a 100644 --- a/swift/common/db_replicator.py +++ b/swift/common/db_replicator.py @@ -31,7 +31,7 @@ import swift.common.db from swift.common.direct_client import quote from swift.common.utils import get_logger, whataremyips, storage_directory, \ renamer, mkdirs, lock_parent_directory, config_true_value, \ - unlink_older_than, dump_recon_cache, rsync_ip, ismount, json + unlink_older_than, dump_recon_cache, rsync_ip, ismount, json, Timestamp from swift.common import ring from swift.common.http import HTTP_NOT_FOUND, HTTP_INSUFFICIENT_STORAGE from swift.common.bufferedhttp import BufferedHTTPConnection @@ -458,16 +458,8 @@ class Replicator(Daemon): return # The db is considered deleted if the delete_timestamp value is greater # than the put_timestamp, and there are no objects. - delete_timestamp = 0 - try: - delete_timestamp = float(info['delete_timestamp']) - except ValueError: - pass - put_timestamp = 0 - try: - put_timestamp = float(info['put_timestamp']) - except ValueError: - pass + delete_timestamp = Timestamp(info.get('delete_timestamp') or 0) + put_timestamp = Timestamp(info.get('put_timestamp') or 0) if delete_timestamp < (now - self.reclaim_age) and \ delete_timestamp > put_timestamp and \ info['count'] in (None, '', 0, '0'): diff --git a/swift/common/direct_client.py b/swift/common/direct_client.py index 6a6efdea59..3fe2c236ae 100644 --- a/swift/common/direct_client.py +++ b/swift/common/direct_client.py @@ -27,7 +27,7 @@ from eventlet import sleep, Timeout from swift.common.bufferedhttp import http_connect from swift.common.exceptions import ClientException -from swift.common.utils import normalize_timestamp, FileLikeIter +from swift.common.utils import Timestamp, FileLikeIter from swift.common.http import HTTP_NO_CONTENT, HTTP_INSUFFICIENT_STORAGE, \ is_success, is_server_error from swift.common.swob import HeaderKeyDict @@ -91,7 +91,7 @@ def _get_direct_account_container(path, stype, node, part, def gen_headers(hdrs_in=None, add_ts=False): hdrs_out = HeaderKeyDict(hdrs_in) if hdrs_in else HeaderKeyDict() if add_ts: - hdrs_out['X-Timestamp'] = normalize_timestamp(time()) + hdrs_out['X-Timestamp'] = Timestamp(time()).internal hdrs_out['User-Agent'] = 'direct-client %s' % os.getpid() return hdrs_out diff --git a/swift/common/exceptions.py b/swift/common/exceptions.py index a3e6cba0f7..e7999bab97 100644 --- a/swift/common/exceptions.py +++ b/swift/common/exceptions.py @@ -14,6 +14,7 @@ # limitations under the License. from eventlet import Timeout +import swift.common.utils class MessageTimeout(Timeout): @@ -30,6 +31,10 @@ class SwiftException(Exception): pass +class InvalidTimestamp(SwiftException): + pass + + class DiskFileError(SwiftException): pass @@ -54,7 +59,8 @@ class DiskFileDeleted(DiskFileNotExist): def __init__(self, metadata=None): self.metadata = metadata or {} - self.timestamp = self.metadata.get('X-Timestamp', 0) + self.timestamp = swift.common.utils.Timestamp( + self.metadata.get('X-Timestamp', 0)) class DiskFileExpired(DiskFileDeleted): diff --git a/swift/common/swob.py b/swift/common/swob.py index 69f4f2f694..c83856f953 100644 --- a/swift/common/swob.py +++ b/swift/common/swob.py @@ -49,7 +49,8 @@ import random import functools import inspect -from swift.common.utils import reiterate, split_path +from swift.common.utils import reiterate, split_path, Timestamp +from swift.common.exceptions import InvalidTimestamp RESPONSE_REASONS = { @@ -762,6 +763,7 @@ class Request(object): body = _req_body_property() charset = None _params_cache = None + _timestamp = None acl = _req_environ_property('swob.ACL') def __init__(self, environ): @@ -843,6 +845,22 @@ class Request(object): return self._params_cache str_params = params + @property + def timestamp(self): + """ + Provides HTTP_X_TIMESTAMP as a :class:`~swift.common.utils.Timestamp` + """ + if self._timestamp is None: + try: + raw_timestamp = self.environ['HTTP_X_TIMESTAMP'] + except KeyError: + raise InvalidTimestamp('Missing X-Timestamp header') + try: + self._timestamp = Timestamp(raw_timestamp) + except ValueError: + raise InvalidTimestamp('Invalid X-Timestamp header') + return self._timestamp + @property def path_qs(self): """The path of the request, without host but with query string.""" diff --git a/swift/common/utils.py b/swift/common/utils.py index da68cb1303..a5badaab9b 100644 --- a/swift/common/utils.py +++ b/swift/common/utils.py @@ -63,7 +63,7 @@ utf8_decoder = codecs.getdecoder('utf-8') utf8_encoder = codecs.getencoder('utf-8') from swift import gettext_ as _ -from swift.common.exceptions import LockTimeout, MessageTimeout +import swift.common.exceptions from swift.common.http import is_success, is_redirection, HTTP_NOT_FOUND # logging doesn't import patched as cleanly as one would like @@ -562,6 +562,120 @@ def drop_buffer_cache(fd, offset, length): 'length': length, 'ret': ret}) +NORMAL_FORMAT = "%016.05f" +INTERNAL_FORMAT = NORMAL_FORMAT + '_%016x' +# Setting this to True will cause the internal format to always display +# extended digits - even when the value is equivalent to the normalized form. +# This isn't ideal during an upgrade when some servers might not understand +# the new time format - but flipping it to True works great for testing. +FORCE_INTERNAL = False # or True + + +class Timestamp(object): + """ + Internal Representation of Swift Time. + + The normalized form of the X-Timestamp header looks like a float + with a fixed width to ensure stable string sorting - normalized + timestamps look like "1402464677.04188" + + To support overwrites of existing data without modifying the original + timestamp but still maintain consistency a second internal offset vector + is append to the normalized timestamp form which compares and sorts + greater than the fixed width float format but less than a newer timestamp. + The internalized format of timestamps looks like + "1402464677.04188_0000000000000000" - the portion after the underscore is + the offset and is a formatted hexadecimal integer. + + The internalized form is not exposed to clients in responses from + Swift. Normal client operations will not create a timestamp with an + offset. + + The Timestamp class in common.utils supports internalized and + normalized formatting of timestamps and also comparison of timestamp + values. When the offset value of a Timestamp is 0 - it's considered + insignificant and need not be represented in the string format; to + support backwards compatibility during a Swift upgrade the + internalized and normalized form of a Timestamp with an + insignificant offset are identical. When a timestamp includes an + offset it will always be represented in the internalized form, but + is still excluded from the normalized form. Timestamps with an + equivalent timestamp portion (the float part) will compare and order + by their offset. Timestamps with a greater timestamp portion will + always compare and order greater than a Timestamp with a lesser + timestamp regardless of it's offset. String comparison and ordering + is guaranteed for the internalized string format, and is backwards + compatible for normalized timestamps which do not include an offset. + """ + + def __init__(self, timestamp, offset=0): + if isinstance(timestamp, basestring): + parts = timestamp.split('_', 1) + self.timestamp = float(parts.pop(0)) + if parts: + self.offset = int(parts[0], 16) + else: + self.offset = 0 + else: + self.timestamp = float(timestamp) + self.offset = getattr(timestamp, 'offset', 0) + # increment offset + if offset >= 0: + self.offset += offset + else: + raise ValueError('offset must be non-negative') + + def __repr__(self): + return INTERNAL_FORMAT % (self.timestamp, self.offset) + + def __str__(self): + raise TypeError('You must specificy which string format is required') + + def __float__(self): + return self.timestamp + + def __int__(self): + return int(self.timestamp) + + def __nonzero__(self): + return bool(self.timestamp or self.offset) + + @property + def normal(self): + return NORMAL_FORMAT % self.timestamp + + @property + def internal(self): + if self.offset or FORCE_INTERNAL: + return INTERNAL_FORMAT % (self.timestamp, self.offset) + else: + return self.normal + + @property + def isoformat(self): + isoformat = datetime.datetime.utcfromtimestamp( + float(self.normal)).isoformat() + # python isoformat() doesn't include msecs when zero + if len(isoformat) < len("1970-01-01T00:00:00.000000"): + isoformat += ".000000" + return isoformat + + def __eq__(self, other): + if not isinstance(other, Timestamp): + other = Timestamp(other) + return self.internal == other.internal + + def __ne__(self, other): + if not isinstance(other, Timestamp): + other = Timestamp(other) + return self.internal != other.internal + + def __cmp__(self, other): + if not isinstance(other, Timestamp): + other = Timestamp(other) + return cmp(self.internal, other.internal) + + def normalize_timestamp(timestamp): """ Format a timestamp (string or numeric) into a standardized @@ -574,15 +688,15 @@ def normalize_timestamp(timestamp): :param timestamp: unix timestamp :returns: normalized timestamp as a string """ - return "%016.05f" % (float(timestamp)) + return Timestamp(timestamp).normal def last_modified_date_to_timestamp(last_modified_date_str): """ - Convert a last modified date (liked you'd get from a container listing, + Convert a last modified date (like you'd get from a container listing, e.g. 2014-02-28T23:22:36.698390) to a float. """ - return float( + return Timestamp( datetime.datetime.strptime( last_modified_date_str, '%Y-%m-%dT%H:%M:%S.%f' ).strftime('%s.%f') @@ -1007,7 +1121,7 @@ class LogAdapter(logging.LoggerAdapter, object): emsg = exc.__class__.__name__ if hasattr(exc, 'seconds'): emsg += ' (%ss)' % exc.seconds - if isinstance(exc, MessageTimeout): + if isinstance(exc, swift.common.exceptions.MessageTimeout): if exc.msg: emsg += ' %s' % exc.msg else: @@ -1441,7 +1555,7 @@ def hash_path(account, container=None, object=None, raw_digest=False): @contextmanager -def lock_path(directory, timeout=10, timeout_class=LockTimeout): +def lock_path(directory, timeout=10, timeout_class=None): """ Context manager that acquires a lock on a directory. This will block until the lock can be acquired, or the timeout time has expired (whichever occurs @@ -1458,6 +1572,8 @@ def lock_path(directory, timeout=10, timeout_class=LockTimeout): constructed as timeout_class(timeout, lockpath). Default: LockTimeout """ + if timeout_class is None: + timeout_class = swift.common.exceptions.LockTimeout mkdirs(directory) lockpath = '%s/.lock' % directory fd = os.open(lockpath, os.O_WRONLY | os.O_CREAT) @@ -1497,7 +1613,7 @@ def lock_file(filename, timeout=10, append=False, unlink=True): fd = os.open(filename, flags) file_obj = os.fdopen(fd, mode) try: - with LockTimeout(timeout, filename): + with swift.common.exceptions.LockTimeout(timeout, filename): while True: try: fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) diff --git a/swift/container/backend.py b/swift/container/backend.py index 6a2f92bebd..ff5129f142 100644 --- a/swift/container/backend.py +++ b/swift/container/backend.py @@ -24,7 +24,7 @@ import errno import sqlite3 -from swift.common.utils import normalize_timestamp, lock_parent_directory +from swift.common.utils import Timestamp, lock_parent_directory from swift.common.db import DatabaseBroker, DatabaseConnectionError, \ PENDING_CAP, PICKLE_PROTOCOL, utf8encode @@ -202,7 +202,7 @@ class ContainerBroker(DatabaseBroker): :param storage_policy_index: storage policy index """ if put_timestamp is None: - put_timestamp = normalize_timestamp(0) + put_timestamp = Timestamp(0).internal # The container_stat view is for compatibility; old versions of Swift # expected a container_stat table with columns "object_count" and # "bytes_used", but when that stuff became per-storage-policy and @@ -224,7 +224,7 @@ class ContainerBroker(DatabaseBroker): INSERT INTO container_info (account, container, created_at, id, put_timestamp, status_changed_at, storage_policy_index) VALUES (?, ?, ?, ?, ?, ?, ?); - """, (self.account, self.container, normalize_timestamp(time.time()), + """, (self.account, self.container, Timestamp(time.time()).internal, str(uuid4()), put_timestamp, put_timestamp, storage_policy_index)) @@ -372,7 +372,7 @@ class ContainerBroker(DatabaseBroker): # value is greater than the put_timestamp, and there are no # objects in the container. return (object_count in (None, '', 0, '0')) and ( - float(delete_timestamp) > float(put_timestamp)) + Timestamp(delete_timestamp) > Timestamp(put_timestamp)) def _is_deleted(self, conn): """ @@ -524,7 +524,7 @@ class ContainerBroker(DatabaseBroker): Update the container_stat policy_index and status_changed_at. """ if timestamp is None: - timestamp = normalize_timestamp(time.time()) + timestamp = Timestamp(time.time()).internal def _setit(conn): conn.execute(''' diff --git a/swift/container/reconciler.py b/swift/container/reconciler.py index 6231a83d35..7bde84147b 100644 --- a/swift/container/reconciler.py +++ b/swift/container/reconciler.py @@ -27,7 +27,7 @@ from swift.common.direct_client import ( from swift.common.internal_client import InternalClient, UnexpectedResponse from swift.common.storage_policy import POLICY_INDEX from swift.common.utils import get_logger, split_path, quorum_size, \ - FileLikeIter, normalize_timestamp, last_modified_date_to_timestamp, \ + FileLikeIter, Timestamp, last_modified_date_to_timestamp, \ LRUCache @@ -68,7 +68,7 @@ def cmp_policy_info(info, remote_info): def has_been_recreated(info): return (info['put_timestamp'] > info['delete_timestamp'] > - normalize_timestamp(0)) + Timestamp(0)) remote_recreated = has_been_recreated(remote_info) recreated = has_been_recreated(info) @@ -98,7 +98,7 @@ def incorrect_policy_index(info, remote_info): def translate_container_headers_to_info(headers): - default_timestamp = normalize_timestamp(0) + default_timestamp = Timestamp(0).internal return { 'storage_policy_index': int(headers[POLICY_INDEX]), 'put_timestamp': headers.get('x-backend-put-timestamp', @@ -117,7 +117,7 @@ def best_policy_index(headers): def get_reconciler_container_name(obj_timestamp): - return str(int(float(obj_timestamp)) // + return str(int(Timestamp(obj_timestamp)) // MISPLACED_OBJECTS_CONTAINER_DIVISOR * MISPLACED_OBJECTS_CONTAINER_DIVISOR) @@ -195,7 +195,7 @@ def add_to_reconciler_queue(container_ring, account, container, obj, # already been popped from the queue to be reprocessed, but # could potentially prevent out of order updates from making it # into the queue - x_timestamp = normalize_timestamp(time.time()) + x_timestamp = Timestamp(time.time()).internal else: x_timestamp = obj_timestamp q_op_type = get_reconciler_content_type(op) @@ -230,10 +230,7 @@ def add_to_reconciler_queue(container_ring, account, container, obj, def slightly_later_timestamp(ts, offset=1): - # I'm guessing to avoid rounding errors Swift uses a 10-microsecond - # resolution instead of Python's 1-microsecond resolution. - offset *= 0.00001 - return normalize_timestamp(float(ts) + offset) + return Timestamp(ts, offset=offset).internal def parse_raw_obj(obj_info): @@ -266,7 +263,7 @@ def parse_raw_obj(obj_info): 'container': container, 'obj': obj, 'q_op': q_op, - 'q_ts': float(obj_info['hash']), + 'q_ts': Timestamp(obj_info['hash']), 'q_record': last_modified_date_to_timestamp( obj_info['last_modified']), 'path': '/%s/%s/%s' % (account, container, obj) @@ -407,7 +404,7 @@ class ContainerReconciler(Daemon): success = False try: self.swift.delete_object(account, container, obj, - acceptable_statuses=(2, 4), + acceptable_statuses=(2, 404), headers=headers) except UnexpectedResponse as err: self.stats_log('cleanup_failed', '%r (%f) was not cleaned up ' @@ -430,7 +427,7 @@ class ContainerReconciler(Daemon): :param obj: the object name :param q_policy_index: the policy index of the source indicated by the queue entry. - :param q_ts: a float, the timestamp of the misplaced object + :param q_ts: the timestamp of the misplaced object :param q_op: the operation of the misplaced request :param path: the full path of the misplaced object for logging @@ -459,12 +456,7 @@ class ContainerReconciler(Daemon): dest_obj = self.swift.get_object_metadata(account, container, obj, headers=headers, acceptable_statuses=(2, 4)) - dest_ts = float( - dest_obj.get('x-timestamp', - dest_obj.get('x-backend-timestamp', - '0.0') - ) - ) + dest_ts = Timestamp(dest_obj.get('x-backend-timestamp', 0)) if dest_ts >= q_ts: self.stats_log('found_object', '%r (%f) in policy_index %s ' 'is newer than queue (%f)', path, dest_ts, @@ -492,7 +484,7 @@ class ContainerReconciler(Daemon): source_obj_info = {} source_obj_iter = None - source_ts = float(source_obj_info.get("X-Timestamp", 0)) + source_ts = Timestamp(source_obj_info.get('x-backend-timestamp', 0)) if source_obj_status == 404 and q_op == 'DELETE': return self.ensure_tombstone_in_right_location( q_policy_index, account, container, obj, q_ts, path, @@ -516,10 +508,10 @@ class ContainerReconciler(Daemon): :param account: the account name of the misplaced object :param container: the container name of the misplaced object :param obj: the name of the misplaced object - :param q_ts: a float, the timestamp of the misplaced object + :param q_ts: the timestamp of the misplaced object :param path: the full path of the misplaced object for logging :param container_policy_index: the policy_index of the destination - :param source_ts: a float, the timestamp of the source object + :param source_ts: the timestamp of the source object :param source_obj_status: the HTTP status source object request :param source_obj_info: the HTTP headers of the source object request :param source_obj_iter: the body iter of the source object request @@ -527,21 +519,22 @@ class ContainerReconciler(Daemon): if source_obj_status // 100 != 2 or source_ts < q_ts: if q_ts < time.time() - self.reclaim_age: # it's old and there are no tombstones or anything; give up - self.stats_log('lost_source', '%r (%f) was not available in ' - 'policy_index %s and has expired', path, q_ts, - q_policy_index, level=logging.CRITICAL) + self.stats_log('lost_source', '%r (%s) was not available in ' + 'policy_index %s and has expired', path, + q_ts.internal, q_policy_index, + level=logging.CRITICAL) return True # the source object is unavailable or older than the queue # entry; a version that will satisfy the queue entry hopefully # exists somewhere in the cluster, so wait and try again - self.stats_log('unavailable_source', '%r (%f) in ' - 'policy_index %s responded %s (%f)', path, - q_ts, q_policy_index, source_obj_status, - source_ts, level=logging.WARNING) + self.stats_log('unavailable_source', '%r (%s) in ' + 'policy_index %s responded %s (%s)', path, + q_ts.internal, q_policy_index, source_obj_status, + source_ts.internal, level=logging.WARNING) return False # optimistically move any source with a timestamp >= q_ts - ts = max(float(source_ts), q_ts) + ts = max(Timestamp(source_ts), q_ts) # move the object put_timestamp = slightly_later_timestamp(ts, offset=2) self.stats_log('copy_attempt', '%r (%f) in policy_index %s will be ' @@ -638,7 +631,7 @@ class ContainerReconciler(Daemon): success = self._reconcile_object(**info) except: # noqa self.logger.exception('Unhandled Exception trying to ' - 'reconcile %r (%s) in policy_index %s', + 'reconcile %r (%f) in policy_index %s', info['path'], info['q_ts'], info['q_policy_index']) if success: diff --git a/swift/container/replicator.py b/swift/container/replicator.py index d60b53c305..aed6042637 100644 --- a/swift/container/replicator.py +++ b/swift/container/replicator.py @@ -28,7 +28,7 @@ from swift.common.storage_policy import POLICIES from swift.common.exceptions import DeviceUnavailable from swift.common.http import is_success from swift.common.db import DatabaseAlreadyExists -from swift.common.utils import (json, normalize_timestamp, hash_path, +from swift.common.utils import (json, Timestamp, hash_path, storage_directory, quorum_size) @@ -59,10 +59,10 @@ class ContainerReplicator(db_replicator.Replicator): if is_success(response.status): remote_info = json.loads(response.data) if incorrect_policy_index(info, remote_info): - status_changed_at = normalize_timestamp(time.time()) + status_changed_at = Timestamp(time.time()) broker.set_storage_policy_index( remote_info['storage_policy_index'], - timestamp=status_changed_at) + timestamp=status_changed_at.internal) broker.merge_timestamps(*(remote_info[key] for key in ( 'created_at', 'put_timestamp', 'delete_timestamp'))) rv = parent._handle_sync_response( @@ -256,7 +256,7 @@ class ContainerReplicatorRpc(db_replicator.ReplicatorRpc): """ info = broker.get_replication_info() if incorrect_policy_index(info, remote_info): - status_changed_at = normalize_timestamp(time.time()) + status_changed_at = Timestamp(time.time()).internal broker.set_storage_policy_index( remote_info['storage_policy_index'], timestamp=status_changed_at) diff --git a/swift/container/server.py b/swift/container/server.py index e028ddf9ef..74c0bb2821 100644 --- a/swift/container/server.py +++ b/swift/container/server.py @@ -16,7 +16,6 @@ import os import time import traceback -from datetime import datetime from swift import gettext_ as _ from xml.etree.cElementTree import Element, SubElement, tostring @@ -30,10 +29,10 @@ from swift.common.container_sync_realms import ContainerSyncRealms from swift.common.request_helpers import get_param, get_listing_content_type, \ split_and_validate_path, is_sys_or_user_meta from swift.common.utils import get_logger, hash_path, public, \ - normalize_timestamp, storage_directory, validate_sync_to, \ + Timestamp, storage_directory, validate_sync_to, \ config_true_value, json, timing_stats, replication, \ override_bytes_from_content_type, get_log_line -from swift.common.constraints import check_mount, check_float, check_utf8 +from swift.common.constraints import check_mount, valid_timestamp, check_utf8 from swift.common import constraints from swift.common.bufferedhttp import http_connect from swift.common.exceptions import ConnectionTimeout @@ -51,13 +50,13 @@ def gen_resp_headers(info, is_deleted=False): """ # backend headers are always included headers = { - 'X-Backend-Timestamp': normalize_timestamp(info.get('created_at', 0)), - 'X-Backend-PUT-Timestamp': normalize_timestamp( - info.get('put_timestamp', 0)), - 'X-Backend-DELETE-Timestamp': normalize_timestamp( - info.get('delete_timestamp', 0)), - 'X-Backend-Status-Changed-At': normalize_timestamp( - info.get('status_changed_at', 0)), + 'X-Backend-Timestamp': Timestamp(info.get('created_at', 0)).internal, + 'X-Backend-PUT-Timestamp': Timestamp(info.get( + 'put_timestamp', 0)).internal, + 'X-Backend-DELETE-Timestamp': Timestamp( + info.get('delete_timestamp', 0)).internal, + 'X-Backend-Status-Changed-At': Timestamp( + info.get('status_changed_at', 0)).internal, POLICY_INDEX: info.get('storage_policy_index', 0), } if not is_deleted: @@ -65,9 +64,9 @@ def gen_resp_headers(info, is_deleted=False): headers.update({ 'X-Container-Object-Count': info.get('object_count', 0), 'X-Container-Bytes-Used': info.get('bytes_used', 0), - 'X-Timestamp': normalize_timestamp(info.get('created_at', 0)), - 'X-PUT-Timestamp': normalize_timestamp( - info.get('put_timestamp', 0)), + 'X-Timestamp': Timestamp(info.get('created_at', 0)).normal, + 'X-PUT-Timestamp': Timestamp( + info.get('put_timestamp', 0)).normal, }) return headers @@ -245,10 +244,7 @@ class ContainerController(object): """Handle HTTP DELETE request.""" drive, part, account, container, obj = split_and_validate_path( req, 4, 5, True) - if 'x-timestamp' not in req.headers or \ - not check_float(req.headers['x-timestamp']): - return HTTPBadRequest(body='Missing timestamp', request=req, - content_type='text/plain') + req_timestamp = valid_timestamp(req) if self.mount_check and not check_mount(self.root, drive): return HTTPInsufficientStorage(drive=drive, request=req) # policy index is only relevant for delete_obj (and transitively for @@ -258,10 +254,7 @@ class ContainerController(object): if account.startswith(self.auto_create_account_prefix) and obj and \ not os.path.exists(broker.db_file): try: - broker.initialize( - normalize_timestamp( - req.headers.get('x-timestamp') or time.time()), - obj_policy_index) + broker.initialize(req_timestamp.internal, obj_policy_index) except DatabaseAlreadyExists: pass if not os.path.exists(broker.db_file): @@ -274,9 +267,9 @@ class ContainerController(object): # delete container if not broker.empty(): return HTTPConflict(request=req) - existed = float(broker.get_info()['put_timestamp']) and \ + existed = Timestamp(broker.get_info()['put_timestamp']) and \ not broker.is_deleted() - broker.delete_db(req.headers['X-Timestamp']) + broker.delete_db(req_timestamp.internal) if not broker.is_deleted(): return HTTPConflict(request=req) resp = self.account_update(req, account, container, broker) @@ -298,6 +291,7 @@ class ContainerController(object): when creating the container :param requested_policy_index: the storage policy index sent in the request, may be None + :returns: created, a bool, if database did not previously exist """ if not os.path.exists(broker.db_file): @@ -329,10 +323,7 @@ class ContainerController(object): """Handle HTTP PUT request.""" drive, part, account, container, obj = split_and_validate_path( req, 4, 5, True) - if 'x-timestamp' not in req.headers or \ - not check_float(req.headers['x-timestamp']): - return HTTPBadRequest(body='Missing timestamp', request=req, - content_type='text/plain') + req_timestamp = valid_timestamp(req) if 'x-container-sync-to' in req.headers: err, sync_to, realm, realm_key = validate_sync_to( req.headers['x-container-sync-to'], self.allowed_sync_hosts, @@ -342,7 +333,6 @@ class ContainerController(object): if self.mount_check and not check_mount(self.root, drive): return HTTPInsufficientStorage(drive=drive, request=req) requested_policy_index = self.get_and_validate_policy_index(req) - timestamp = normalize_timestamp(req.headers['x-timestamp']) broker = self._get_container_broker(drive, part, account, container) if obj: # put container object # obj put expects the policy_index header, default is for @@ -351,12 +341,12 @@ class ContainerController(object): if account.startswith(self.auto_create_account_prefix) and \ not os.path.exists(broker.db_file): try: - broker.initialize(timestamp, obj_policy_index) + broker.initialize(req_timestamp.internal, obj_policy_index) except DatabaseAlreadyExists: pass if not os.path.exists(broker.db_file): return HTTPNotFound() - broker.put_object(obj, timestamp, + broker.put_object(obj, req_timestamp.internal, int(req.headers['x-size']), req.headers['x-content-type'], req.headers['x-etag'], 0, @@ -370,12 +360,12 @@ class ContainerController(object): else: new_container_policy = requested_policy_index created = self._update_or_create(req, broker, - timestamp, + req_timestamp.internal, new_container_policy, requested_policy_index) metadata = {} metadata.update( - (key, (value, timestamp)) + (key, (value, req_timestamp.internal)) for key, value in req.headers.iteritems() if key.lower() in self.save_headers or is_sys_or_user_meta('container', key)) @@ -433,11 +423,7 @@ class ContainerController(object): return {'subdir': name} response = {'bytes': size, 'hash': etag, 'name': name, 'content_type': content_type} - last_modified = datetime.utcfromtimestamp(float(created)).isoformat() - # python isoformat() doesn't include msecs when zero - if len(last_modified) < len("1970-01-01T00:00:00.000000"): - last_modified += ".000000" - response['last_modified'] = last_modified + response['last_modified'] = Timestamp(created).isoformat override_bytes_from_content_type(response, logger=self.logger) return response @@ -541,10 +527,7 @@ class ContainerController(object): def POST(self, req): """Handle HTTP POST request.""" drive, part, account, container = split_and_validate_path(req, 4) - if 'x-timestamp' not in req.headers or \ - not check_float(req.headers['x-timestamp']): - return HTTPBadRequest(body='Missing or bad timestamp', - request=req, content_type='text/plain') + req_timestamp = valid_timestamp(req) if 'x-container-sync-to' in req.headers: err, sync_to, realm, realm_key = validate_sync_to( req.headers['x-container-sync-to'], self.allowed_sync_hosts, @@ -556,10 +539,10 @@ class ContainerController(object): broker = self._get_container_broker(drive, part, account, container) if broker.is_deleted(): return HTTPNotFound(request=req) - timestamp = normalize_timestamp(req.headers['x-timestamp']) metadata = {} metadata.update( - (key, (value, timestamp)) for key, value in req.headers.iteritems() + (key, (value, req_timestamp.internal)) + for key, value in req.headers.iteritems() if key.lower() in self.save_headers or is_sys_or_user_meta('container', key)) if metadata: diff --git a/swift/container/sync.py b/swift/container/sync.py index aa1ecc6929..b5fb5e203a 100644 --- a/swift/container/sync.py +++ b/swift/container/sync.py @@ -32,7 +32,7 @@ from swift.common.ring import Ring from swift.common.utils import ( audit_location_generator, clean_content_type, config_true_value, FileLikeIter, get_logger, hash_path, quote, urlparse, validate_sync_to, - whataremyips) + whataremyips, Timestamp) from swift.common.daemon import Daemon from swift.common.http import HTTP_UNAUTHORIZED, HTTP_NOT_FOUND from swift.common.storage_policy import POLICIES, POLICY_INDEX @@ -373,7 +373,7 @@ class ContainerSync(Daemon): row['name']) shuffle(nodes) exc = None - looking_for_timestamp = float(row['created_at']) + looking_for_timestamp = Timestamp(row['created_at']) timestamp = -1 headers = body = None headers_out = {POLICY_INDEX: str(info['storage_policy_index'])} @@ -383,7 +383,8 @@ class ContainerSync(Daemon): node, part, info['account'], info['container'], row['name'], headers=headers_out, resp_chunk_size=65536) - this_timestamp = float(these_headers['x-timestamp']) + this_timestamp = Timestamp( + these_headers['x-timestamp']) if this_timestamp > timestamp: timestamp = this_timestamp headers = these_headers diff --git a/swift/container/updater.py b/swift/container/updater.py index 2a13164479..4097f136b6 100644 --- a/swift/container/updater.py +++ b/swift/container/updater.py @@ -30,7 +30,7 @@ from swift.common.bufferedhttp import http_connect from swift.common.exceptions import ConnectionTimeout from swift.common.ring import Ring from swift.common.utils import get_logger, config_true_value, ismount, \ - dump_recon_cache, quorum_size + dump_recon_cache, quorum_size, Timestamp from swift.common.daemon import Daemon from swift.common.http import is_success, HTTP_INTERNAL_SERVER_ERROR from swift.common.storage_policy import POLICY_INDEX @@ -210,7 +210,7 @@ class ContainerUpdater(Daemon): info = broker.get_info() # Don't send updates if the container was auto-created since it # definitely doesn't have up to date statistics. - if float(info['put_timestamp']) <= 0: + if Timestamp(info['put_timestamp']) <= 0: return if self.account_suppressions.get(info['account'], 0) > time.time(): return diff --git a/swift/obj/diskfile.py b/swift/obj/diskfile.py index 91de3d4fa0..be3cde7416 100644 --- a/swift/obj/diskfile.py +++ b/swift/obj/diskfile.py @@ -49,7 +49,7 @@ from eventlet import Timeout from swift import gettext_ as _ from swift.common.constraints import check_mount -from swift.common.utils import mkdirs, normalize_timestamp, \ +from swift.common.utils import mkdirs, Timestamp, \ storage_directory, hash_path, renamer, fallocate, fsync, \ fdatasync, drop_buffer_cache, ThreadPool, lock_path, write_pickle, \ config_true_value, listdir, split_path, ismount, remove_file @@ -228,7 +228,7 @@ def hash_cleanup_listdir(hsh_path, reclaim_age=ONE_WEEK): if files[0].endswith('.ts'): # remove tombstones older than reclaim_age ts = files[0].rsplit('.', 1)[0] - if (time.time() - float(ts)) > reclaim_age: + if (time.time() - float(Timestamp(ts))) > reclaim_age: remove_file(join(hsh_path, files[0])) files.remove(files[0]) elif files: @@ -552,7 +552,7 @@ class DiskFileManager(object): write_pickle, data, os.path.join(async_dir, ohash[-3:], ohash + '-' + - normalize_timestamp(timestamp)), + Timestamp(timestamp).internal), os.path.join(device_path, get_tmp_dir(policy_idx))) self.logger.increment('async_pendings') @@ -794,7 +794,7 @@ class DiskFileWriter(object): :param metadata: dictionary of metadata to be associated with the object """ - timestamp = normalize_timestamp(metadata['X-Timestamp']) + timestamp = Timestamp(metadata['X-Timestamp']).internal metadata['name'] = self._name target_path = join(self._datadir, timestamp + self._extension) @@ -1060,7 +1060,7 @@ class DiskFile(object): def timestamp(self): if self._metadata is None: raise DiskFileNotOpen() - return self._metadata.get('X-Timestamp') + return Timestamp(self._metadata.get('X-Timestamp')) @classmethod def from_hash_dir(cls, mgr, hash_dir_path, device_path, partition): @@ -1449,7 +1449,7 @@ class DiskFile(object): :raises DiskFileError: this implementation will raise the same errors as the `create()` method. """ - timestamp = normalize_timestamp(timestamp) + timestamp = Timestamp(timestamp).internal with self.create() as deleter: deleter._extension = '.ts' diff --git a/swift/obj/mem_diskfile.py b/swift/obj/mem_diskfile.py index 939e783703..d02cdbe43e 100644 --- a/swift/obj/mem_diskfile.py +++ b/swift/obj/mem_diskfile.py @@ -22,7 +22,7 @@ from contextlib import contextmanager from eventlet import Timeout -from swift.common.utils import normalize_timestamp +from swift.common.utils import Timestamp from swift.common.exceptions import DiskFileQuarantined, DiskFileNotExist, \ DiskFileCollision, DiskFileDeleted, DiskFileNotOpen from swift.common.swob import multi_range_iterator @@ -394,7 +394,6 @@ class DiskFile(object): :param timestamp: timestamp to compare with each file """ - timestamp = normalize_timestamp(timestamp) fp, md = self._filesystem.get_object(self._name) - if md['X-Timestamp'] < timestamp: + if md['X-Timestamp'] < Timestamp(timestamp): self._filesystem.del_object(self._name) diff --git a/swift/obj/server.py b/swift/obj/server.py index 4d771ae7ff..d7b4ef292d 100644 --- a/swift/obj/server.py +++ b/swift/obj/server.py @@ -29,10 +29,10 @@ from eventlet import sleep, Timeout from swift.common.utils import public, get_logger, \ config_true_value, timing_stats, replication, \ - normalize_delete_at_timestamp, get_log_line + normalize_delete_at_timestamp, get_log_line, Timestamp from swift.common.bufferedhttp import http_connect from swift.common.constraints import check_object_creation, \ - check_float, check_utf8 + valid_timestamp, check_utf8 from swift.common.exceptions import ConnectionTimeout, DiskFileQuarantined, \ DiskFileNotExist, DiskFileCollision, DiskFileNoSpace, DiskFileDeleted, \ DiskFileDeviceUnavailable, DiskFileExpired, ChunkReadTimeout @@ -271,7 +271,7 @@ class ObjectController(object): headers_in = request.headers headers_out = HeaderKeyDict({ POLICY_INDEX: 0, # system accounts are always Policy-0 - 'x-timestamp': headers_in['x-timestamp'], + 'x-timestamp': request.timestamp.internal, 'x-trans-id': headers_in.get('x-trans-id', '-'), 'referer': request.as_referer()}) if op != 'DELETE': @@ -325,10 +325,7 @@ class ObjectController(object): """Handle HTTP POST requests for the Swift Object Server.""" device, partition, account, container, obj, policy_idx = \ get_name_and_placement(request, 5, 5, True) - if 'x-timestamp' not in request.headers or \ - not check_float(request.headers['x-timestamp']): - return HTTPBadRequest(body='Missing timestamp', request=request, - content_type='text/plain') + req_timestamp = valid_timestamp(request) new_delete_at = int(request.headers.get('X-Delete-At') or 0) if new_delete_at and new_delete_at < time.time(): return HTTPBadRequest(body='X-Delete-At in past', request=request, @@ -343,10 +340,10 @@ class ObjectController(object): orig_metadata = disk_file.read_metadata() except (DiskFileNotExist, DiskFileQuarantined): return HTTPNotFound(request=request) - orig_timestamp = orig_metadata.get('X-Timestamp', '0') - if orig_timestamp >= request.headers['x-timestamp']: + orig_timestamp = Timestamp(orig_metadata.get('X-Timestamp', 0)) + if orig_timestamp >= req_timestamp: return HTTPConflict(request=request) - metadata = {'X-Timestamp': request.headers['x-timestamp']} + metadata = {'X-Timestamp': req_timestamp.internal} metadata.update(val for val in request.headers.iteritems() if is_user_meta('object', val[0])) for header_key in self.allowed_headers: @@ -371,10 +368,7 @@ class ObjectController(object): """Handle HTTP PUT requests for the Swift Object Server.""" device, partition, account, container, obj, policy_idx = \ get_name_and_placement(request, 5, 5, True) - if 'x-timestamp' not in request.headers or \ - not check_float(request.headers['x-timestamp']): - return HTTPBadRequest(body='Missing timestamp', request=request, - content_type='text/plain') + req_timestamp = valid_timestamp(request) error_response = check_object_creation(request, obj) if error_response: return error_response @@ -407,8 +401,8 @@ class ObjectController(object): # The current ETag matches, so return 412 return HTTPPreconditionFailed(request=request) - orig_timestamp = orig_metadata.get('X-Timestamp') - if orig_timestamp and orig_timestamp >= request.headers['x-timestamp']: + orig_timestamp = Timestamp(orig_metadata.get('X-Timestamp', 0)) + if orig_timestamp and orig_timestamp >= req_timestamp: return HTTPConflict(request=request) orig_delete_at = int(orig_metadata.get('X-Delete-At') or 0) upload_expiration = time.time() + self.max_upload_time @@ -445,7 +439,7 @@ class ObjectController(object): request.headers['etag'].lower() != etag: return HTTPUnprocessableEntity(request=request) metadata = { - 'X-Timestamp': request.headers['x-timestamp'], + 'X-Timestamp': request.timestamp.internal, 'Content-Type': request.headers['content-type'], 'ETag': etag, 'Content-Length': str(upload_size), @@ -499,8 +493,7 @@ class ObjectController(object): with disk_file.open(): metadata = disk_file.get_metadata() obj_size = int(metadata['Content-Length']) - file_x_ts = metadata['X-Timestamp'] - file_x_ts_flt = float(file_x_ts) + file_x_ts = Timestamp(metadata['X-Timestamp']) keep_cache = (self.keep_cache_private or ('X-Auth-Token' not in request.headers and 'X-Storage-Token' not in request.headers)) @@ -514,19 +507,20 @@ class ObjectController(object): key.lower() in self.allowed_headers: response.headers[key] = value response.etag = metadata['ETag'] - response.last_modified = math.ceil(file_x_ts_flt) + response.last_modified = math.ceil(float(file_x_ts)) response.content_length = obj_size try: response.content_encoding = metadata[ 'Content-Encoding'] except KeyError: pass - response.headers['X-Timestamp'] = file_x_ts + response.headers['X-Timestamp'] = file_x_ts.normal + response.headers['X-Backend-Timestamp'] = file_x_ts.internal resp = request.get_response(response) except (DiskFileNotExist, DiskFileQuarantined) as e: headers = {} if hasattr(e, 'timestamp'): - headers['X-Backend-Timestamp'] = e.timestamp + headers['X-Backend-Timestamp'] = e.timestamp.internal resp = HTTPNotFound(request=request, headers=headers, conditional_response=True) return resp @@ -548,7 +542,7 @@ class ObjectController(object): except (DiskFileNotExist, DiskFileQuarantined) as e: headers = {} if hasattr(e, 'timestamp'): - headers['X-Backend-Timestamp'] = e.timestamp + headers['X-Backend-Timestamp'] = e.timestamp.internal return HTTPNotFound(request=request, headers=headers, conditional_response=True) response = Response(request=request, conditional_response=True) @@ -559,10 +553,11 @@ class ObjectController(object): key.lower() in self.allowed_headers: response.headers[key] = value response.etag = metadata['ETag'] - ts = metadata['X-Timestamp'] + ts = Timestamp(metadata['X-Timestamp']) response.last_modified = math.ceil(float(ts)) # Needed for container sync feature - response.headers['X-Timestamp'] = ts + response.headers['X-Timestamp'] = ts.normal + response.headers['X-Backend-Timestamp'] = ts.internal response.content_length = int(metadata['Content-Length']) try: response.content_encoding = metadata['Content-Encoding'] @@ -576,10 +571,7 @@ class ObjectController(object): """Handle HTTP DELETE requests for the Swift Object Server.""" device, partition, account, container, obj, policy_idx = \ get_name_and_placement(request, 5, 5, True) - if 'x-timestamp' not in request.headers or \ - not check_float(request.headers['x-timestamp']): - return HTTPBadRequest(body='Missing timestamp', request=request, - content_type='text/plain') + req_timestamp = valid_timestamp(request) try: disk_file = self.get_diskfile( device, partition, account, container, obj, @@ -601,8 +593,8 @@ class ObjectController(object): orig_metadata = {} response_class = HTTPNotFound else: - orig_timestamp = orig_metadata.get('X-Timestamp', 0) - if orig_timestamp < request.headers['x-timestamp']: + orig_timestamp = Timestamp(orig_metadata.get('X-Timestamp', 0)) + if orig_timestamp < req_timestamp: response_class = HTTPNoContent else: response_class = HTTPConflict @@ -633,12 +625,11 @@ class ObjectController(object): self.delete_at_update('DELETE', orig_delete_at, account, container, obj, request, device, policy_idx) - req_timestamp = request.headers['X-Timestamp'] if orig_timestamp < req_timestamp: disk_file.delete(req_timestamp) self.container_update( 'DELETE', account, container, obj, request, - HeaderKeyDict({'x-timestamp': req_timestamp}), + HeaderKeyDict({'x-timestamp': req_timestamp.internal}), device, policy_idx) return response_class(request=request) diff --git a/swift/proxy/controllers/base.py b/swift/proxy/controllers/base.py index 233935254d..5d65e3c783 100644 --- a/swift/proxy/controllers/base.py +++ b/swift/proxy/controllers/base.py @@ -36,7 +36,7 @@ from eventlet import sleep from eventlet.timeout import Timeout from swift.common.wsgi import make_pre_authed_env -from swift.common.utils import normalize_timestamp, config_true_value, \ +from swift.common.utils import Timestamp, config_true_value, \ public, split_path, list_from_csv, GreenthreadSafeIterator, \ quorum_size, GreenAsyncPile from swift.common.bufferedhttp import http_connect @@ -926,7 +926,7 @@ class Controller(object): headers = HeaderKeyDict(additional) if additional else HeaderKeyDict() if transfer: self.transfer_headers(orig_req.headers, headers) - headers.setdefault('x-timestamp', normalize_timestamp(time.time())) + headers.setdefault('x-timestamp', Timestamp(time.time()).internal) if orig_req: referer = orig_req.as_referer() else: @@ -1158,7 +1158,7 @@ class Controller(object): """ partition, nodes = self.app.account_ring.get_nodes(account) path = '/%s' % account - headers = {'X-Timestamp': normalize_timestamp(time.time()), + headers = {'X-Timestamp': Timestamp(time.time()).internal, 'X-Trans-Id': self.trans_id, 'Connection': 'close'} resp = self.make_requests(Request.blank('/v1' + path), diff --git a/swift/proxy/controllers/container.py b/swift/proxy/controllers/container.py index 69eb68bc46..5920561525 100644 --- a/swift/proxy/controllers/container.py +++ b/swift/proxy/controllers/container.py @@ -17,7 +17,7 @@ from swift import gettext_ as _ from urllib import unquote import time -from swift.common.utils import public, csv_append, normalize_timestamp +from swift.common.utils import public, csv_append, Timestamp from swift.common.constraints import check_metadata from swift.common import constraints from swift.common.http import HTTP_ACCEPTED @@ -209,7 +209,7 @@ class ContainerController(Controller): def _backend_requests(self, req, n_outgoing, account_partition, accounts, policy_index=None): - additional = {'X-Timestamp': normalize_timestamp(time.time())} + additional = {'X-Timestamp': Timestamp(time.time()).internal} if policy_index is None: additional['X-Backend-Storage-Policy-Default'] = \ int(POLICIES.default) diff --git a/swift/proxy/controllers/obj.py b/swift/proxy/controllers/obj.py index 346238c419..63162f50ad 100644 --- a/swift/proxy/controllers/obj.py +++ b/swift/proxy/controllers/obj.py @@ -37,8 +37,8 @@ from eventlet.timeout import Timeout from swift.common.utils import ( clean_content_type, config_true_value, ContextPool, csv_append, - GreenAsyncPile, GreenthreadSafeIterator, json, - normalize_delete_at_timestamp, normalize_timestamp, public, quorum_size) + GreenAsyncPile, GreenthreadSafeIterator, json, Timestamp, + normalize_delete_at_timestamp, public, quorum_size) from swift.common.bufferedhttp import http_connect from swift.common.constraints import check_metadata, check_object_creation, \ check_copy_from_header @@ -309,7 +309,7 @@ class ObjectController(Controller): req.headers[POLICY_INDEX] = policy_index partition, nodes = obj_ring.get_nodes( self.account_name, self.container_name, self.object_name) - req.headers['X-Timestamp'] = normalize_timestamp(time.time()) + req.headers['X-Timestamp'] = Timestamp(time.time()).internal headers = self._backend_requests( req, len(nodes), container_partition, containers, @@ -510,19 +510,18 @@ class ObjectController(Controller): # Used by container sync feature if 'x-timestamp' in req.headers: try: - req.headers['X-Timestamp'] = \ - normalize_timestamp(req.headers['x-timestamp']) + req_timestamp = Timestamp(req.headers['X-Timestamp']) if hresp.environ and 'swift_x_timestamp' in hresp.environ and \ - float(hresp.environ['swift_x_timestamp']) >= \ - float(req.headers['x-timestamp']): + hresp.environ['swift_x_timestamp'] >= req_timestamp: return HTTPAccepted(request=req) except ValueError: return HTTPBadRequest( request=req, content_type='text/plain', body='X-Timestamp should be a UNIX timestamp float value; ' 'was %r' % req.headers['x-timestamp']) + req.headers['X-Timestamp'] = req_timestamp.internal else: - req.headers['X-Timestamp'] = normalize_timestamp(time.time()) + req.headers['X-Timestamp'] = Timestamp(time.time()).internal # Sometimes the 'content-type' header exists, but is set to None. content_type_manually_set = True detect_content_type = \ @@ -554,7 +553,7 @@ class ObjectController(Controller): ts_source = time.mktime(time.strptime( hresp.headers['last-modified'], '%a, %d %b %Y %H:%M:%S GMT')) - new_ts = normalize_timestamp(ts_source) + new_ts = Timestamp(ts_source).internal vers_obj_name = lprefix + new_ts copy_headers = { 'Destination': '%s/%s' % (lcontainer, vers_obj_name)} @@ -766,7 +765,8 @@ class ObjectController(Controller): resp.headers['X-Copied-From-Last-Modified'] = \ source_resp.headers['last-modified'] copy_headers_into(req, resp) - resp.last_modified = math.ceil(float(req.headers['X-Timestamp'])) + resp.last_modified = math.ceil( + float(Timestamp(req.headers['X-Timestamp']))) return resp @public @@ -858,15 +858,15 @@ class ObjectController(Controller): # Used by container sync feature if 'x-timestamp' in req.headers: try: - req.headers['X-Timestamp'] = \ - normalize_timestamp(req.headers['x-timestamp']) + req_timestamp = Timestamp(req.headers['X-Timestamp']) except ValueError: return HTTPBadRequest( request=req, content_type='text/plain', body='X-Timestamp should be a UNIX timestamp float value; ' 'was %r' % req.headers['x-timestamp']) + req.headers['X-Timestamp'] = req_timestamp.internal else: - req.headers['X-Timestamp'] = normalize_timestamp(time.time()) + req.headers['X-Timestamp'] = Timestamp(time.time()).internal headers = self._backend_requests( req, len(nodes), container_partition, containers) diff --git a/test/probe/test_object_expirer.py b/test/probe/test_object_expirer.py index 4e336483c1..243e419578 100644 --- a/test/probe/test_object_expirer.py +++ b/test/probe/test_object_expirer.py @@ -21,6 +21,7 @@ from nose import SkipTest from swift.common.internal_client import InternalClient from swift.common.manager import Manager from swift.common.storage_policy import POLICIES +from swift.common.utils import Timestamp from test.probe.common import reset_environment, get_to_final_state from test.probe.test_container_merge_policy_index import BrainSplitter @@ -60,12 +61,12 @@ class TestObjectExpirer(unittest.TestCase): # create an expiring object and a container with the wrong policy self.brain.stop_primary_half() self.brain.put_container(int(old_policy)) - self.brain.put_object(headers={'X-Delete-After': 1}) + self.brain.put_object(headers={'X-Delete-After': 2}) # get the object timestamp metadata = self.client.get_object_metadata( self.account, self.container_name, self.object_name, headers={'X-Backend-Storage-Policy-Index': int(old_policy)}) - create_timestamp = metadata['x-timestamp'] + create_timestamp = Timestamp(metadata['x-timestamp']) self.brain.start_primary_half() # get the expiring object updates in their queue, while we have all # the servers up @@ -89,7 +90,7 @@ class TestObjectExpirer(unittest.TestCase): acceptable_statuses=(4,), headers={'X-Backend-Storage-Policy-Index': int(old_policy)}) self.assert_('x-backend-timestamp' in metadata) - self.assertEqual(metadata['x-backend-timestamp'], + self.assertEqual(Timestamp(metadata['x-backend-timestamp']), create_timestamp) # but it is still in the listing @@ -124,8 +125,8 @@ class TestObjectExpirer(unittest.TestCase): (found_in_policy, policy)) found_in_policy = policy self.assert_('x-backend-timestamp' in metadata) - self.assert_(float(metadata['x-backend-timestamp']) > - float(create_timestamp)) + self.assert_(Timestamp(metadata['x-backend-timestamp']) > + create_timestamp) if __name__ == "__main__": unittest.main() diff --git a/test/unit/account/test_backend.py b/test/unit/account/test_backend.py index 5b47f102f1..eb0bccc8a4 100644 --- a/test/unit/account/test_backend.py +++ b/test/unit/account/test_backend.py @@ -29,7 +29,7 @@ from contextlib import contextmanager import random from swift.account.backend import AccountBroker -from swift.common.utils import normalize_timestamp +from swift.common.utils import Timestamp from test.unit import patch_policies, with_tempdir from swift.common.db import DatabaseConnectionError from swift.common.storage_policy import StoragePolicy, POLICIES @@ -57,7 +57,7 @@ class TestAccountBroker(unittest.TestCase): self.fail("Unexpected exception raised: %r" % e) else: self.fail("Expected a DatabaseConnectionError exception") - broker.initialize(normalize_timestamp('1')) + broker.initialize(Timestamp('1').internal) with broker.get() as conn: curs = conn.cursor() curs.execute('SELECT 1') @@ -67,7 +67,7 @@ class TestAccountBroker(unittest.TestCase): # Test AccountBroker throwing a conn away after exception first_conn = None broker = AccountBroker(':memory:', account='a') - broker.initialize(normalize_timestamp('1')) + broker.initialize(Timestamp('1').internal) with broker.get() as conn: first_conn = conn try: @@ -81,20 +81,20 @@ class TestAccountBroker(unittest.TestCase): def test_empty(self): # Test AccountBroker.empty broker = AccountBroker(':memory:', account='a') - broker.initialize(normalize_timestamp('1')) + broker.initialize(Timestamp('1').internal) self.assert_(broker.empty()) - broker.put_container('o', normalize_timestamp(time()), 0, 0, 0, + broker.put_container('o', Timestamp(time()).internal, 0, 0, 0, POLICIES.default.idx) self.assert_(not broker.empty()) sleep(.00001) - broker.put_container('o', 0, normalize_timestamp(time()), 0, 0, + broker.put_container('o', 0, Timestamp(time()).internal, 0, 0, POLICIES.default.idx) self.assert_(broker.empty()) def test_reclaim(self): broker = AccountBroker(':memory:', account='test_account') - broker.initialize(normalize_timestamp('1')) - broker.put_container('c', normalize_timestamp(time()), 0, 0, 0, + broker.initialize(Timestamp('1').internal) + broker.put_container('c', Timestamp(time()).internal, 0, 0, 0, POLICIES.default.idx) with broker.get() as conn: self.assertEqual(conn.execute( @@ -103,7 +103,7 @@ class TestAccountBroker(unittest.TestCase): self.assertEqual(conn.execute( "SELECT count(*) FROM container " "WHERE deleted = 1").fetchone()[0], 0) - broker.reclaim(normalize_timestamp(time() - 999), time()) + broker.reclaim(Timestamp(time() - 999).internal, time()) with broker.get() as conn: self.assertEqual(conn.execute( "SELECT count(*) FROM container " @@ -112,7 +112,7 @@ class TestAccountBroker(unittest.TestCase): "SELECT count(*) FROM container " "WHERE deleted = 1").fetchone()[0], 0) sleep(.00001) - broker.put_container('c', 0, normalize_timestamp(time()), 0, 0, + broker.put_container('c', 0, Timestamp(time()).internal, 0, 0, POLICIES.default.idx) with broker.get() as conn: self.assertEqual(conn.execute( @@ -121,7 +121,7 @@ class TestAccountBroker(unittest.TestCase): self.assertEqual(conn.execute( "SELECT count(*) FROM container " "WHERE deleted = 1").fetchone()[0], 1) - broker.reclaim(normalize_timestamp(time() - 999), time()) + broker.reclaim(Timestamp(time() - 999).internal, time()) with broker.get() as conn: self.assertEqual(conn.execute( "SELECT count(*) FROM container " @@ -130,7 +130,7 @@ class TestAccountBroker(unittest.TestCase): "SELECT count(*) FROM container " "WHERE deleted = 1").fetchone()[0], 1) sleep(.00001) - broker.reclaim(normalize_timestamp(time()), time()) + broker.reclaim(Timestamp(time()).internal, time()) with broker.get() as conn: self.assertEqual(conn.execute( "SELECT count(*) FROM container " @@ -142,15 +142,15 @@ class TestAccountBroker(unittest.TestCase): broker.put_container('x', 0, 0, 0, 0, POLICIES.default.idx) broker.put_container('y', 0, 0, 0, 0, POLICIES.default.idx) broker.put_container('z', 0, 0, 0, 0, POLICIES.default.idx) - broker.reclaim(normalize_timestamp(time()), time()) + broker.reclaim(Timestamp(time()).internal, time()) # self.assertEqual(len(res), 2) # self.assert_(isinstance(res, tuple)) # containers, account_name = res # self.assert_(containers is None) # self.assert_(account_name is None) # Now delete the account - broker.delete_db(normalize_timestamp(time())) - broker.reclaim(normalize_timestamp(time()), time()) + broker.delete_db(Timestamp(time()).internal) + broker.reclaim(Timestamp(time()).internal, time()) # self.assertEqual(len(res), 2) # self.assert_(isinstance(res, tuple)) # containers, account_name = res @@ -162,34 +162,34 @@ class TestAccountBroker(unittest.TestCase): # self.assert_('a' not in containers) def test_delete_db_status(self): - start = int(time()) - ts = itertools.count(start) + ts = (Timestamp(t).internal for t in itertools.count(int(time()))) + start = ts.next() broker = AccountBroker(':memory:', account='a') - broker.initialize(normalize_timestamp(ts.next())) + broker.initialize(start) info = broker.get_info() - self.assertEqual(info['put_timestamp'], normalize_timestamp(start)) - self.assert_(float(info['created_at']) >= start) + self.assertEqual(info['put_timestamp'], Timestamp(start).internal) + self.assert_(Timestamp(info['created_at']) >= start) self.assertEqual(info['delete_timestamp'], '0') if self.__class__ == TestAccountBrokerBeforeMetadata: self.assertEqual(info['status_changed_at'], '0') else: self.assertEqual(info['status_changed_at'], - normalize_timestamp(start)) + Timestamp(start).internal) # delete it - delete_timestamp = normalize_timestamp(ts.next()) + delete_timestamp = ts.next() broker.delete_db(delete_timestamp) info = broker.get_info() - self.assertEqual(info['put_timestamp'], normalize_timestamp(start)) - self.assert_(float(info['created_at']) >= start) + self.assertEqual(info['put_timestamp'], Timestamp(start).internal) + self.assert_(Timestamp(info['created_at']) >= start) self.assertEqual(info['delete_timestamp'], delete_timestamp) self.assertEqual(info['status_changed_at'], delete_timestamp) def test_delete_container(self): # Test AccountBroker.delete_container broker = AccountBroker(':memory:', account='a') - broker.initialize(normalize_timestamp('1')) - broker.put_container('o', normalize_timestamp(time()), 0, 0, 0, + broker.initialize(Timestamp('1').internal) + broker.put_container('o', Timestamp(time()).internal, 0, 0, 0, POLICIES.default.idx) with broker.get() as conn: self.assertEqual(conn.execute( @@ -199,7 +199,7 @@ class TestAccountBroker(unittest.TestCase): "SELECT count(*) FROM container " "WHERE deleted = 1").fetchone()[0], 0) sleep(.00001) - broker.put_container('o', 0, normalize_timestamp(time()), 0, 0, + broker.put_container('o', 0, Timestamp(time()).internal, 0, 0, POLICIES.default.idx) with broker.get() as conn: self.assertEqual(conn.execute( @@ -212,10 +212,10 @@ class TestAccountBroker(unittest.TestCase): def test_put_container(self): # Test AccountBroker.put_container broker = AccountBroker(':memory:', account='a') - broker.initialize(normalize_timestamp('1')) + broker.initialize(Timestamp('1').internal) # Create initial container - timestamp = normalize_timestamp(time()) + timestamp = Timestamp(time()).internal broker.put_container('"{}"', timestamp, 0, 0, 0, POLICIES.default.idx) with broker.get() as conn: @@ -243,7 +243,7 @@ class TestAccountBroker(unittest.TestCase): # Put new event sleep(.00001) - timestamp = normalize_timestamp(time()) + timestamp = Timestamp(time()).internal broker.put_container('"{}"', timestamp, 0, 0, 0, POLICIES.default.idx) with broker.get() as conn: @@ -257,7 +257,7 @@ class TestAccountBroker(unittest.TestCase): "SELECT deleted FROM container").fetchone()[0], 0) # Put old event - otimestamp = normalize_timestamp(float(timestamp) - 1) + otimestamp = Timestamp(float(Timestamp(timestamp)) - 1).internal broker.put_container('"{}"', otimestamp, 0, 0, 0, POLICIES.default.idx) with broker.get() as conn: @@ -271,7 +271,7 @@ class TestAccountBroker(unittest.TestCase): "SELECT deleted FROM container").fetchone()[0], 0) # Put old delete event - dtimestamp = normalize_timestamp(float(timestamp) - 1) + dtimestamp = Timestamp(float(Timestamp(timestamp)) - 1).internal broker.put_container('"{}"', 0, dtimestamp, 0, 0, POLICIES.default.idx) with broker.get() as conn: @@ -289,7 +289,7 @@ class TestAccountBroker(unittest.TestCase): # Put new delete event sleep(.00001) - timestamp = normalize_timestamp(time()) + timestamp = Timestamp(time()).internal broker.put_container('"{}"', 0, timestamp, 0, 0, POLICIES.default.idx) with broker.get() as conn: @@ -304,7 +304,7 @@ class TestAccountBroker(unittest.TestCase): # Put new event sleep(.00001) - timestamp = normalize_timestamp(time()) + timestamp = Timestamp(time()).internal broker.put_container('"{}"', timestamp, 0, 0, 0, POLICIES.default.idx) with broker.get() as conn: @@ -320,46 +320,46 @@ class TestAccountBroker(unittest.TestCase): def test_get_info(self): # Test AccountBroker.get_info broker = AccountBroker(':memory:', account='test1') - broker.initialize(normalize_timestamp('1')) + broker.initialize(Timestamp('1').internal) info = broker.get_info() self.assertEqual(info['account'], 'test1') self.assertEqual(info['hash'], '00000000000000000000000000000000') - self.assertEqual(info['put_timestamp'], normalize_timestamp(1)) + self.assertEqual(info['put_timestamp'], Timestamp(1).internal) self.assertEqual(info['delete_timestamp'], '0') if self.__class__ == TestAccountBrokerBeforeMetadata: self.assertEqual(info['status_changed_at'], '0') else: - self.assertEqual(info['status_changed_at'], normalize_timestamp(1)) + self.assertEqual(info['status_changed_at'], Timestamp(1).internal) info = broker.get_info() self.assertEqual(info['container_count'], 0) - broker.put_container('c1', normalize_timestamp(time()), 0, 0, 0, + broker.put_container('c1', Timestamp(time()).internal, 0, 0, 0, POLICIES.default.idx) info = broker.get_info() self.assertEqual(info['container_count'], 1) sleep(.00001) - broker.put_container('c2', normalize_timestamp(time()), 0, 0, 0, + broker.put_container('c2', Timestamp(time()).internal, 0, 0, 0, POLICIES.default.idx) info = broker.get_info() self.assertEqual(info['container_count'], 2) sleep(.00001) - broker.put_container('c2', normalize_timestamp(time()), 0, 0, 0, + broker.put_container('c2', Timestamp(time()).internal, 0, 0, 0, POLICIES.default.idx) info = broker.get_info() self.assertEqual(info['container_count'], 2) sleep(.00001) - broker.put_container('c1', 0, normalize_timestamp(time()), 0, 0, + broker.put_container('c1', 0, Timestamp(time()).internal, 0, 0, POLICIES.default.idx) info = broker.get_info() self.assertEqual(info['container_count'], 1) sleep(.00001) - broker.put_container('c2', 0, normalize_timestamp(time()), 0, 0, + broker.put_container('c2', 0, Timestamp(time()).internal, 0, 0, POLICIES.default.idx) info = broker.get_info() self.assertEqual(info['container_count'], 0) @@ -367,20 +367,20 @@ class TestAccountBroker(unittest.TestCase): def test_list_containers_iter(self): # Test AccountBroker.list_containers_iter broker = AccountBroker(':memory:', account='a') - broker.initialize(normalize_timestamp('1')) + broker.initialize(Timestamp('1').internal) for cont1 in xrange(4): for cont2 in xrange(125): broker.put_container('%d-%04d' % (cont1, cont2), - normalize_timestamp(time()), 0, 0, 0, + Timestamp(time()).internal, 0, 0, 0, POLICIES.default.idx) for cont in xrange(125): broker.put_container('2-0051-%04d' % cont, - normalize_timestamp(time()), 0, 0, 0, + Timestamp(time()).internal, 0, 0, 0, POLICIES.default.idx) for cont in xrange(125): broker.put_container('3-%04d-0049' % cont, - normalize_timestamp(time()), 0, 0, 0, + Timestamp(time()).internal, 0, 0, 0, POLICIES.default.idx) listing = broker.list_containers_iter(100, '', None, None, '') @@ -445,7 +445,7 @@ class TestAccountBroker(unittest.TestCase): '3-0047-', '3-0048', '3-0048-', '3-0049', '3-0049-', '3-0050']) - broker.put_container('3-0049-', normalize_timestamp(time()), 0, 0, 0, + broker.put_container('3-0049-', Timestamp(time()).internal, 0, 0, 0, POLICIES.default.idx) listing = broker.list_containers_iter(10, '3-0048', None, None, None) self.assertEqual(len(listing), 10) @@ -470,26 +470,26 @@ class TestAccountBroker(unittest.TestCase): # Test AccountBroker.list_containers_iter for an # account that has an odd container with a trailing delimiter broker = AccountBroker(':memory:', account='a') - broker.initialize(normalize_timestamp('1')) - broker.put_container('a', normalize_timestamp(time()), 0, 0, 0, + broker.initialize(Timestamp('1').internal) + broker.put_container('a', Timestamp(time()).internal, 0, 0, 0, POLICIES.default.idx) - broker.put_container('a-', normalize_timestamp(time()), 0, 0, 0, + broker.put_container('a-', Timestamp(time()).internal, 0, 0, 0, POLICIES.default.idx) - broker.put_container('a-a', normalize_timestamp(time()), 0, 0, 0, + broker.put_container('a-a', Timestamp(time()).internal, 0, 0, 0, POLICIES.default.idx) - broker.put_container('a-a-a', normalize_timestamp(time()), 0, 0, 0, + broker.put_container('a-a-a', Timestamp(time()).internal, 0, 0, 0, POLICIES.default.idx) - broker.put_container('a-a-b', normalize_timestamp(time()), 0, 0, 0, + broker.put_container('a-a-b', Timestamp(time()).internal, 0, 0, 0, POLICIES.default.idx) - broker.put_container('a-b', normalize_timestamp(time()), 0, 0, 0, + broker.put_container('a-b', Timestamp(time()).internal, 0, 0, 0, POLICIES.default.idx) - broker.put_container('b', normalize_timestamp(time()), 0, 0, 0, + broker.put_container('b', Timestamp(time()).internal, 0, 0, 0, POLICIES.default.idx) - broker.put_container('b-a', normalize_timestamp(time()), 0, 0, 0, + broker.put_container('b-a', Timestamp(time()).internal, 0, 0, 0, POLICIES.default.idx) - broker.put_container('b-b', normalize_timestamp(time()), 0, 0, 0, + broker.put_container('b-b', Timestamp(time()).internal, 0, 0, 0, POLICIES.default.idx) - broker.put_container('c', normalize_timestamp(time()), 0, 0, 0, + broker.put_container('c', Timestamp(time()).internal, 0, 0, 0, POLICIES.default.idx) listing = broker.list_containers_iter(15, None, None, None, None) self.assertEqual(len(listing), 10) @@ -510,27 +510,30 @@ class TestAccountBroker(unittest.TestCase): def test_chexor(self): broker = AccountBroker(':memory:', account='a') - broker.initialize(normalize_timestamp('1')) - broker.put_container('a', normalize_timestamp(1), - normalize_timestamp(0), 0, 0, + broker.initialize(Timestamp('1').internal) + broker.put_container('a', Timestamp(1).internal, + Timestamp(0).internal, 0, 0, POLICIES.default.idx) - broker.put_container('b', normalize_timestamp(2), - normalize_timestamp(0), 0, 0, + broker.put_container('b', Timestamp(2).internal, + Timestamp(0).internal, 0, 0, POLICIES.default.idx) hasha = hashlib.md5( - '%s-%s' % ('a', '0000000001.00000-0000000000.00000-0-0') + '%s-%s' % ('a', "%s-%s-%s-%s" % ( + Timestamp(1).internal, Timestamp(0).internal, 0, 0)) ).digest() hashb = hashlib.md5( - '%s-%s' % ('b', '0000000002.00000-0000000000.00000-0-0') + '%s-%s' % ('b', "%s-%s-%s-%s" % ( + Timestamp(2).internal, Timestamp(0).internal, 0, 0)) ).digest() hashc = \ ''.join(('%02x' % (ord(a) ^ ord(b)) for a, b in zip(hasha, hashb))) self.assertEqual(broker.get_info()['hash'], hashc) - broker.put_container('b', normalize_timestamp(3), - normalize_timestamp(0), 0, 0, + broker.put_container('b', Timestamp(3).internal, + Timestamp(0).internal, 0, 0, POLICIES.default.idx) hashb = hashlib.md5( - '%s-%s' % ('b', '0000000003.00000-0000000000.00000-0-0') + '%s-%s' % ('b', "%s-%s-%s-%s" % ( + Timestamp(3).internal, Timestamp(0).internal, 0, 0)) ).digest() hashc = \ ''.join(('%02x' % (ord(a) ^ ord(b)) for a, b in zip(hasha, hashb))) @@ -538,12 +541,12 @@ class TestAccountBroker(unittest.TestCase): def test_merge_items(self): broker1 = AccountBroker(':memory:', account='a') - broker1.initialize(normalize_timestamp('1')) + broker1.initialize(Timestamp('1').internal) broker2 = AccountBroker(':memory:', account='a') - broker2.initialize(normalize_timestamp('1')) - broker1.put_container('a', normalize_timestamp(1), 0, 0, 0, + broker2.initialize(Timestamp('1').internal) + broker1.put_container('a', Timestamp(1).internal, 0, 0, 0, POLICIES.default.idx) - broker1.put_container('b', normalize_timestamp(2), 0, 0, 0, + broker1.put_container('b', Timestamp(2).internal, 0, 0, 0, POLICIES.default.idx) id = broker1.get_info()['id'] broker2.merge_items(broker1.get_items_since( @@ -551,7 +554,7 @@ class TestAccountBroker(unittest.TestCase): items = broker2.get_items_since(-1, 1000) self.assertEqual(len(items), 2) self.assertEqual(['a', 'b'], sorted([rec['name'] for rec in items])) - broker1.put_container('c', normalize_timestamp(3), 0, 0, 0, + broker1.put_container('c', Timestamp(3).internal, 0, 0, 0, POLICIES.default.idx) broker2.merge_items(broker1.get_items_since( broker2.get_sync(id), 1000), id) @@ -567,14 +570,14 @@ class TestAccountBroker(unittest.TestCase): broker_path = os.path.join(tempdir, 'test-load-old.db') try: broker = AccountBroker(broker_path, account='real') - broker.initialize(normalize_timestamp(1)) + broker.initialize(Timestamp(1).internal) with open(broker_path + '.pending', 'a+b') as pending: pending.write(':') pending.write(pickle.dumps( # name, put_timestamp, delete_timestamp, object_count, # bytes_used, deleted - ('oldcon', normalize_timestamp(200), - normalize_timestamp(0), + ('oldcon', Timestamp(200).internal, + Timestamp(0).internal, 896, 9216695, 0)).encode('base64')) broker._commit_puts() @@ -593,9 +596,9 @@ class TestAccountBroker(unittest.TestCase): StoragePolicy(2, 'two', False), StoragePolicy(3, 'three', False)]) def test_get_policy_stats(self): - ts = itertools.count() + ts = (Timestamp(t).internal for t in itertools.count(int(time()))) broker = AccountBroker(':memory:', account='a') - broker.initialize(normalize_timestamp(ts.next())) + broker.initialize(ts.next()) # check empty policy_stats self.assertTrue(broker.empty()) policy_stats = broker.get_policy_stats() @@ -604,7 +607,7 @@ class TestAccountBroker(unittest.TestCase): # add some empty containers for policy in POLICIES: container_name = 'c-%s' % policy.name - put_timestamp = normalize_timestamp(ts.next()) + put_timestamp = ts.next() broker.put_container(container_name, put_timestamp, 0, 0, 0, @@ -618,7 +621,7 @@ class TestAccountBroker(unittest.TestCase): # update the containers object & byte count for policy in POLICIES: container_name = 'c-%s' % policy.name - put_timestamp = normalize_timestamp(ts.next()) + put_timestamp = ts.next() count = policy.idx * 100 # good as any integer broker.put_container(container_name, put_timestamp, 0, @@ -640,7 +643,7 @@ class TestAccountBroker(unittest.TestCase): # now delete the containers one by one for policy in POLICIES: container_name = 'c-%s' % policy.name - delete_timestamp = normalize_timestamp(ts.next()) + delete_timestamp = ts.next() broker.put_container(container_name, 0, delete_timestamp, 0, 0, @@ -654,9 +657,9 @@ class TestAccountBroker(unittest.TestCase): @patch_policies([StoragePolicy(0, 'zero', False), StoragePolicy(1, 'one', True)]) def test_policy_stats_tracking(self): - ts = itertools.count() + ts = (Timestamp(t).internal for t in itertools.count(int(time()))) broker = AccountBroker(':memory:', account='a') - broker.initialize(normalize_timestamp(ts.next())) + broker.initialize(ts.next()) # policy 0 broker.put_container('con1', ts.next(), 0, 12, 2798641, 0) @@ -731,7 +734,7 @@ def premetadata_create_account_stat_table(self, conn, put_timestamp): conn.execute(''' UPDATE account_stat SET account = ?, created_at = ?, id = ?, put_timestamp = ? - ''', (self.account, normalize_timestamp(time()), str(uuid4()), + ''', (self.account, Timestamp(time()).internal, str(uuid4()), put_timestamp)) @@ -764,7 +767,7 @@ class TestAccountBrokerBeforeMetadata(TestAccountBroker): AccountBroker.create_account_stat_table = \ premetadata_create_account_stat_table broker = AccountBroker(':memory:', account='a') - broker.initialize(normalize_timestamp('1')) + broker.initialize(Timestamp('1').internal) exc = None with broker.get() as conn: try: @@ -777,7 +780,7 @@ class TestAccountBrokerBeforeMetadata(TestAccountBroker): AccountBroker.create_account_stat_table = \ self._imported_create_account_stat_table broker = AccountBroker(':memory:', account='a') - broker.initialize(normalize_timestamp('1')) + broker.initialize(Timestamp('1').internal) with broker.get() as conn: conn.execute('SELECT metadata FROM account_stat') @@ -851,7 +854,7 @@ class TestAccountBrokerBeforeSPI(TestAccountBroker): self._imported_initialize = AccountBroker._initialize AccountBroker._initialize = prespi_AccountBroker_initialize broker = AccountBroker(':memory:', account='a') - broker.initialize(normalize_timestamp('1')) + broker.initialize(Timestamp('1').internal) exc = None with broker.get() as conn: try: @@ -872,7 +875,7 @@ class TestAccountBrokerBeforeSPI(TestAccountBroker): self._imported_create_container_table AccountBroker._initialize = self._imported_initialize broker = AccountBroker(':memory:', account='a') - broker.initialize(normalize_timestamp('1')) + broker.initialize(Timestamp('1').internal) with broker.get() as conn: conn.execute('SELECT storage_policy_index FROM container') @@ -882,7 +885,7 @@ class TestAccountBrokerBeforeSPI(TestAccountBroker): # first init an acct DB without the policy_stat table present broker = AccountBroker(db_path, account='a') - broker.initialize(normalize_timestamp('1')) + broker.initialize(Timestamp('1').internal) with broker.get() as conn: try: conn.execute(''' @@ -900,7 +903,7 @@ class TestAccountBrokerBeforeSPI(TestAccountBroker): self.assertEqual(len(stats), 0) # now do a PUT to create the table - broker.put_container('o', normalize_timestamp(time()), 0, 0, 0, + broker.put_container('o', Timestamp(time()).internal, 0, 0, 0, POLICIES.default.idx) broker._commit_puts_stale_ok() @@ -918,7 +921,7 @@ class TestAccountBrokerBeforeSPI(TestAccountBroker): # first init an acct DB without the policy_stat table present broker = AccountBroker(db_path, account='a') - broker.initialize(normalize_timestamp('1')) + broker.initialize(Timestamp('1').internal) with broker.get() as conn: try: conn.execute(''' @@ -940,7 +943,7 @@ class TestAccountBrokerBeforeSPI(TestAccountBroker): delete_timestamp, object_count, bytes_used, deleted) VALUES (?, ?, ?, ?, ?, ?) - ''', ('test_name', normalize_timestamp(time()), 0, 1, 2, 0)) + ''', ('test_name', Timestamp(time()).internal, 0, 1, 2, 0)) conn.commit() # make sure we can iter containers without the migration @@ -955,7 +958,7 @@ class TestAccountBrokerBeforeSPI(TestAccountBroker): # which will update the DB schema as well as update policy_stats # for legacy containers in the DB (those without an SPI) other_policy = [p for p in POLICIES if p.idx != 0][0] - broker.put_container('test_second', normalize_timestamp(time()), + broker.put_container('test_second', Timestamp(time()).internal, 0, 3, 4, other_policy.idx) broker._commit_puts_stale_ok() @@ -994,14 +997,15 @@ class TestAccountBrokerBeforeSPI(TestAccountBroker): def test_half_upgraded_database(self, tempdir): db_path = os.path.join(tempdir, 'account.db') ts = itertools.count() + ts = (Timestamp(t).internal for t in itertools.count(int(time()))) broker = AccountBroker(db_path, account='a') - broker.initialize(normalize_timestamp(ts.next())) + broker.initialize(ts.next()) self.assertTrue(broker.empty()) # add a container (to pending file) - broker.put_container('c', normalize_timestamp(ts.next()), 0, 0, 0, + broker.put_container('c', ts.next(), 0, 0, 0, POLICIES.default.idx) real_get = broker.get diff --git a/test/unit/account/test_utils.py b/test/unit/account/test_utils.py new file mode 100644 index 0000000000..46f8835bff --- /dev/null +++ b/test/unit/account/test_utils.py @@ -0,0 +1,128 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import itertools +import time +import unittest + +import mock + +from swift.account import utils, backend +from swift.common.storage_policy import POLICIES +from swift.common.utils import Timestamp +from swift.common.swob import HeaderKeyDict + +from test.unit import patch_policies + + +class TestFakeAccountBroker(unittest.TestCase): + + def test_fake_broker_get_info(self): + broker = utils.FakeAccountBroker() + now = time.time() + with mock.patch('time.time', new=lambda: now): + info = broker.get_info() + timestamp = Timestamp(now) + expected = { + 'container_count': 0, + 'object_count': 0, + 'bytes_used': 0, + 'created_at': timestamp.internal, + 'put_timestamp': timestamp.internal, + } + self.assertEqual(info, expected) + + def test_fake_broker_list_containers_iter(self): + broker = utils.FakeAccountBroker() + self.assertEqual(broker.list_containers_iter(), []) + + def test_fake_broker_metadata(self): + broker = utils.FakeAccountBroker() + self.assertEqual(broker.metadata, {}) + + def test_fake_broker_get_policy_stats(self): + broker = utils.FakeAccountBroker() + self.assertEqual(broker.get_policy_stats(), {}) + + +class TestAccountUtils(unittest.TestCase): + + def test_get_response_headers_fake_broker(self): + broker = utils.FakeAccountBroker() + now = time.time() + expected = { + 'X-Account-Container-Count': 0, + 'X-Account-Object-Count': 0, + 'X-Account-Bytes-Used': 0, + 'X-Timestamp': Timestamp(now).normal, + 'X-PUT-Timestamp': Timestamp(now).normal, + } + with mock.patch('time.time', new=lambda: now): + resp_headers = utils.get_response_headers(broker) + self.assertEqual(resp_headers, expected) + + def test_get_response_headers_empty_memory_broker(self): + broker = backend.AccountBroker(':memory:', account='a') + now = time.time() + with mock.patch('time.time', new=lambda: now): + broker.initialize(Timestamp(now).internal) + expected = { + 'X-Account-Container-Count': 0, + 'X-Account-Object-Count': 0, + 'X-Account-Bytes-Used': 0, + 'X-Timestamp': Timestamp(now).normal, + 'X-PUT-Timestamp': Timestamp(now).normal, + } + resp_headers = utils.get_response_headers(broker) + self.assertEqual(resp_headers, expected) + + @patch_policies + def test_get_response_headers_with_data(self): + broker = backend.AccountBroker(':memory:', account='a') + now = time.time() + with mock.patch('time.time', new=lambda: now): + broker.initialize(Timestamp(now).internal) + # add some container data + ts = (Timestamp(t).internal for t in itertools.count(int(now))) + total_containers = 0 + total_objects = 0 + total_bytes = 0 + for policy in POLICIES: + delete_timestamp = ts.next() + put_timestamp = ts.next() + object_count = int(policy) + bytes_used = int(policy) * 10 + broker.put_container('c-%s' % policy.name, put_timestamp, + delete_timestamp, object_count, bytes_used, + int(policy)) + total_containers += 1 + total_objects += object_count + total_bytes += bytes_used + expected = HeaderKeyDict({ + 'X-Account-Container-Count': total_containers, + 'X-Account-Object-Count': total_objects, + 'X-Account-Bytes-Used': total_bytes, + 'X-Timestamp': Timestamp(now).normal, + 'X-PUT-Timestamp': Timestamp(now).normal, + }) + for policy in POLICIES: + prefix = 'X-Account-Storage-Policy-%s-' % policy.name + expected[prefix + 'Object-Count'] = int(policy) + expected[prefix + 'Bytes-Used'] = int(policy) * 10 + resp_headers = utils.get_response_headers(broker) + for key, value in resp_headers.items(): + expected_value = expected.pop(key) + self.assertEqual(expected_value, str(value), + 'value for %r was %r not %r' % ( + key, value, expected_value)) + self.assertFalse(expected) diff --git a/test/unit/cli/test_info.py b/test/unit/cli/test_info.py index d86c70354c..8a4387214b 100644 --- a/test/unit/cli/test_info.py +++ b/test/unit/cli/test_info.py @@ -115,10 +115,10 @@ class TestCliInfo(TestCliInfoBase): Account: acct Account Hash: dc5be2aa4347a22a0fee6bc7de505b47 Metadata: - Created at: 1970-01-01 00:01:40.100000 (100.1) - Put Timestamp: 1970-01-01 00:01:46.300000 (106.3) - Delete Timestamp: 1970-01-01 00:01:47.900000 (107.9) - Status Timestamp: 1970-01-01 00:01:48.300000 (108.3) + Created at: 1970-01-01T00:01:40.100000 (100.1) + Put Timestamp: 1970-01-01T00:01:46.300000 (106.3) + Delete Timestamp: 1970-01-01T00:01:47.900000 (107.9) + Status Timestamp: 1970-01-01T00:01:48.300000 (108.3) Container Count: 3 Object Count: 20 Bytes Used: 42 @@ -158,15 +158,15 @@ No system metadata found in db file Container: cont Container Hash: d49d0ecbb53be1fcc49624f2f7c7ccae Metadata: - Created at: 1970-01-01 00:01:40.100000 (0000000100.10000) - Put Timestamp: 1970-01-01 00:01:46.300000 (0000000106.30000) - Delete Timestamp: 1970-01-01 00:01:47.900000 (0000000107.90000) - Status Timestamp: 1970-01-01 00:01:48.300000 (0000000108.30000) + Created at: 1970-01-01T00:01:40.100000 (0000000100.10000) + Put Timestamp: 1970-01-01T00:01:46.300000 (0000000106.30000) + Delete Timestamp: 1970-01-01T00:01:47.900000 (0000000107.90000) + Status Timestamp: 1970-01-01T00:01:48.300000 (0000000108.30000) Object Count: 20 Bytes Used: 42 Storage Policy: %s (0) - Reported Put Timestamp: 1970-01-01 02:48:26.300000 (0000010106.30000) - Reported Delete Timestamp: 1970-01-01 02:48:27.900000 (0000010107.90000) + Reported Put Timestamp: 1970-01-01T02:48:26.300000 (0000010106.30000) + Reported Delete Timestamp: 1970-01-01T02:48:27.900000 (0000010107.90000) Reported Object Count: 20 Reported Bytes Used: 42 Chexor: abaddeadbeefcafe @@ -452,8 +452,9 @@ class TestPrintObjFullMeta(TestCliInfoBase): Object: dummy Object hash: 128fdf98bddd1b1e8695f4340e67a67a Content-Type: application/octet-stream -Timestamp: 1970-01-01 00:01:46.300000 (106.3) -User Metadata: {'X-Object-Meta-Mtime': '107.3'}''' +Timestamp: 1970-01-01T00:01:46.300000 (%s) +User Metadata: {'X-Object-Meta-Mtime': '107.3'}''' % ( + utils.Timestamp(106.3).internal) self.assertEquals(out.getvalue().strip(), exp_out) @@ -469,8 +470,9 @@ User Metadata: {'X-Object-Meta-Mtime': '107.3'}''' print_obj_metadata(metadata) exp_out = '''Path: Not found in metadata Content-Type: application/octet-stream -Timestamp: 1970-01-01 00:01:46.300000 (106.3) -User Metadata: {'X-Object-Meta-Mtime': '107.3'}''' +Timestamp: 1970-01-01T00:01:46.300000 (%s) +User Metadata: {'X-Object-Meta-Mtime': '107.3'}''' % ( + utils.Timestamp(106.3).internal) self.assertEquals(out.getvalue().strip(), exp_out) @@ -485,8 +487,9 @@ User Metadata: {'X-Object-Meta-Mtime': '107.3'}''' Object: dummy Object hash: 128fdf98bddd1b1e8695f4340e67a67a Content-Type: Not found in metadata -Timestamp: 1970-01-01 00:01:46.300000 (106.3) -User Metadata: {'X-Object-Meta-Mtime': '107.3'}''' +Timestamp: 1970-01-01T00:01:46.300000 (%s) +User Metadata: {'X-Object-Meta-Mtime': '107.3'}''' % ( + utils.Timestamp(106.3).internal) self.assertEquals(out.getvalue().strip(), exp_out) diff --git a/test/unit/common/test_constraints.py b/test/unit/common/test_constraints.py index c6539cf3d7..753f858de7 100644 --- a/test/unit/common/test_constraints.py +++ b/test/unit/common/test_constraints.py @@ -16,6 +16,7 @@ import unittest import mock import tempfile +import time from test import safe_repr from test.unit import MockTrue @@ -182,6 +183,20 @@ class TestConstraints(unittest.TestCase): self.assertFalse(constraints.check_float('')) self.assertTrue(constraints.check_float('0')) + def test_valid_timestamp(self): + self.assertRaises(HTTPException, + constraints.valid_timestamp, + Request.blank('/')) + self.assertRaises(HTTPException, + constraints.valid_timestamp, + Request.blank('/', headers={ + 'X-Timestamp': 'asdf'})) + timestamp = utils.Timestamp(time.time()) + req = Request.blank('/', headers={'X-Timestamp': timestamp.internal}) + self.assertEqual(timestamp, constraints.valid_timestamp(req)) + req = Request.blank('/', headers={'X-Timestamp': timestamp.normal}) + self.assertEqual(timestamp, constraints.valid_timestamp(req)) + def test_check_utf8(self): unicode_sample = u'\uc77c\uc601' valid_utf8_str = unicode_sample.encode('utf-8') diff --git a/test/unit/common/test_db.py b/test/unit/common/test_db.py index 461e869f6b..e0c164de9b 100644 --- a/test/unit/common/test_db.py +++ b/test/unit/common/test_db.py @@ -35,7 +35,7 @@ import swift.common.db from swift.common.db import chexor, dict_factory, get_db_connection, \ DatabaseBroker, DatabaseConnectionError, DatabaseAlreadyExists, \ GreenDBConnection, PICKLE_PROTOCOL -from swift.common.utils import normalize_timestamp, mkdirs, json +from swift.common.utils import normalize_timestamp, mkdirs, json, Timestamp from swift.common.exceptions import LockTimeout from test.unit import with_tempdir @@ -217,7 +217,7 @@ class ExampleBroker(DatabaseBroker): INSERT INTO test_stat ( created_at, put_timestamp, status_changed_at) VALUES (?, ?, ?); - """, (normalize_timestamp(time.time()), put_timestamp, + """, (Timestamp(time.time()).internal, put_timestamp, put_timestamp)) def merge_items(self, item_list): @@ -268,8 +268,8 @@ class ExampleBroker(DatabaseBroker): def _is_deleted(self, conn): info = conn.execute('SELECT * FROM test_stat').fetchone() return (info['test_count'] in (None, '', 0, '0')) and \ - (normalize_timestamp(info['delete_timestamp']) > - normalize_timestamp(info['put_timestamp'])) + (Timestamp(info['delete_timestamp']) > + Timestamp(info['put_timestamp'])) class TestExampleBroker(unittest.TestCase): @@ -282,7 +282,7 @@ class TestExampleBroker(unittest.TestCase): policy = 0 def test_merge_timestamps_simple_delete(self): - ts = (normalize_timestamp(t) for t in + ts = (Timestamp(t).internal for t in itertools.count(int(time.time()))) put_timestamp = ts.next() broker = self.broker_class(':memory:', account='a', container='c') @@ -302,8 +302,7 @@ class TestExampleBroker(unittest.TestCase): self.assertEqual(info['created_at'], created_at) self.assertEqual(info['put_timestamp'], put_timestamp) self.assertEqual(info['delete_timestamp'], delete_timestamp) - self.assert_(info['status_changed_at'] > - normalize_timestamp(put_timestamp)) + self.assert_(info['status_changed_at'] > Timestamp(put_timestamp)) def put_item(self, broker, timestamp): broker.put_test('test', timestamp) @@ -312,7 +311,7 @@ class TestExampleBroker(unittest.TestCase): broker.delete_test('test', timestamp) def test_merge_timestamps_delete_with_objects(self): - ts = (normalize_timestamp(t) for t in + ts = (Timestamp(t).internal for t in itertools.count(int(time.time()))) put_timestamp = ts.next() broker = self.broker_class(':memory:', account='a', container='c') @@ -345,7 +344,7 @@ class TestExampleBroker(unittest.TestCase): self.assert_(broker.is_deleted()) def test_merge_timestamps_simple_recreate(self): - ts = (normalize_timestamp(t) for t in + ts = (Timestamp(t).internal for t in itertools.count(int(time.time()))) put_timestamp = ts.next() broker = self.broker_class(':memory:', account='a', container='c') @@ -361,7 +360,7 @@ class TestExampleBroker(unittest.TestCase): self.assertEqual(info['delete_timestamp'], delete_timestamp) orig_status_changed_at = info['status_changed_at'] self.assert_(orig_status_changed_at > - normalize_timestamp(virgin_status_changed_at)) + Timestamp(virgin_status_changed_at)) # recreate recreate_timestamp = ts.next() status_changed_at = time.time() @@ -375,7 +374,7 @@ class TestExampleBroker(unittest.TestCase): self.assert_(info['status_changed_at'], status_changed_at) def test_merge_timestamps_recreate_with_objects(self): - ts = (normalize_timestamp(t) for t in + ts = (Timestamp(t).internal for t in itertools.count(int(time.time()))) put_timestamp = ts.next() broker = self.broker_class(':memory:', account='a', container='c') @@ -390,8 +389,8 @@ class TestExampleBroker(unittest.TestCase): self.assertEqual(info['put_timestamp'], put_timestamp) self.assertEqual(info['delete_timestamp'], delete_timestamp) orig_status_changed_at = info['status_changed_at'] - self.assert_(normalize_timestamp(orig_status_changed_at) >= - normalize_timestamp(put_timestamp)) + self.assert_(Timestamp(orig_status_changed_at) >= + Timestamp(put_timestamp)) # add object self.put_item(broker, ts.next()) count_key = '%s_count' % broker.db_contains_type @@ -411,7 +410,7 @@ class TestExampleBroker(unittest.TestCase): self.assertFalse(broker.is_deleted()) def test_merge_timestamps_update_put_no_status_change(self): - ts = (normalize_timestamp(t) for t in + ts = (Timestamp(t).internal for t in itertools.count(int(time.time()))) put_timestamp = ts.next() broker = self.broker_class(':memory:', account='a', container='c') @@ -426,7 +425,7 @@ class TestExampleBroker(unittest.TestCase): self.assertEqual(orig_status_changed_at, info['status_changed_at']) def test_merge_timestamps_update_delete_no_status_change(self): - ts = (normalize_timestamp(t) for t in + ts = (Timestamp(t).internal for t in itertools.count(int(time.time()))) put_timestamp = ts.next() broker = self.broker_class(':memory:', account='a', container='c') @@ -458,15 +457,15 @@ class TestExampleBroker(unittest.TestCase): broker = self.broker_class(':memory:', account='test', container='c') created_at = time.time() with patch('swift.common.db.time.time', new=lambda: created_at): - broker.initialize(normalize_timestamp(1), + broker.initialize(Timestamp(1).internal, storage_policy_index=int(self.policy)) info = broker.get_info() count_key = '%s_count' % broker.db_contains_type expected = { count_key: 0, - 'created_at': normalize_timestamp(created_at), - 'put_timestamp': normalize_timestamp(1), - 'status_changed_at': normalize_timestamp(1), + 'created_at': Timestamp(created_at).internal, + 'put_timestamp': Timestamp(1).internal, + 'status_changed_at': Timestamp(1).internal, 'delete_timestamp': '0', } for k, v in expected.items(): @@ -476,14 +475,14 @@ class TestExampleBroker(unittest.TestCase): def test_get_raw_metadata(self): broker = self.broker_class(':memory:', account='test', container='c') - broker.initialize(normalize_timestamp(0), + broker.initialize(Timestamp(0).internal, storage_policy_index=int(self.policy)) self.assertEqual(broker.metadata, {}) self.assertEqual(broker.get_raw_metadata(), '') key = u'test\u062a'.encode('utf-8') value = u'value\u062a' metadata = { - key: [value, normalize_timestamp(1)] + key: [value, Timestamp(1).internal] } broker.update_metadata(metadata) self.assertEqual(broker.metadata, metadata) @@ -491,7 +490,7 @@ class TestExampleBroker(unittest.TestCase): json.dumps(metadata)) def test_put_timestamp(self): - ts = (normalize_timestamp(t) for t in + ts = (Timestamp(t).internal for t in itertools.count(int(time.time()))) broker = self.broker_class(':memory:', account='a', container='c') orig_put_timestamp = ts.next() @@ -514,7 +513,7 @@ class TestExampleBroker(unittest.TestCase): newer_put_timestamp) def test_status_changed_at(self): - ts = (normalize_timestamp(t) for t in + ts = (Timestamp(t).internal for t in itertools.count(int(time.time()))) broker = self.broker_class(':memory:', account='test', container='c') put_timestamp = ts.next() @@ -525,7 +524,7 @@ class TestExampleBroker(unittest.TestCase): self.assertEquals(broker.get_info()['status_changed_at'], put_timestamp) self.assertEquals(broker.get_info()['created_at'], - normalize_timestamp(created_at)) + Timestamp(created_at).internal) status_changed_at = ts.next() broker.update_status_changed_at(status_changed_at) self.assertEqual(broker.get_info()['status_changed_at'], @@ -543,7 +542,7 @@ class TestExampleBroker(unittest.TestCase): def test_get_syncs(self): broker = self.broker_class(':memory:', account='a', container='c') - broker.initialize(normalize_timestamp(time.time()), + broker.initialize(Timestamp(time.time()).internal, storage_policy_index=int(self.policy)) self.assertEqual([], broker.get_syncs()) broker.merge_syncs([{'sync_point': 1, 'remote_id': 'remote1'}]) @@ -557,7 +556,7 @@ class TestExampleBroker(unittest.TestCase): @with_tempdir def test_commit_pending(self, tempdir): - ts = (normalize_timestamp(t) for t in + ts = (Timestamp(t).internal for t in itertools.count(int(time.time()))) broker = self.broker_class(os.path.join(tempdir, 'test.db'), account='a', container='c') diff --git a/test/unit/common/test_direct_client.py b/test/unit/common/test_direct_client.py index bd0e3d885c..90b6c1754f 100644 --- a/test/unit/common/test_direct_client.py +++ b/test/unit/common/test_direct_client.py @@ -25,7 +25,7 @@ import mock from swift.common import direct_client from swift.common.exceptions import ClientException -from swift.common.utils import json, normalize_timestamp +from swift.common.utils import json, Timestamp from swift.common.swob import HeaderKeyDict, RESPONSE_REASONS from swift.common.storage_policy import POLICY_INDEX, POLICIES @@ -116,9 +116,9 @@ class TestDirectClient(unittest.TestCase): now = time.time() headers = direct_client.gen_headers(add_ts=True) self.assertEqual(headers['user-agent'], stub_user_agent) - self.assert_(now - 1 < float(headers['x-timestamp']) < now + 1) + self.assert_(now - 1 < Timestamp(headers['x-timestamp']) < now + 1) self.assertEqual(headers['x-timestamp'], - normalize_timestamp(float(headers['x-timestamp']))) + Timestamp(headers['x-timestamp']).internal) self.assertEqual(2, len(headers)) headers = direct_client.gen_headers(hdrs_in={'foo-bar': '47'}) @@ -142,9 +142,9 @@ class TestDirectClient(unittest.TestCase): expected_header_count += 1 self.assertEqual( headers['x-timestamp'], - normalize_timestamp(float(headers['x-timestamp']))) + Timestamp(headers['x-timestamp']).internal) self.assert_( - now - 1 < float(headers['x-timestamp']) < now + 1) + now - 1 < Timestamp(headers['x-timestamp']) < now + 1) self.assertEqual(expected_header_count, len(headers)) def test_direct_get_account(self): @@ -275,7 +275,7 @@ class TestDirectClient(unittest.TestCase): self.assert_('HEAD' in str(err)) def test_direct_head_container_deleted(self): - important_timestamp = normalize_timestamp(time.time()) + important_timestamp = Timestamp(time.time()).internal headers = HeaderKeyDict({'X-Backend-Important-Timestamp': important_timestamp}) @@ -432,7 +432,7 @@ class TestDirectClient(unittest.TestCase): self.assert_('HEAD' in str(err)) def test_direct_head_object_not_found(self): - important_timestamp = normalize_timestamp(time.time()) + important_timestamp = Timestamp(time.time()).internal stub_headers = {'X-Backend-Important-Timestamp': important_timestamp} with mocked_http_conn(404, headers=stub_headers) as conn: try: diff --git a/test/unit/common/test_swob.py b/test/unit/common/test_swob.py index 5ced5b68d9..7a070b1883 100644 --- a/test/unit/common/test_swob.py +++ b/test/unit/common/test_swob.py @@ -23,6 +23,7 @@ from StringIO import StringIO from urllib import quote import swift.common.swob +from swift.common import utils, exceptions class TestHeaderEnvironProxy(unittest.TestCase): @@ -464,6 +465,25 @@ class TestRequest(unittest.TestCase): self.assertEquals(req.params['a'], 'b') self.assertEquals(req.params['c'], 'd') + def test_timestamp_missing(self): + req = swift.common.swob.Request.blank('/') + self.assertRaises(exceptions.InvalidTimestamp, + getattr, req, 'timestamp') + + def test_timestamp_invalid(self): + req = swift.common.swob.Request.blank( + '/', headers={'X-Timestamp': 'asdf'}) + self.assertRaises(exceptions.InvalidTimestamp, + getattr, req, 'timestamp') + + def test_timestamp(self): + req = swift.common.swob.Request.blank( + '/', headers={'X-Timestamp': '1402447134.13507_00000001'}) + expected = utils.Timestamp('1402447134.13507', offset=1) + self.assertEqual(req.timestamp, expected) + self.assertEqual(req.timestamp.normal, expected.normal) + self.assertEqual(req.timestamp.internal, expected.internal) + def test_path(self): req = swift.common.swob.Request.blank('/hi?a=b&c=d') self.assertEquals(req.path, '/hi') diff --git a/test/unit/common/test_utils.py b/test/unit/common/test_utils.py index f0b90a8f3d..4d874c4da8 100644 --- a/test/unit/common/test_utils.py +++ b/test/unit/common/test_utils.py @@ -148,6 +148,562 @@ def reset_loggers(): delattr(utils.get_logger, 'console_handler4logger') +class TestTimestamp(unittest.TestCase): + """Tests for swift.common.utils.Timestamp""" + + def test_invalid_input(self): + self.assertRaises(ValueError, utils.Timestamp, time.time(), offset=-1) + + def test_invalid_string_conversion(self): + t = utils.Timestamp(time.time()) + self.assertRaises(TypeError, str, t) + + def test_normal_format_no_offset(self): + expected = '1402436408.91203' + test_values = ( + '1402436408.91203', + '1402436408.91203_00000000', + '1402436408.912030000', + '1402436408.912030000_0000000000000', + '000001402436408.912030000', + '000001402436408.912030000_0000000000', + 1402436408.91203, + 1402436408.912029, + 1402436408.9120300000000000, + 1402436408.91202999999999999, + utils.Timestamp(1402436408.91203), + utils.Timestamp(1402436408.91203, offset=0), + utils.Timestamp(1402436408.912029), + utils.Timestamp(1402436408.912029, offset=0), + utils.Timestamp('1402436408.91203'), + utils.Timestamp('1402436408.91203', offset=0), + utils.Timestamp('1402436408.91203_00000000'), + utils.Timestamp('1402436408.91203_00000000', offset=0), + ) + for value in test_values: + timestamp = utils.Timestamp(value) + self.assertEqual(timestamp.normal, expected) + # timestamp instance can also compare to string or float + self.assertEqual(timestamp, expected) + self.assertEqual(timestamp, float(expected)) + self.assertEqual(timestamp, utils.normalize_timestamp(expected)) + + def test_isoformat(self): + expected = '2014-06-10T22:47:32.054580' + test_values = ( + '1402440452.05458', + '1402440452.054579', + '1402440452.05458_00000000', + '1402440452.054579_00000000', + '1402440452.054580000', + '1402440452.054579999', + '1402440452.054580000_0000000000000', + '1402440452.054579999_0000ff00', + '000001402440452.054580000', + '000001402440452.0545799', + '000001402440452.054580000_0000000000', + '000001402440452.054579999999_00000fffff', + 1402440452.05458, + 1402440452.054579, + 1402440452.0545800000000000, + 1402440452.054579999, + utils.Timestamp(1402440452.05458), + utils.Timestamp(1402440452.0545799), + utils.Timestamp(1402440452.05458, offset=0), + utils.Timestamp(1402440452.05457999999, offset=0), + utils.Timestamp(1402440452.05458, offset=100), + utils.Timestamp(1402440452.054579, offset=100), + utils.Timestamp('1402440452.05458'), + utils.Timestamp('1402440452.054579999'), + utils.Timestamp('1402440452.05458', offset=0), + utils.Timestamp('1402440452.054579', offset=0), + utils.Timestamp('1402440452.05458', offset=300), + utils.Timestamp('1402440452.05457999', offset=300), + utils.Timestamp('1402440452.05458_00000000'), + utils.Timestamp('1402440452.05457999_00000000'), + utils.Timestamp('1402440452.05458_00000000', offset=0), + utils.Timestamp('1402440452.05457999_00000aaa', offset=0), + utils.Timestamp('1402440452.05458_00000000', offset=400), + utils.Timestamp('1402440452.054579_0a', offset=400), + ) + for value in test_values: + self.assertEqual(utils.Timestamp(value).isoformat, expected) + expected = '1970-01-01T00:00:00.000000' + test_values = ( + '0', + '0000000000.00000', + '0000000000.00000_ffffffffffff', + 0, + 0.0, + ) + for value in test_values: + self.assertEqual(utils.Timestamp(value).isoformat, expected) + + def test_not_equal(self): + ts = '1402436408.91203_0000000000000001' + test_values = ( + utils.Timestamp('1402436408.91203_0000000000000002'), + utils.Timestamp('1402436408.91203'), + utils.Timestamp(1402436408.91203), + utils.Timestamp(1402436408.91204), + utils.Timestamp(1402436408.91203, offset=0), + utils.Timestamp(1402436408.91203, offset=2), + ) + for value in test_values: + self.assertTrue(value != ts) + + def test_no_force_internal_no_offset(self): + """Test that internal is the same as normal with no offset""" + with mock.patch('swift.common.utils.FORCE_INTERNAL', new=False): + self.assertEqual(utils.Timestamp(0).internal, '0000000000.00000') + self.assertEqual(utils.Timestamp(1402437380.58186).internal, + '1402437380.58186') + self.assertEqual(utils.Timestamp(1402437380.581859).internal, + '1402437380.58186') + self.assertEqual(utils.Timestamp(0).internal, + utils.normalize_timestamp(0)) + + def test_no_force_internal_with_offset(self): + """Test that internal always includes the offset if significant""" + with mock.patch('swift.common.utils.FORCE_INTERNAL', new=False): + self.assertEqual(utils.Timestamp(0, offset=1).internal, + '0000000000.00000_0000000000000001') + self.assertEqual( + utils.Timestamp(1402437380.58186, offset=16).internal, + '1402437380.58186_0000000000000010') + self.assertEqual( + utils.Timestamp(1402437380.581859, offset=240).internal, + '1402437380.58186_00000000000000f0') + self.assertEqual( + utils.Timestamp('1402437380.581859_00000001', + offset=240).internal, + '1402437380.58186_00000000000000f1') + + def test_force_internal(self): + """Test that internal always includes the offset if forced""" + with mock.patch('swift.common.utils.FORCE_INTERNAL', new=True): + self.assertEqual(utils.Timestamp(0).internal, + '0000000000.00000_0000000000000000') + self.assertEqual(utils.Timestamp(1402437380.58186).internal, + '1402437380.58186_0000000000000000') + self.assertEqual(utils.Timestamp(1402437380.581859).internal, + '1402437380.58186_0000000000000000') + self.assertEqual(utils.Timestamp(0, offset=1).internal, + '0000000000.00000_0000000000000001') + self.assertEqual( + utils.Timestamp(1402437380.58186, offset=16).internal, + '1402437380.58186_0000000000000010') + self.assertEqual( + utils.Timestamp(1402437380.581859, offset=16).internal, + '1402437380.58186_0000000000000010') + + def test_internal_format_no_offset(self): + expected = '1402436408.91203_0000000000000000' + test_values = ( + '1402436408.91203', + '1402436408.91203_00000000', + '1402436408.912030000', + '1402436408.912030000_0000000000000', + '000001402436408.912030000', + '000001402436408.912030000_0000000000', + 1402436408.91203, + 1402436408.9120300000000000, + 1402436408.912029, + 1402436408.912029999999999999, + utils.Timestamp(1402436408.91203), + utils.Timestamp(1402436408.91203, offset=0), + utils.Timestamp(1402436408.912029), + utils.Timestamp(1402436408.91202999999999999, offset=0), + utils.Timestamp('1402436408.91203'), + utils.Timestamp('1402436408.91203', offset=0), + utils.Timestamp('1402436408.912029'), + utils.Timestamp('1402436408.912029', offset=0), + utils.Timestamp('1402436408.912029999999999'), + utils.Timestamp('1402436408.912029999999999', offset=0), + ) + for value in test_values: + # timestamp instance is always equivalent + self.assertEqual(utils.Timestamp(value), expected) + if utils.FORCE_INTERNAL: + # the FORCE_INTERNAL flag makes the internal format always + # include the offset portion of the timestamp even when it's + # not significant and would be bad during upgrades + self.assertEqual(utils.Timestamp(value).internal, expected) + else: + # unless we FORCE_INTERNAL, when there's no offset the + # internal format is equivalent to the normalized format + self.assertEqual(utils.Timestamp(value).internal, + '1402436408.91203') + + def test_internal_format_with_offset(self): + expected = '1402436408.91203_00000000000000f0' + test_values = ( + '1402436408.91203_000000f0', + '1402436408.912030000_0000000000f0', + '1402436408.912029_000000f0', + '1402436408.91202999999_0000000000f0', + '000001402436408.912030000_000000000f0', + '000001402436408.9120299999_000000000f0', + utils.Timestamp(1402436408.91203, offset=240), + utils.Timestamp(1402436408.912029, offset=240), + utils.Timestamp('1402436408.91203', offset=240), + utils.Timestamp('1402436408.91203_00000000', offset=240), + utils.Timestamp('1402436408.91203_0000000f', offset=225), + utils.Timestamp('1402436408.9120299999', offset=240), + utils.Timestamp('1402436408.9120299999_00000000', offset=240), + utils.Timestamp('1402436408.9120299999_00000010', offset=224), + ) + for value in test_values: + timestamp = utils.Timestamp(value) + self.assertEqual(timestamp.internal, expected) + # can compare with offset if the string is internalized + self.assertEqual(timestamp, expected) + # if comparison value only includes the normalized portion and the + # timestamp includes an offset, it is considered greater + normal = utils.Timestamp(expected).normal + self.assertTrue(timestamp > normal, + '%r is not bigger than %r given %r' % ( + timestamp, normal, value)) + self.assertTrue(timestamp > float(normal), + '%r is not bigger than %f given %r' % ( + timestamp, float(normal), value)) + + def test_int(self): + expected = 1402437965 + test_values = ( + '1402437965.91203', + '1402437965.91203_00000000', + '1402437965.912030000', + '1402437965.912030000_0000000000000', + '000001402437965.912030000', + '000001402437965.912030000_0000000000', + 1402437965.91203, + 1402437965.9120300000000000, + 1402437965.912029, + 1402437965.912029999999999999, + utils.Timestamp(1402437965.91203), + utils.Timestamp(1402437965.91203, offset=0), + utils.Timestamp(1402437965.91203, offset=500), + utils.Timestamp(1402437965.912029), + utils.Timestamp(1402437965.91202999999999999, offset=0), + utils.Timestamp(1402437965.91202999999999999, offset=300), + utils.Timestamp('1402437965.91203'), + utils.Timestamp('1402437965.91203', offset=0), + utils.Timestamp('1402437965.91203', offset=400), + utils.Timestamp('1402437965.912029'), + utils.Timestamp('1402437965.912029', offset=0), + utils.Timestamp('1402437965.912029', offset=200), + utils.Timestamp('1402437965.912029999999999'), + utils.Timestamp('1402437965.912029999999999', offset=0), + utils.Timestamp('1402437965.912029999999999', offset=100), + ) + for value in test_values: + timestamp = utils.Timestamp(value) + self.assertEqual(int(timestamp), expected) + self.assertTrue(timestamp > expected) + + def test_float(self): + expected = 1402438115.91203 + test_values = ( + '1402438115.91203', + '1402438115.91203_00000000', + '1402438115.912030000', + '1402438115.912030000_0000000000000', + '000001402438115.912030000', + '000001402438115.912030000_0000000000', + 1402438115.91203, + 1402438115.9120300000000000, + 1402438115.912029, + 1402438115.912029999999999999, + utils.Timestamp(1402438115.91203), + utils.Timestamp(1402438115.91203, offset=0), + utils.Timestamp(1402438115.91203, offset=500), + utils.Timestamp(1402438115.912029), + utils.Timestamp(1402438115.91202999999999999, offset=0), + utils.Timestamp(1402438115.91202999999999999, offset=300), + utils.Timestamp('1402438115.91203'), + utils.Timestamp('1402438115.91203', offset=0), + utils.Timestamp('1402438115.91203', offset=400), + utils.Timestamp('1402438115.912029'), + utils.Timestamp('1402438115.912029', offset=0), + utils.Timestamp('1402438115.912029', offset=200), + utils.Timestamp('1402438115.912029999999999'), + utils.Timestamp('1402438115.912029999999999', offset=0), + utils.Timestamp('1402438115.912029999999999', offset=100), + ) + tolerance = 0.00001 + minimum = expected - tolerance + maximum = expected + tolerance + for value in test_values: + timestamp = utils.Timestamp(value) + self.assertTrue(float(timestamp) > minimum, + '%f is not bigger than %f given %r' % ( + timestamp, minimum, value)) + self.assertTrue(float(timestamp) < maximum, + '%f is not smaller than %f given %r' % ( + timestamp, maximum, value)) + # direct comparision of timestamp works too + self.assertTrue(timestamp > minimum, + '%s is not bigger than %f given %r' % ( + timestamp.normal, minimum, value)) + self.assertTrue(timestamp < maximum, + '%s is not smaller than %f given %r' % ( + timestamp.normal, maximum, value)) + # ... even against strings + self.assertTrue(timestamp > '%f' % minimum, + '%s is not bigger than %s given %r' % ( + timestamp.normal, minimum, value)) + self.assertTrue(timestamp < '%f' % maximum, + '%s is not smaller than %s given %r' % ( + timestamp.normal, maximum, value)) + + def test_false(self): + self.assertFalse(utils.Timestamp(0)) + self.assertFalse(utils.Timestamp(0, offset=0)) + self.assertFalse(utils.Timestamp('0')) + self.assertFalse(utils.Timestamp('0', offset=0)) + self.assertFalse(utils.Timestamp(0.0)) + self.assertFalse(utils.Timestamp(0.0, offset=0)) + self.assertFalse(utils.Timestamp('0.0')) + self.assertFalse(utils.Timestamp('0.0', offset=0)) + self.assertFalse(utils.Timestamp(00000000.00000000)) + self.assertFalse(utils.Timestamp(00000000.00000000, offset=0)) + self.assertFalse(utils.Timestamp('00000000.00000000')) + self.assertFalse(utils.Timestamp('00000000.00000000', offset=0)) + + def test_true(self): + self.assertTrue(utils.Timestamp(1)) + self.assertTrue(utils.Timestamp(1, offset=1)) + self.assertTrue(utils.Timestamp(0, offset=1)) + self.assertTrue(utils.Timestamp('1')) + self.assertTrue(utils.Timestamp('1', offset=1)) + self.assertTrue(utils.Timestamp('0', offset=1)) + self.assertTrue(utils.Timestamp(1.1)) + self.assertTrue(utils.Timestamp(1.1, offset=1)) + self.assertTrue(utils.Timestamp(0.0, offset=1)) + self.assertTrue(utils.Timestamp('1.1')) + self.assertTrue(utils.Timestamp('1.1', offset=1)) + self.assertTrue(utils.Timestamp('0.0', offset=1)) + self.assertTrue(utils.Timestamp(11111111.11111111)) + self.assertTrue(utils.Timestamp(11111111.11111111, offset=1)) + self.assertTrue(utils.Timestamp(00000000.00000000, offset=1)) + self.assertTrue(utils.Timestamp('11111111.11111111')) + self.assertTrue(utils.Timestamp('11111111.11111111', offset=1)) + self.assertTrue(utils.Timestamp('00000000.00000000', offset=1)) + + def test_greater_no_offset(self): + now = time.time() + older = now - 1 + timestamp = utils.Timestamp(now) + test_values = ( + 0, '0', 0.0, '0.0', '0000.0000', '000.000_000', + 1, '1', 1.1, '1.1', '1111.1111', '111.111_111', + 1402443112.213252, '1402443112.213252', '1402443112.213252_ffff', + older, '%f' % older, '%f_0000ffff' % older, + ) + for value in test_values: + other = utils.Timestamp(value) + self.assertNotEqual(timestamp, other) # sanity + self.assertTrue(timestamp > value, + '%r is not greater than %r given %r' % ( + timestamp, value, value)) + self.assertTrue(timestamp > other, + '%r is not greater than %r given %r' % ( + timestamp, other, value)) + self.assertTrue(timestamp > other.normal, + '%r is not greater than %r given %r' % ( + timestamp, other.normal, value)) + self.assertTrue(timestamp > other.internal, + '%r is not greater than %r given %r' % ( + timestamp, other.internal, value)) + self.assertTrue(timestamp > float(other), + '%r is not greater than %r given %r' % ( + timestamp, float(other), value)) + self.assertTrue(timestamp > int(other), + '%r is not greater than %r given %r' % ( + timestamp, int(other), value)) + + def test_greater_with_offset(self): + now = time.time() + older = now - 1 + test_values = ( + 0, '0', 0.0, '0.0', '0000.0000', '000.000_000', + 1, '1', 1.1, '1.1', '1111.1111', '111.111_111', + 1402443346.935174, '1402443346.93517', '1402443346.935169_ffff', + older, '%f' % older, '%f_0000ffff' % older, + now, '%f' % now, '%f_00000000' % now, + ) + for offset in range(1, 1000, 100): + timestamp = utils.Timestamp(now, offset=offset) + for value in test_values: + other = utils.Timestamp(value) + self.assertNotEqual(timestamp, other) # sanity + self.assertTrue(timestamp > value, + '%r is not greater than %r given %r' % ( + timestamp, value, value)) + self.assertTrue(timestamp > other, + '%r is not greater than %r given %r' % ( + timestamp, other, value)) + self.assertTrue(timestamp > other.normal, + '%r is not greater than %r given %r' % ( + timestamp, other.normal, value)) + self.assertTrue(timestamp > other.internal, + '%r is not greater than %r given %r' % ( + timestamp, other.internal, value)) + self.assertTrue(timestamp > float(other), + '%r is not greater than %r given %r' % ( + timestamp, float(other), value)) + self.assertTrue(timestamp > int(other), + '%r is not greater than %r given %r' % ( + timestamp, int(other), value)) + + def test_smaller_no_offset(self): + now = time.time() + newer = now + 1 + timestamp = utils.Timestamp(now) + test_values = ( + 9999999999.99999, '9999999999.99999', '9999999999.99999_ffff', + newer, '%f' % newer, '%f_0000ffff' % newer, + ) + for value in test_values: + other = utils.Timestamp(value) + self.assertNotEqual(timestamp, other) # sanity + self.assertTrue(timestamp < value, + '%r is not smaller than %r given %r' % ( + timestamp, value, value)) + self.assertTrue(timestamp < other, + '%r is not smaller than %r given %r' % ( + timestamp, other, value)) + self.assertTrue(timestamp < other.normal, + '%r is not smaller than %r given %r' % ( + timestamp, other.normal, value)) + self.assertTrue(timestamp < other.internal, + '%r is not smaller than %r given %r' % ( + timestamp, other.internal, value)) + self.assertTrue(timestamp < float(other), + '%r is not smaller than %r given %r' % ( + timestamp, float(other), value)) + self.assertTrue(timestamp < int(other), + '%r is not smaller than %r given %r' % ( + timestamp, int(other), value)) + + def test_smaller_with_offset(self): + now = time.time() + newer = now + 1 + test_values = ( + 9999999999.99999, '9999999999.99999', '9999999999.99999_ffff', + newer, '%f' % newer, '%f_0000ffff' % newer, + ) + for offset in range(1, 1000, 100): + timestamp = utils.Timestamp(now, offset=offset) + for value in test_values: + other = utils.Timestamp(value) + self.assertNotEqual(timestamp, other) # sanity + self.assertTrue(timestamp < value, + '%r is not smaller than %r given %r' % ( + timestamp, value, value)) + self.assertTrue(timestamp < other, + '%r is not smaller than %r given %r' % ( + timestamp, other, value)) + self.assertTrue(timestamp < other.normal, + '%r is not smaller than %r given %r' % ( + timestamp, other.normal, value)) + self.assertTrue(timestamp < other.internal, + '%r is not smaller than %r given %r' % ( + timestamp, other.internal, value)) + self.assertTrue(timestamp < float(other), + '%r is not smaller than %r given %r' % ( + timestamp, float(other), value)) + self.assertTrue(timestamp < int(other), + '%r is not smaller than %r given %r' % ( + timestamp, int(other), value)) + + def test_ordering(self): + given = [ + '1402444820.62590_000000000000000a', + '1402444820.62589_0000000000000001', + '1402444821.52589_0000000000000004', + '1402444920.62589_0000000000000004', + '1402444821.62589_000000000000000a', + '1402444821.72589_000000000000000a', + '1402444920.62589_0000000000000002', + '1402444820.62589_0000000000000002', + '1402444820.62589_000000000000000a', + '1402444820.62590_0000000000000004', + '1402444920.62589_000000000000000a', + '1402444820.62590_0000000000000002', + '1402444821.52589_0000000000000002', + '1402444821.52589_0000000000000000', + '1402444920.62589', + '1402444821.62589_0000000000000004', + '1402444821.72589_0000000000000001', + '1402444820.62590', + '1402444820.62590_0000000000000001', + '1402444820.62589_0000000000000004', + '1402444821.72589_0000000000000000', + '1402444821.52589_000000000000000a', + '1402444821.72589_0000000000000004', + '1402444821.62589', + '1402444821.52589_0000000000000001', + '1402444821.62589_0000000000000001', + '1402444821.62589_0000000000000002', + '1402444821.72589_0000000000000002', + '1402444820.62589', + '1402444920.62589_0000000000000001'] + expected = [ + '1402444820.62589', + '1402444820.62589_0000000000000001', + '1402444820.62589_0000000000000002', + '1402444820.62589_0000000000000004', + '1402444820.62589_000000000000000a', + '1402444820.62590', + '1402444820.62590_0000000000000001', + '1402444820.62590_0000000000000002', + '1402444820.62590_0000000000000004', + '1402444820.62590_000000000000000a', + '1402444821.52589', + '1402444821.52589_0000000000000001', + '1402444821.52589_0000000000000002', + '1402444821.52589_0000000000000004', + '1402444821.52589_000000000000000a', + '1402444821.62589', + '1402444821.62589_0000000000000001', + '1402444821.62589_0000000000000002', + '1402444821.62589_0000000000000004', + '1402444821.62589_000000000000000a', + '1402444821.72589', + '1402444821.72589_0000000000000001', + '1402444821.72589_0000000000000002', + '1402444821.72589_0000000000000004', + '1402444821.72589_000000000000000a', + '1402444920.62589', + '1402444920.62589_0000000000000001', + '1402444920.62589_0000000000000002', + '1402444920.62589_0000000000000004', + '1402444920.62589_000000000000000a', + ] + # less visual version + """ + now = time.time() + given = [ + utils.Timestamp(now + i, offset=offset).internal + for i in (0, 0.00001, 0.9, 1.0, 1.1, 100.0) + for offset in (0, 1, 2, 4, 10) + ] + expected = [t for t in given] + random.shuffle(given) + """ + self.assertEqual(len(given), len(expected)) # sanity + timestamps = [utils.Timestamp(t) for t in given] + # our expected values don't include insignificant offsets + with mock.patch('swift.common.utils.FORCE_INTERNAL', new=False): + self.assertEqual( + [t.internal for t in sorted(timestamps)], expected) + # string sorting works as well + self.assertEqual( + sorted([t.internal for t in timestamps]), expected) + + class TestUtils(unittest.TestCase): """Tests for swift.common.utils """ diff --git a/test/unit/container/test_backend.py b/test/unit/container/test_backend.py index 41b0c3f10c..c2d7208ba1 100644 --- a/test/unit/container/test_backend.py +++ b/test/unit/container/test_backend.py @@ -28,7 +28,7 @@ import sqlite3 import pickle from swift.container.backend import ContainerBroker -from swift.common.utils import normalize_timestamp +from swift.common.utils import Timestamp from swift.common.storage_policy import POLICIES import mock @@ -44,7 +44,7 @@ class TestContainerBroker(unittest.TestCase): # Test ContainerBroker.__init__ broker = ContainerBroker(':memory:', account='a', container='c') self.assertEqual(broker.db_file, ':memory:') - broker.initialize(normalize_timestamp('1'), 0) + broker.initialize(Timestamp('1').internal, 0) with broker.get() as conn: curs = conn.cursor() curs.execute('SELECT 1') @@ -52,11 +52,11 @@ class TestContainerBroker(unittest.TestCase): @patch_policies def test_storage_policy_property(self): - ts = itertools.count(1) + ts = (Timestamp(t).internal for t in itertools.count(int(time()))) for policy in POLICIES: broker = ContainerBroker(':memory:', account='a', container='policy_%s' % policy.name) - broker.initialize(normalize_timestamp(ts.next()), policy.idx) + broker.initialize(ts.next(), policy.idx) with broker.get() as conn: try: conn.execute('''SELECT storage_policy_index @@ -78,7 +78,7 @@ class TestContainerBroker(unittest.TestCase): # unhandled exception first_conn = None broker = ContainerBroker(':memory:', account='a', container='c') - broker.initialize(normalize_timestamp('1'), 0) + broker.initialize(Timestamp('1').internal, 0) with broker.get() as conn: first_conn = conn try: @@ -92,20 +92,20 @@ class TestContainerBroker(unittest.TestCase): def test_empty(self): # Test ContainerBroker.empty broker = ContainerBroker(':memory:', account='a', container='c') - broker.initialize(normalize_timestamp('1'), 0) + broker.initialize(Timestamp('1').internal, 0) self.assert_(broker.empty()) - broker.put_object('o', normalize_timestamp(time()), 0, 'text/plain', + broker.put_object('o', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') self.assert_(not broker.empty()) sleep(.00001) - broker.delete_object('o', normalize_timestamp(time())) + broker.delete_object('o', Timestamp(time()).internal) self.assert_(broker.empty()) def test_reclaim(self): broker = ContainerBroker(':memory:', account='test_account', container='test_container') - broker.initialize(normalize_timestamp('1'), 0) - broker.put_object('o', normalize_timestamp(time()), 0, 'text/plain', + broker.initialize(Timestamp('1').internal, 0) + broker.put_object('o', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') with broker.get() as conn: self.assertEquals(conn.execute( @@ -114,7 +114,7 @@ class TestContainerBroker(unittest.TestCase): self.assertEquals(conn.execute( "SELECT count(*) FROM object " "WHERE deleted = 1").fetchone()[0], 0) - broker.reclaim(normalize_timestamp(time() - 999), time()) + broker.reclaim(Timestamp(time() - 999).internal, time()) with broker.get() as conn: self.assertEquals(conn.execute( "SELECT count(*) FROM object " @@ -123,7 +123,7 @@ class TestContainerBroker(unittest.TestCase): "SELECT count(*) FROM object " "WHERE deleted = 1").fetchone()[0], 0) sleep(.00001) - broker.delete_object('o', normalize_timestamp(time())) + broker.delete_object('o', Timestamp(time()).internal) with broker.get() as conn: self.assertEquals(conn.execute( "SELECT count(*) FROM object " @@ -131,7 +131,7 @@ class TestContainerBroker(unittest.TestCase): self.assertEquals(conn.execute( "SELECT count(*) FROM object " "WHERE deleted = 1").fetchone()[0], 1) - broker.reclaim(normalize_timestamp(time() - 999), time()) + broker.reclaim(Timestamp(time() - 999).internal, time()) with broker.get() as conn: self.assertEquals(conn.execute( "SELECT count(*) FROM object " @@ -140,7 +140,7 @@ class TestContainerBroker(unittest.TestCase): "SELECT count(*) FROM object " "WHERE deleted = 1").fetchone()[0], 1) sleep(.00001) - broker.reclaim(normalize_timestamp(time()), time()) + broker.reclaim(Timestamp(time()).internal, time()) with broker.get() as conn: self.assertEquals(conn.execute( "SELECT count(*) FROM object " @@ -149,21 +149,21 @@ class TestContainerBroker(unittest.TestCase): "SELECT count(*) FROM object " "WHERE deleted = 1").fetchone()[0], 0) # Test the return values of reclaim() - broker.put_object('w', normalize_timestamp(time()), 0, 'text/plain', + broker.put_object('w', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('x', normalize_timestamp(time()), 0, 'text/plain', + broker.put_object('x', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('y', normalize_timestamp(time()), 0, 'text/plain', + broker.put_object('y', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('z', normalize_timestamp(time()), 0, 'text/plain', + broker.put_object('z', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') # Test before deletion - broker.reclaim(normalize_timestamp(time()), time()) - broker.delete_db(normalize_timestamp(time())) + broker.reclaim(Timestamp(time()).internal, time()) + broker.delete_db(Timestamp(time()).internal) def test_get_info_is_deleted(self): start = int(time()) - ts = (normalize_timestamp(t) for t in itertools.count(start)) + ts = (Timestamp(t).internal for t in itertools.count(start)) broker = ContainerBroker(':memory:', account='test_account', container='test_container') # create it @@ -172,8 +172,8 @@ class TestContainerBroker(unittest.TestCase): self.assertEqual(is_deleted, broker.is_deleted()) self.assertEqual(is_deleted, False) # sanity self.assertEqual(info, broker.get_info()) - self.assertEqual(info['put_timestamp'], normalize_timestamp(start)) - self.assert_(float(info['created_at']) >= start) + self.assertEqual(info['put_timestamp'], Timestamp(start).internal) + self.assert_(Timestamp(info['created_at']) >= start) self.assertEqual(info['delete_timestamp'], '0') if self.__class__ in (TestContainerBrokerBeforeMetadata, TestContainerBrokerBeforeXSync, @@ -181,7 +181,7 @@ class TestContainerBroker(unittest.TestCase): self.assertEqual(info['status_changed_at'], '0') else: self.assertEqual(info['status_changed_at'], - normalize_timestamp(start)) + Timestamp(start).internal) # delete it delete_timestamp = ts.next() @@ -190,8 +190,8 @@ class TestContainerBroker(unittest.TestCase): self.assertEqual(is_deleted, True) # sanity self.assertEqual(is_deleted, broker.is_deleted()) self.assertEqual(info, broker.get_info()) - self.assertEqual(info['put_timestamp'], normalize_timestamp(start)) - self.assert_(float(info['created_at']) >= start) + self.assertEqual(info['put_timestamp'], Timestamp(start).internal) + self.assert_(Timestamp(info['created_at']) >= start) self.assertEqual(info['delete_timestamp'], delete_timestamp) self.assertEqual(info['status_changed_at'], delete_timestamp) @@ -202,16 +202,16 @@ class TestContainerBroker(unittest.TestCase): self.assertEqual(is_deleted, False) # sanity self.assertEqual(is_deleted, broker.is_deleted()) self.assertEqual(info, broker.get_info()) - self.assertEqual(info['put_timestamp'], normalize_timestamp(start)) - self.assert_(float(info['created_at']) >= start) + self.assertEqual(info['put_timestamp'], Timestamp(start).internal) + self.assert_(Timestamp(info['created_at']) >= start) self.assertEqual(info['delete_timestamp'], delete_timestamp) self.assertEqual(info['status_changed_at'], delete_timestamp) def test_delete_object(self): # Test ContainerBroker.delete_object broker = ContainerBroker(':memory:', account='a', container='c') - broker.initialize(normalize_timestamp('1'), 0) - broker.put_object('o', normalize_timestamp(time()), 0, 'text/plain', + broker.initialize(Timestamp('1').internal, 0) + broker.put_object('o', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') with broker.get() as conn: self.assertEquals(conn.execute( @@ -221,7 +221,7 @@ class TestContainerBroker(unittest.TestCase): "SELECT count(*) FROM object " "WHERE deleted = 1").fetchone()[0], 0) sleep(.00001) - broker.delete_object('o', normalize_timestamp(time())) + broker.delete_object('o', Timestamp(time()).internal) with broker.get() as conn: self.assertEquals(conn.execute( "SELECT count(*) FROM object " @@ -233,10 +233,10 @@ class TestContainerBroker(unittest.TestCase): def test_put_object(self): # Test ContainerBroker.put_object broker = ContainerBroker(':memory:', account='a', container='c') - broker.initialize(normalize_timestamp('1'), 0) + broker.initialize(Timestamp('1').internal, 0) # Create initial object - timestamp = normalize_timestamp(time()) + timestamp = Timestamp(time()).internal broker.put_object('"{}"', timestamp, 123, 'application/x-test', '5af83e3196bf99f440f31f2e1a6c9afe') @@ -280,7 +280,7 @@ class TestContainerBroker(unittest.TestCase): # Put new event sleep(.00001) - timestamp = normalize_timestamp(time()) + timestamp = Timestamp(time()).internal broker.put_object('"{}"', timestamp, 124, 'application/x-test', 'aa0749bacbc79ec65fe206943d8fe449') @@ -302,7 +302,7 @@ class TestContainerBroker(unittest.TestCase): "SELECT deleted FROM object").fetchone()[0], 0) # Put old event - otimestamp = normalize_timestamp(float(timestamp) - 1) + otimestamp = Timestamp(float(Timestamp(timestamp)) - 1).internal broker.put_object('"{}"', otimestamp, 124, 'application/x-test', 'aa0749bacbc79ec65fe206943d8fe449') @@ -324,7 +324,7 @@ class TestContainerBroker(unittest.TestCase): "SELECT deleted FROM object").fetchone()[0], 0) # Put old delete event - dtimestamp = normalize_timestamp(float(timestamp) - 1) + dtimestamp = Timestamp(float(Timestamp(timestamp)) - 1).internal broker.put_object('"{}"', dtimestamp, 0, '', '', deleted=1) with broker.get() as conn: @@ -346,7 +346,7 @@ class TestContainerBroker(unittest.TestCase): # Put new delete event sleep(.00001) - timestamp = normalize_timestamp(time()) + timestamp = Timestamp(time()).internal broker.put_object('"{}"', timestamp, 0, '', '', deleted=1) with broker.get() as conn: @@ -360,7 +360,7 @@ class TestContainerBroker(unittest.TestCase): # Put new event sleep(.00001) - timestamp = normalize_timestamp(time()) + timestamp = Timestamp(time()).internal broker.put_object('"{}"', timestamp, 123, 'application/x-test', '5af83e3196bf99f440f31f2e1a6c9afe') @@ -383,12 +383,12 @@ class TestContainerBroker(unittest.TestCase): # We'll use this later sleep(.0001) - in_between_timestamp = normalize_timestamp(time()) + in_between_timestamp = Timestamp(time()).internal # New post event sleep(.0001) previous_timestamp = timestamp - timestamp = normalize_timestamp(time()) + timestamp = Timestamp(time()).internal with broker.get() as conn: self.assertEquals(conn.execute( "SELECT name FROM object").fetchone()[0], @@ -432,7 +432,7 @@ class TestContainerBroker(unittest.TestCase): @patch_policies def test_put_misplaced_object_does_not_effect_container_stats(self): policy = random.choice(list(POLICIES)) - ts = (normalize_timestamp(t) for t in + ts = (Timestamp(t).internal for t in itertools.count(int(time()))) broker = ContainerBroker(':memory:', account='a', container='c') @@ -460,7 +460,7 @@ class TestContainerBroker(unittest.TestCase): @patch_policies def test_has_multiple_policies(self): policy = random.choice(list(POLICIES)) - ts = (normalize_timestamp(t) for t in + ts = (Timestamp(t).internal for t in itertools.count(int(time()))) broker = ContainerBroker(':memory:', account='a', container='c') @@ -484,7 +484,7 @@ class TestContainerBroker(unittest.TestCase): @patch_policies def test_get_policy_info(self): policy = random.choice(list(POLICIES)) - ts = (normalize_timestamp(t) for t in + ts = (Timestamp(t).internal for t in itertools.count(int(time()))) broker = ContainerBroker(':memory:', account='a', container='c') @@ -521,7 +521,7 @@ class TestContainerBroker(unittest.TestCase): self.assertEqual(policy_stats, expected) def test_policy_stat_tracking(self): - ts = (normalize_timestamp(t) for t in + ts = (Timestamp(t).internal for t in itertools.count(int(time()))) broker = ContainerBroker(':memory:', account='a', container='c') @@ -558,13 +558,13 @@ class TestContainerBroker(unittest.TestCase): # Test ContainerBroker.get_info broker = ContainerBroker(':memory:', account='test1', container='test2') - broker.initialize(normalize_timestamp('1'), 0) + broker.initialize(Timestamp('1').internal, 0) info = broker.get_info() self.assertEquals(info['account'], 'test1') self.assertEquals(info['container'], 'test2') self.assertEquals(info['hash'], '00000000000000000000000000000000') - self.assertEqual(info['put_timestamp'], normalize_timestamp(1)) + self.assertEqual(info['put_timestamp'], Timestamp(1).internal) self.assertEqual(info['delete_timestamp'], '0') if self.__class__ in (TestContainerBrokerBeforeMetadata, TestContainerBrokerBeforeXSync, @@ -572,40 +572,40 @@ class TestContainerBroker(unittest.TestCase): self.assertEqual(info['status_changed_at'], '0') else: self.assertEqual(info['status_changed_at'], - normalize_timestamp(1)) + Timestamp(1).internal) info = broker.get_info() self.assertEquals(info['object_count'], 0) self.assertEquals(info['bytes_used'], 0) - broker.put_object('o1', normalize_timestamp(time()), 123, 'text/plain', + broker.put_object('o1', Timestamp(time()).internal, 123, 'text/plain', '5af83e3196bf99f440f31f2e1a6c9afe') info = broker.get_info() self.assertEquals(info['object_count'], 1) self.assertEquals(info['bytes_used'], 123) sleep(.00001) - broker.put_object('o2', normalize_timestamp(time()), 123, 'text/plain', + broker.put_object('o2', Timestamp(time()).internal, 123, 'text/plain', '5af83e3196bf99f440f31f2e1a6c9afe') info = broker.get_info() self.assertEquals(info['object_count'], 2) self.assertEquals(info['bytes_used'], 246) sleep(.00001) - broker.put_object('o2', normalize_timestamp(time()), 1000, + broker.put_object('o2', Timestamp(time()).internal, 1000, 'text/plain', '5af83e3196bf99f440f31f2e1a6c9afe') info = broker.get_info() self.assertEquals(info['object_count'], 2) self.assertEquals(info['bytes_used'], 1123) sleep(.00001) - broker.delete_object('o1', normalize_timestamp(time())) + broker.delete_object('o1', Timestamp(time()).internal) info = broker.get_info() self.assertEquals(info['object_count'], 1) self.assertEquals(info['bytes_used'], 1000) sleep(.00001) - broker.delete_object('o2', normalize_timestamp(time())) + broker.delete_object('o2', Timestamp(time()).internal) info = broker.get_info() self.assertEquals(info['object_count'], 0) self.assertEquals(info['bytes_used'], 0) @@ -617,7 +617,7 @@ class TestContainerBroker(unittest.TestCase): def test_set_x_syncs(self): broker = ContainerBroker(':memory:', account='test1', container='test2') - broker.initialize(normalize_timestamp('1'), 0) + broker.initialize(Timestamp('1').internal, 0) info = broker.get_info() self.assertEquals(info['x_container_sync_point1'], -1) @@ -631,7 +631,7 @@ class TestContainerBroker(unittest.TestCase): def test_get_report_info(self): broker = ContainerBroker(':memory:', account='test1', container='test2') - broker.initialize(normalize_timestamp('1'), 0) + broker.initialize(Timestamp('1').internal, 0) info = broker.get_info() self.assertEquals(info['account'], 'test1') @@ -641,7 +641,7 @@ class TestContainerBroker(unittest.TestCase): self.assertEquals(info['reported_object_count'], 0) self.assertEquals(info['reported_bytes_used'], 0) - broker.put_object('o1', normalize_timestamp(time()), 123, 'text/plain', + broker.put_object('o1', Timestamp(time()).internal, 123, 'text/plain', '5af83e3196bf99f440f31f2e1a6c9afe') info = broker.get_info() self.assertEquals(info['object_count'], 1) @@ -650,7 +650,7 @@ class TestContainerBroker(unittest.TestCase): self.assertEquals(info['reported_bytes_used'], 0) sleep(.00001) - broker.put_object('o2', normalize_timestamp(time()), 123, 'text/plain', + broker.put_object('o2', Timestamp(time()).internal, 123, 'text/plain', '5af83e3196bf99f440f31f2e1a6c9afe') info = broker.get_info() self.assertEquals(info['object_count'], 2) @@ -659,7 +659,7 @@ class TestContainerBroker(unittest.TestCase): self.assertEquals(info['reported_bytes_used'], 0) sleep(.00001) - broker.put_object('o2', normalize_timestamp(time()), 1000, + broker.put_object('o2', Timestamp(time()).internal, 1000, 'text/plain', '5af83e3196bf99f440f31f2e1a6c9afe') info = broker.get_info() self.assertEquals(info['object_count'], 2) @@ -667,9 +667,9 @@ class TestContainerBroker(unittest.TestCase): self.assertEquals(info['reported_object_count'], 0) self.assertEquals(info['reported_bytes_used'], 0) - put_timestamp = normalize_timestamp(time()) + put_timestamp = Timestamp(time()).internal sleep(.001) - delete_timestamp = normalize_timestamp(time()) + delete_timestamp = Timestamp(time()).internal broker.reported(put_timestamp, delete_timestamp, 2, 1123) info = broker.get_info() self.assertEquals(info['object_count'], 2) @@ -680,7 +680,7 @@ class TestContainerBroker(unittest.TestCase): self.assertEquals(info['reported_bytes_used'], 1123) sleep(.00001) - broker.delete_object('o1', normalize_timestamp(time())) + broker.delete_object('o1', Timestamp(time()).internal) info = broker.get_info() self.assertEquals(info['object_count'], 1) self.assertEquals(info['bytes_used'], 1000) @@ -688,7 +688,7 @@ class TestContainerBroker(unittest.TestCase): self.assertEquals(info['reported_bytes_used'], 1123) sleep(.00001) - broker.delete_object('o2', normalize_timestamp(time())) + broker.delete_object('o2', Timestamp(time()).internal) info = broker.get_info() self.assertEquals(info['object_count'], 0) self.assertEquals(info['bytes_used'], 0) @@ -698,20 +698,20 @@ class TestContainerBroker(unittest.TestCase): def test_list_objects_iter(self): # Test ContainerBroker.list_objects_iter broker = ContainerBroker(':memory:', account='a', container='c') - broker.initialize(normalize_timestamp('1'), 0) + broker.initialize(Timestamp('1').internal, 0) for obj1 in xrange(4): for obj2 in xrange(125): broker.put_object('%d/%04d' % (obj1, obj2), - normalize_timestamp(time()), 0, 'text/plain', + Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') for obj in xrange(125): broker.put_object('2/0051/%04d' % obj, - normalize_timestamp(time()), 0, 'text/plain', + Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') for obj in xrange(125): broker.put_object('3/%04d/0049' % obj, - normalize_timestamp(time()), 0, 'text/plain', + Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') listing = broker.list_objects_iter(100, '', None, None, '') @@ -777,7 +777,7 @@ class TestContainerBroker(unittest.TestCase): '3/0047/', '3/0048', '3/0048/', '3/0049', '3/0049/', '3/0050']) - broker.put_object('3/0049/', normalize_timestamp(time()), 0, + broker.put_object('3/0049/', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') listing = broker.list_objects_iter(10, '3/0048', None, None, None) self.assertEquals(len(listing), 10) @@ -817,20 +817,20 @@ class TestContainerBroker(unittest.TestCase): # Test ContainerBroker.list_objects_iter using a # delimiter that is not a slash broker = ContainerBroker(':memory:', account='a', container='c') - broker.initialize(normalize_timestamp('1'), 0) + broker.initialize(Timestamp('1').internal, 0) for obj1 in xrange(4): for obj2 in xrange(125): broker.put_object('%d:%04d' % (obj1, obj2), - normalize_timestamp(time()), 0, 'text/plain', + Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') for obj in xrange(125): broker.put_object('2:0051:%04d' % obj, - normalize_timestamp(time()), 0, 'text/plain', + Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') for obj in xrange(125): broker.put_object('3:%04d:0049' % obj, - normalize_timestamp(time()), 0, 'text/plain', + Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') listing = broker.list_objects_iter(100, '', None, None, '') @@ -895,7 +895,7 @@ class TestContainerBroker(unittest.TestCase): '3:0047:', '3:0048', '3:0048:', '3:0049', '3:0049:', '3:0050']) - broker.put_object('3:0049:', normalize_timestamp(time()), 0, + broker.put_object('3:0049:', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') listing = broker.list_objects_iter(10, '3:0048', None, None, None) self.assertEquals(len(listing), 10) @@ -934,25 +934,25 @@ class TestContainerBroker(unittest.TestCase): def test_list_objects_iter_prefix_delim(self): # Test ContainerBroker.list_objects_iter broker = ContainerBroker(':memory:', account='a', container='c') - broker.initialize(normalize_timestamp('1'), 0) + broker.initialize(Timestamp('1').internal, 0) broker.put_object( - '/pets/dogs/1', normalize_timestamp(0), 0, + '/pets/dogs/1', Timestamp(0).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker.put_object( - '/pets/dogs/2', normalize_timestamp(0), 0, + '/pets/dogs/2', Timestamp(0).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker.put_object( - '/pets/fish/a', normalize_timestamp(0), 0, + '/pets/fish/a', Timestamp(0).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker.put_object( - '/pets/fish/b', normalize_timestamp(0), 0, + '/pets/fish/b', Timestamp(0).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker.put_object( - '/pets/fish_info.txt', normalize_timestamp(0), 0, + '/pets/fish_info.txt', Timestamp(0).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker.put_object( - '/snakes', normalize_timestamp(0), 0, + '/snakes', Timestamp(0).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') #def list_objects_iter(self, limit, marker, prefix, delimiter, @@ -971,50 +971,50 @@ class TestContainerBroker(unittest.TestCase): # Test ContainerBroker.list_objects_iter for a # container that has an odd file with a trailing delimiter broker = ContainerBroker(':memory:', account='a', container='c') - broker.initialize(normalize_timestamp('1'), 0) - broker.put_object('a', normalize_timestamp(time()), 0, + broker.initialize(Timestamp('1').internal, 0) + broker.put_object('a', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('a/', normalize_timestamp(time()), 0, + broker.put_object('a/', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('a/a', normalize_timestamp(time()), 0, + broker.put_object('a/a', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('a/a/a', normalize_timestamp(time()), 0, + broker.put_object('a/a/a', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('a/a/b', normalize_timestamp(time()), 0, + broker.put_object('a/a/b', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('a/b', normalize_timestamp(time()), 0, + broker.put_object('a/b', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('b', normalize_timestamp(time()), 0, + broker.put_object('b', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('b/a', normalize_timestamp(time()), 0, + broker.put_object('b/a', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('b/b', normalize_timestamp(time()), 0, + broker.put_object('b/b', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('c', normalize_timestamp(time()), 0, + broker.put_object('c', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('a/0', normalize_timestamp(time()), 0, + broker.put_object('a/0', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('0', normalize_timestamp(time()), 0, + broker.put_object('0', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('0/', normalize_timestamp(time()), 0, + broker.put_object('0/', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('00', normalize_timestamp(time()), 0, + broker.put_object('00', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('0/0', normalize_timestamp(time()), 0, + broker.put_object('0/0', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('0/00', normalize_timestamp(time()), 0, + broker.put_object('0/00', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('0/1', normalize_timestamp(time()), 0, + broker.put_object('0/1', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('0/1/', normalize_timestamp(time()), 0, + broker.put_object('0/1/', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('0/1/0', normalize_timestamp(time()), 0, + broker.put_object('0/1/0', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('1', normalize_timestamp(time()), 0, + broker.put_object('1', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('1/', normalize_timestamp(time()), 0, + broker.put_object('1/', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('1/0', normalize_timestamp(time()), 0, + broker.put_object('1/0', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') listing = broker.list_objects_iter(25, None, None, None, None) self.assertEquals(len(listing), 22) @@ -1051,50 +1051,50 @@ class TestContainerBroker(unittest.TestCase): # Test ContainerBroker.list_objects_iter for a # container that has an odd file with a trailing delimiter broker = ContainerBroker(':memory:', account='a', container='c') - broker.initialize(normalize_timestamp('1'), 0) - broker.put_object('a', normalize_timestamp(time()), 0, + broker.initialize(Timestamp('1').internal, 0) + broker.put_object('a', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('a:', normalize_timestamp(time()), 0, + broker.put_object('a:', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('a:a', normalize_timestamp(time()), 0, + broker.put_object('a:a', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('a:a:a', normalize_timestamp(time()), 0, + broker.put_object('a:a:a', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('a:a:b', normalize_timestamp(time()), 0, + broker.put_object('a:a:b', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('a:b', normalize_timestamp(time()), 0, + broker.put_object('a:b', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('b', normalize_timestamp(time()), 0, + broker.put_object('b', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('b:a', normalize_timestamp(time()), 0, + broker.put_object('b:a', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('b:b', normalize_timestamp(time()), 0, + broker.put_object('b:b', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('c', normalize_timestamp(time()), 0, + broker.put_object('c', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('a:0', normalize_timestamp(time()), 0, + broker.put_object('a:0', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('0', normalize_timestamp(time()), 0, + broker.put_object('0', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('0:', normalize_timestamp(time()), 0, + broker.put_object('0:', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('00', normalize_timestamp(time()), 0, + broker.put_object('00', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('0:0', normalize_timestamp(time()), 0, + broker.put_object('0:0', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('0:00', normalize_timestamp(time()), 0, + broker.put_object('0:00', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('0:1', normalize_timestamp(time()), 0, + broker.put_object('0:1', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('0:1:', normalize_timestamp(time()), 0, + broker.put_object('0:1:', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('0:1:0', normalize_timestamp(time()), 0, + broker.put_object('0:1:0', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('1', normalize_timestamp(time()), 0, + broker.put_object('1', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('1:', normalize_timestamp(time()), 0, + broker.put_object('1:', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('1:0', normalize_timestamp(time()), 0, + broker.put_object('1:0', Timestamp(time()).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') listing = broker.list_objects_iter(25, None, None, None, None) self.assertEquals(len(listing), 22) @@ -1129,19 +1129,19 @@ class TestContainerBroker(unittest.TestCase): def test_chexor(self): broker = ContainerBroker(':memory:', account='a', container='c') - broker.initialize(normalize_timestamp('1'), 0) - broker.put_object('a', normalize_timestamp(1), 0, + broker.initialize(Timestamp('1').internal, 0) + broker.put_object('a', Timestamp(1).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('b', normalize_timestamp(2), 0, + broker.put_object('b', Timestamp(2).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - hasha = hashlib.md5('%s-%s' % ('a', '0000000001.00000')).digest() - hashb = hashlib.md5('%s-%s' % ('b', '0000000002.00000')).digest() + hasha = hashlib.md5('%s-%s' % ('a', Timestamp(1).internal)).digest() + hashb = hashlib.md5('%s-%s' % ('b', Timestamp(2).internal)).digest() hashc = ''.join( - ('%2x' % (ord(a) ^ ord(b)) for a, b in zip(hasha, hashb))) + ('%02x' % (ord(a) ^ ord(b)) for a, b in zip(hasha, hashb))) self.assertEquals(broker.get_info()['hash'], hashc) - broker.put_object('b', normalize_timestamp(3), 0, + broker.put_object('b', Timestamp(3).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - hashb = hashlib.md5('%s-%s' % ('b', '0000000003.00000')).digest() + hashb = hashlib.md5('%s-%s' % ('b', Timestamp(3).internal)).digest() hashc = ''.join( ('%02x' % (ord(a) ^ ord(b)) for a, b in zip(hasha, hashb))) self.assertEquals(broker.get_info()['hash'], hashc) @@ -1149,7 +1149,7 @@ class TestContainerBroker(unittest.TestCase): def test_newid(self): # test DatabaseBroker.newid broker = ContainerBroker(':memory:', account='a', container='c') - broker.initialize(normalize_timestamp('1'), 0) + broker.initialize(Timestamp('1').internal, 0) id = broker.get_info()['id'] broker.newid('someid') self.assertNotEquals(id, broker.get_info()['id']) @@ -1157,11 +1157,11 @@ class TestContainerBroker(unittest.TestCase): def test_get_items_since(self): # test DatabaseBroker.get_items_since broker = ContainerBroker(':memory:', account='a', container='c') - broker.initialize(normalize_timestamp('1'), 0) - broker.put_object('a', normalize_timestamp(1), 0, + broker.initialize(Timestamp('1').internal, 0) + broker.put_object('a', Timestamp(1).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') max_row = broker.get_replication_info()['max_row'] - broker.put_object('b', normalize_timestamp(2), 0, + broker.put_object('b', Timestamp(2).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') items = broker.get_items_since(max_row, 1000) self.assertEquals(len(items), 1) @@ -1170,9 +1170,9 @@ class TestContainerBroker(unittest.TestCase): def test_sync_merging(self): # exercise the DatabaseBroker sync functions a bit broker1 = ContainerBroker(':memory:', account='a', container='c') - broker1.initialize(normalize_timestamp('1'), 0) + broker1.initialize(Timestamp('1').internal, 0) broker2 = ContainerBroker(':memory:', account='a', container='c') - broker2.initialize(normalize_timestamp('1'), 0) + broker2.initialize(Timestamp('1').internal, 0) self.assertEquals(broker2.get_sync('12345'), -1) broker1.merge_syncs([{'sync_point': 3, 'remote_id': '12345'}]) broker2.merge_syncs(broker1.get_syncs()) @@ -1180,12 +1180,12 @@ class TestContainerBroker(unittest.TestCase): def test_merge_items(self): broker1 = ContainerBroker(':memory:', account='a', container='c') - broker1.initialize(normalize_timestamp('1'), 0) + broker1.initialize(Timestamp('1').internal, 0) broker2 = ContainerBroker(':memory:', account='a', container='c') - broker2.initialize(normalize_timestamp('1'), 0) - broker1.put_object('a', normalize_timestamp(1), 0, + broker2.initialize(Timestamp('1').internal, 0) + broker1.put_object('a', Timestamp(1).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker1.put_object('b', normalize_timestamp(2), 0, + broker1.put_object('b', Timestamp(2).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') id = broker1.get_info()['id'] broker2.merge_items(broker1.get_items_since( @@ -1193,7 +1193,7 @@ class TestContainerBroker(unittest.TestCase): items = broker2.get_items_since(-1, 1000) self.assertEquals(len(items), 2) self.assertEquals(['a', 'b'], sorted([rec['name'] for rec in items])) - broker1.put_object('c', normalize_timestamp(3), 0, + broker1.put_object('c', Timestamp(3).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker2.merge_items(broker1.get_items_since( broker2.get_sync(id), 1000), id) @@ -1205,17 +1205,17 @@ class TestContainerBroker(unittest.TestCase): def test_merge_items_overwrite(self): # test DatabaseBroker.merge_items broker1 = ContainerBroker(':memory:', account='a', container='c') - broker1.initialize(normalize_timestamp('1'), 0) + broker1.initialize(Timestamp('1').internal, 0) id = broker1.get_info()['id'] broker2 = ContainerBroker(':memory:', account='a', container='c') - broker2.initialize(normalize_timestamp('1'), 0) - broker1.put_object('a', normalize_timestamp(2), 0, + broker2.initialize(Timestamp('1').internal, 0) + broker1.put_object('a', Timestamp(2).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker1.put_object('b', normalize_timestamp(3), 0, + broker1.put_object('b', Timestamp(3).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker2.merge_items(broker1.get_items_since( broker2.get_sync(id), 1000), id) - broker1.put_object('a', normalize_timestamp(4), 0, + broker1.put_object('a', Timestamp(4).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker2.merge_items(broker1.get_items_since( broker2.get_sync(id), 1000), id) @@ -1223,24 +1223,24 @@ class TestContainerBroker(unittest.TestCase): self.assertEquals(['a', 'b'], sorted([rec['name'] for rec in items])) for rec in items: if rec['name'] == 'a': - self.assertEquals(rec['created_at'], normalize_timestamp(4)) + self.assertEquals(rec['created_at'], Timestamp(4).internal) if rec['name'] == 'b': - self.assertEquals(rec['created_at'], normalize_timestamp(3)) + self.assertEquals(rec['created_at'], Timestamp(3).internal) def test_merge_items_post_overwrite_out_of_order(self): # test DatabaseBroker.merge_items broker1 = ContainerBroker(':memory:', account='a', container='c') - broker1.initialize(normalize_timestamp('1'), 0) + broker1.initialize(Timestamp('1').internal, 0) id = broker1.get_info()['id'] broker2 = ContainerBroker(':memory:', account='a', container='c') - broker2.initialize(normalize_timestamp('1'), 0) - broker1.put_object('a', normalize_timestamp(2), 0, + broker2.initialize(Timestamp('1').internal, 0) + broker1.put_object('a', Timestamp(2).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker1.put_object('b', normalize_timestamp(3), 0, + broker1.put_object('b', Timestamp(3).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker2.merge_items(broker1.get_items_since( broker2.get_sync(id), 1000), id) - broker1.put_object('a', normalize_timestamp(4), 0, + broker1.put_object('a', Timestamp(4).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker2.merge_items(broker1.get_items_since( broker2.get_sync(id), 1000), id) @@ -1248,18 +1248,18 @@ class TestContainerBroker(unittest.TestCase): self.assertEquals(['a', 'b'], sorted([rec['name'] for rec in items])) for rec in items: if rec['name'] == 'a': - self.assertEquals(rec['created_at'], normalize_timestamp(4)) + self.assertEquals(rec['created_at'], Timestamp(4).internal) if rec['name'] == 'b': - self.assertEquals(rec['created_at'], normalize_timestamp(3)) + self.assertEquals(rec['created_at'], Timestamp(3).internal) self.assertEquals(rec['content_type'], 'text/plain') items = broker2.get_items_since(-1, 1000) self.assertEquals(['a', 'b'], sorted([rec['name'] for rec in items])) for rec in items: if rec['name'] == 'a': - self.assertEquals(rec['created_at'], normalize_timestamp(4)) + self.assertEquals(rec['created_at'], Timestamp(4).internal) if rec['name'] == 'b': - self.assertEquals(rec['created_at'], normalize_timestamp(3)) - broker1.put_object('b', normalize_timestamp(5), 0, + self.assertEquals(rec['created_at'], Timestamp(3).internal) + broker1.put_object('b', Timestamp(5).internal, 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker2.merge_items(broker1.get_items_since( broker2.get_sync(id), 1000), id) @@ -1267,13 +1267,13 @@ class TestContainerBroker(unittest.TestCase): self.assertEquals(['a', 'b'], sorted([rec['name'] for rec in items])) for rec in items: if rec['name'] == 'a': - self.assertEquals(rec['created_at'], normalize_timestamp(4)) + self.assertEquals(rec['created_at'], Timestamp(4).internal) if rec['name'] == 'b': - self.assertEquals(rec['created_at'], normalize_timestamp(5)) + self.assertEquals(rec['created_at'], Timestamp(5).internal) self.assertEquals(rec['content_type'], 'text/plain') def test_set_storage_policy_index(self): - ts = (normalize_timestamp(t) for t in + ts = (Timestamp(t).internal for t in itertools.count(int(time()))) broker = ContainerBroker(':memory:', account='test_account', container='test_container') @@ -1329,7 +1329,7 @@ class TestContainerBroker(unittest.TestCase): # never-had-an-object container to make sure we handle it broker = ContainerBroker(':memory:', account='test_account', container='test_container') - broker.initialize(normalize_timestamp('1'), 0) + broker.initialize(Timestamp('1').internal, 0) info = broker.get_info() self.assertEqual(0, info['storage_policy_index']) @@ -1340,14 +1340,14 @@ class TestContainerBroker(unittest.TestCase): def test_reconciler_sync(self): broker = ContainerBroker(':memory:', account='test_account', container='test_container') - broker.initialize(normalize_timestamp('1'), 0) + broker.initialize(Timestamp('1').internal, 0) self.assertEquals(-1, broker.get_reconciler_sync()) broker.update_reconciler_sync(10) self.assertEquals(10, broker.get_reconciler_sync()) @with_tempdir def test_legacy_pending_files(self, tempdir): - ts = (normalize_timestamp(t) for t in + ts = (Timestamp(t).internal for t in itertools.count(int(time()))) db_path = os.path.join(tempdir, 'container.db') @@ -1458,7 +1458,7 @@ def premetadata_create_container_info_table(self, conn, put_timestamp, :param put_timestamp: put timestamp """ if put_timestamp is None: - put_timestamp = normalize_timestamp(0) + put_timestamp = Timestamp(0).internal conn.executescript(''' CREATE TABLE container_stat ( account TEXT, @@ -1485,7 +1485,7 @@ def premetadata_create_container_info_table(self, conn, put_timestamp, UPDATE container_stat SET account = ?, container = ?, created_at = ?, id = ?, put_timestamp = ? - ''', (self.account, self.container, normalize_timestamp(time()), + ''', (self.account, self.container, Timestamp(time()).internal, str(uuid4()), put_timestamp)) @@ -1499,7 +1499,7 @@ class TestContainerBrokerBeforeMetadata(ContainerBrokerMigrationMixin, def setUp(self): super(TestContainerBrokerBeforeMetadata, self).setUp() broker = ContainerBroker(':memory:', account='a', container='c') - broker.initialize(normalize_timestamp('1'), 0) + broker.initialize(Timestamp('1').internal, 0) exc = None with broker.get() as conn: try: @@ -1511,7 +1511,7 @@ class TestContainerBrokerBeforeMetadata(ContainerBrokerMigrationMixin, def tearDown(self): super(TestContainerBrokerBeforeMetadata, self).tearDown() broker = ContainerBroker(':memory:', account='a', container='c') - broker.initialize(normalize_timestamp('1'), 0) + broker.initialize(Timestamp('1').internal, 0) with broker.get() as conn: conn.execute('SELECT metadata FROM container_stat') @@ -1529,7 +1529,7 @@ def prexsync_create_container_info_table(self, conn, put_timestamp, :param put_timestamp: put timestamp """ if put_timestamp is None: - put_timestamp = normalize_timestamp(0) + put_timestamp = Timestamp(0).internal conn.executescript(""" CREATE TABLE container_stat ( account TEXT, @@ -1557,7 +1557,7 @@ def prexsync_create_container_info_table(self, conn, put_timestamp, UPDATE container_stat SET account = ?, container = ?, created_at = ?, id = ?, put_timestamp = ? - ''', (self.account, self.container, normalize_timestamp(time()), + ''', (self.account, self.container, Timestamp(time()).internal, str(uuid4()), put_timestamp)) @@ -1573,7 +1573,7 @@ class TestContainerBrokerBeforeXSync(ContainerBrokerMigrationMixin, ContainerBroker.create_container_info_table = \ prexsync_create_container_info_table broker = ContainerBroker(':memory:', account='a', container='c') - broker.initialize(normalize_timestamp('1'), 0) + broker.initialize(Timestamp('1').internal, 0) exc = None with broker.get() as conn: try: @@ -1586,7 +1586,7 @@ class TestContainerBrokerBeforeXSync(ContainerBrokerMigrationMixin, def tearDown(self): super(TestContainerBrokerBeforeXSync, self).tearDown() broker = ContainerBroker(':memory:', account='a', container='c') - broker.initialize(normalize_timestamp('1'), 0) + broker.initialize(Timestamp('1').internal, 0) with broker.get() as conn: conn.execute('SELECT x_container_sync_point1 FROM container_stat') @@ -1641,7 +1641,7 @@ def prespi_create_container_info_table(self, conn, put_timestamp, :param put_timestamp: put timestamp """ if put_timestamp is None: - put_timestamp = normalize_timestamp(0) + put_timestamp = Timestamp(0).internal conn.executescript(""" CREATE TABLE container_stat ( account TEXT, @@ -1671,7 +1671,7 @@ def prespi_create_container_info_table(self, conn, put_timestamp, UPDATE container_stat SET account = ?, container = ?, created_at = ?, id = ?, put_timestamp = ? - ''', (self.account, self.container, normalize_timestamp(time()), + ''', (self.account, self.container, Timestamp(time()).internal, str(uuid4()), put_timestamp)) @@ -1688,7 +1688,7 @@ class TestContainerBrokerBeforeSPI(ContainerBrokerMigrationMixin, prespi_create_container_info_table broker = ContainerBroker(':memory:', account='a', container='c') - broker.initialize(normalize_timestamp('1'), 0) + broker.initialize(Timestamp('1').internal, 0) exc = None with broker.get() as conn: try: @@ -1701,7 +1701,7 @@ class TestContainerBrokerBeforeSPI(ContainerBrokerMigrationMixin, def tearDown(self): super(TestContainerBrokerBeforeSPI, self).tearDown() broker = ContainerBroker(':memory:', account='a', container='c') - broker.initialize(normalize_timestamp('1'), 0) + broker.initialize(Timestamp('1').internal, 0) with broker.get() as conn: conn.execute('SELECT storage_policy_index FROM container_stat') @@ -1712,7 +1712,7 @@ class TestContainerBrokerBeforeSPI(ContainerBrokerMigrationMixin, # initialize an un-migrated database broker = ContainerBroker(db_path, account='a', container='c') - put_timestamp = normalize_timestamp(int(time())) + put_timestamp = Timestamp(int(time())).internal broker.initialize(put_timestamp, None) with broker.get() as conn: try: @@ -1729,7 +1729,7 @@ class TestContainerBrokerBeforeSPI(ContainerBrokerMigrationMixin, 'from object table!') # manually insert an existing row to avoid automatic migration - obj_put_timestamp = normalize_timestamp(time()) + obj_put_timestamp = Timestamp(time()).internal with broker.get() as conn: conn.execute(''' INSERT INTO object (name, created_at, size, @@ -1767,7 +1767,7 @@ class TestContainerBrokerBeforeSPI(ContainerBrokerMigrationMixin, self.assertEqual(info[k], v, 'The value for %s was %r not %r' % ( k, info[k], v)) - self.assert_(float(info['created_at']) > float(put_timestamp)) + self.assert_(Timestamp(info['created_at']) > Timestamp(put_timestamp)) self.assertNotEqual(int(info['hash'], 16), 0) orig_hash = info['hash'] # get_replication_info @@ -1776,7 +1776,7 @@ class TestContainerBrokerBeforeSPI(ContainerBrokerMigrationMixin, expected['count'] = expected.pop('object_count') for k, v in expected.items(): self.assertEqual(info[k], v) - self.assert_(float(info['created_at']) > float(put_timestamp)) + self.assert_(Timestamp(info['created_at']) > Timestamp(put_timestamp)) self.assertEqual(info['hash'], orig_hash) self.assertEqual(info['max_row'], 1) self.assertEqual(info['metadata'], '') @@ -1839,7 +1839,7 @@ class TestContainerBrokerBeforeSPI(ContainerBrokerMigrationMixin, # now do a PUT with a different value for storage_policy_index # which will update the DB schema as well as update policy_stats # for legacy objects in the DB (those without an SPI) - second_object_put_timestamp = normalize_timestamp(time()) + second_object_put_timestamp = Timestamp(time()).internal other_policy = [p for p in POLICIES if p.idx != 0][0] broker.put_object('test_second', second_object_put_timestamp, 456, 'text/plain', diff --git a/test/unit/container/test_reconciler.py b/test/unit/container/test_reconciler.py index 42ad854ba3..fe1bc18a86 100644 --- a/test/unit/container/test_reconciler.py +++ b/test/unit/container/test_reconciler.py @@ -30,14 +30,15 @@ from swift.container import reconciler from swift.container.server import gen_resp_headers from swift.common.direct_client import ClientException from swift.common import swob -from swift.common.utils import split_path, normalize_timestamp +from swift.common.utils import split_path, Timestamp from test.unit import debug_logger, FakeRing, fake_http_connect from test.unit.common.middleware.helpers import FakeSwift def timestamp_to_last_modified(timestamp): - return datetime.fromtimestamp(timestamp).strftime('%Y-%m-%dT%H:%M:%S.%f') + return datetime.fromtimestamp( + float(Timestamp(timestamp))).strftime('%Y-%m-%dT%H:%M:%S.%f') def container_resp_headers(**kwargs): @@ -122,15 +123,16 @@ class FakeInternalClient(reconciler.InternalClient): # empty container continue obj_path = container_path + '/' + obj_name - headers = {'X-Timestamp': normalize_timestamp(timestamp)} + ts = Timestamp(timestamp) + headers = {'X-Timestamp': ts.normal, + 'X-Backend-Timestamp': ts.internal} # register object response self.app.storage_policy[storage_policy_index].register( 'GET', obj_path, swob.HTTPOk, headers) self.app.storage_policy[storage_policy_index].register( 'DELETE', obj_path, swob.HTTPNoContent, {}) # container listing entry - last_modified = timestamp_to_last_modified( - float(timestamp)) + last_modified = timestamp_to_last_modified(timestamp) obj_data = { 'bytes': 0, # listing data is unicode @@ -183,7 +185,7 @@ class TestReconcilerUtils(unittest.TestCase): def test_parse_raw_obj(self): got = reconciler.parse_raw_obj({ 'name': "2:/AUTH_bob/con/obj", - 'hash': normalize_timestamp(2017551.49350), + 'hash': Timestamp(2017551.49350).internal, 'last_modified': timestamp_to_last_modified(2017551.49352), 'content_type': 'application/x-delete', }) @@ -197,7 +199,7 @@ class TestReconcilerUtils(unittest.TestCase): got = reconciler.parse_raw_obj({ 'name': "1:/AUTH_bob/con/obj", - 'hash': normalize_timestamp(1234.20190), + 'hash': Timestamp(1234.20190).internal, 'last_modified': timestamp_to_last_modified(1234.20192), 'content_type': 'application/x-put', }) @@ -212,7 +214,7 @@ class TestReconcilerUtils(unittest.TestCase): # negative test obj_info = { 'name': "1:/AUTH_bob/con/obj", - 'hash': normalize_timestamp(1234.20190), + 'hash': Timestamp(1234.20190).internal, 'last_modified': timestamp_to_last_modified(1234.20192), } self.assertRaises(ValueError, reconciler.parse_raw_obj, obj_info) @@ -235,15 +237,15 @@ class TestReconcilerUtils(unittest.TestCase): mock_path = 'swift.container.reconciler.direct_head_container' stub_resp_headers = [ container_resp_headers( - status_changed_at=normalize_timestamp(ts.next()), + status_changed_at=Timestamp(ts.next()).internal, storage_policy_index=0, ), container_resp_headers( - status_changed_at=normalize_timestamp(ts.next()), + status_changed_at=Timestamp(ts.next()).internal, storage_policy_index=1, ), container_resp_headers( - status_changed_at=normalize_timestamp(ts.next()), + status_changed_at=Timestamp(ts.next()).internal, storage_policy_index=0, ), ] @@ -278,7 +280,7 @@ class TestReconcilerUtils(unittest.TestCase): 'Container Server blew up', http_status=500, http_reason='Server Error', http_headers=container_resp_headers( - status_changed_at=normalize_timestamp(0), + status_changed_at=Timestamp(0).internal, storage_policy_index=0, ), ), @@ -295,11 +297,11 @@ class TestReconcilerUtils(unittest.TestCase): mock_path = 'swift.container.reconciler.direct_head_container' stub_resp_headers = [ container_resp_headers( - status_changed_at=normalize_timestamp(ts.next()), + status_changed_at=Timestamp(ts.next()).internal, storage_policy_index=1, ), container_resp_headers( - status_changed_at=normalize_timestamp(ts.next()), + status_changed_at=Timestamp(ts.next()).internal, storage_policy_index=0, ), socket.error(errno.ECONNREFUSED, os.strerror(errno.ECONNREFUSED)), @@ -316,7 +318,7 @@ class TestReconcilerUtils(unittest.TestCase): mock_path = 'swift.container.reconciler.direct_head_container' stub_resp_headers = [ container_resp_headers( - status_changed_at=normalize_timestamp(ts.next()), + status_changed_at=Timestamp(ts.next()).internal, storage_policy_index=0, ), socket.error(errno.ECONNREFUSED, os.strerror(errno.ECONNREFUSED)), @@ -324,7 +326,7 @@ class TestReconcilerUtils(unittest.TestCase): 'Container Server blew up', http_status=500, http_reason='Server Error', http_headers=container_resp_headers( - status_changed_at=normalize_timestamp(ts.next()), + status_changed_at=Timestamp(ts.next()).internal, storage_policy_index=1, ), ), @@ -339,7 +341,7 @@ class TestReconcilerUtils(unittest.TestCase): def test_get_container_policy_index_for_deleted(self): mock_path = 'swift.container.reconciler.direct_head_container' headers = container_resp_headers( - status_changed_at=normalize_timestamp(time.time()), + status_changed_at=Timestamp(time.time()).internal, storage_policy_index=1, ) stub_resp_headers = [ @@ -484,15 +486,15 @@ class TestReconcilerUtils(unittest.TestCase): mock_path = 'swift.container.reconciler.direct_head_container' stub_resp_headers = [ container_resp_headers( - status_changed_at=normalize_timestamp(ts.next()), + status_changed_at=Timestamp(ts.next()).internal, storage_policy_index=0, ), container_resp_headers( - status_changed_at=normalize_timestamp(ts.next()), + status_changed_at=Timestamp(ts.next()).internal, storage_policy_index=1, ), container_resp_headers( - status_changed_at=normalize_timestamp(ts.next()), + status_changed_at=Timestamp(ts.next()).internal, storage_policy_index=0, ), ] @@ -536,8 +538,8 @@ class TestReconcilerUtils(unittest.TestCase): 'partition': partition, 'method': method, 'path': path, 'headers': headers, 'query_string': query_string}) - x_timestamp = normalize_timestamp(time.time()) - headers = {'x-timestamp': x_timestamp} + x_timestamp = Timestamp(time.time()) + headers = {'x-timestamp': x_timestamp.internal} fake_hc = fake_http_connect(200, 200, 200, give_connect=test_connect) with mock.patch(mock_path, fake_hc): reconciler.direct_delete_container_entry( @@ -620,7 +622,7 @@ class TestReconcilerUtils(unittest.TestCase): 'headers': headers, 'query_string': query_string}) fake_hc = fake_http_connect(200, 200, 200, give_connect=test_connect) - now = float(normalize_timestamp(time.time())) + now = time.time() with contextlib.nested( mock.patch(mock_path, fake_hc), mock.patch('swift.container.reconciler.time.time', @@ -639,7 +641,7 @@ class TestReconcilerUtils(unittest.TestCase): for args in connect_args: self.assertEqual(args['headers']['X-Timestamp'], - normalize_timestamp(now)) + Timestamp(now).internal) self.assertEqual(args['headers']['X-Etag'], '5948918.63946') self.assertEqual(args['path'], '/.misplaced_objects/5947200/17:/a/c/o') @@ -820,7 +822,7 @@ class TestReconciler(unittest.TestCase): # we DELETE the object from the wrong place with source_ts + offset 1 # timestamp to make sure the change takes effect self.assertEqual(delete_headers.get('X-Timestamp'), - normalize_timestamp(3618.84188)) + Timestamp(3618.84187, offset=1).internal) # and pop the queue for that one self.assertEqual(self.reconciler.stats['pop_queue'], 1) self.assertEqual(deleted_container_entries, [( @@ -884,14 +886,14 @@ class TestReconciler(unittest.TestCase): put_headers = self.fake_swift.storage_policy[0].headers[1] # we PUT the object in the right place with q_ts + offset 2 self.assertEqual(put_headers.get('X-Timestamp'), - normalize_timestamp('3618.84189')) + Timestamp(3618.84187, offset=2)) # cleans up the old self.assertEqual(self.reconciler.stats['cleanup_attempt'], 1) self.assertEqual(self.reconciler.stats['cleanup_success'], 1) # we DELETE the object from the wrong place with source_ts + offset 1 # timestamp to make sure the change takes effect self.assertEqual(delete_headers.get('X-Timestamp'), - normalize_timestamp('3618.84188')) + Timestamp(3618.84187, offset=1)) # and when we're done, we pop the entry from the queue self.assertEqual(self.reconciler.stats['pop_queue'], 1) self.assertEqual(deleted_container_entries, @@ -931,14 +933,14 @@ class TestReconciler(unittest.TestCase): put_headers = self.fake_swift.storage_policy[1].headers[1] # we PUT the object in the right place with q_ts + offset 2 self.assertEqual(put_headers.get('X-Timestamp'), - normalize_timestamp('3618.84189')) + Timestamp(3618.84187, offset=2).internal) # cleans up the old self.assertEqual(self.reconciler.stats['cleanup_attempt'], 1) self.assertEqual(self.reconciler.stats['cleanup_success'], 1) # we DELETE the object from the wrong place with source_ts + offset 1 # timestamp to make sure the change takes effect self.assertEqual(delete_headers.get('X-Timestamp'), - normalize_timestamp('3618.84188')) + Timestamp(3618.84187, offset=1).internal) # and when we're done, we pop the entry from the queue self.assertEqual(self.reconciler.stats['pop_queue'], 1) self.assertEqual(deleted_container_entries, @@ -987,14 +989,14 @@ class TestReconciler(unittest.TestCase): put_headers = self.fake_swift.storage_policy[0].headers[1] # we PUT the object in the right place with q_ts + offset 2 self.assertEqual(put_headers.get('X-Timestamp'), - normalize_timestamp('3618.84189')) + Timestamp(3618.84187, offset=2).internal) # cleans up the old self.assertEqual(self.reconciler.stats['cleanup_attempt'], 1) self.assertEqual(self.reconciler.stats['cleanup_success'], 1) # we DELETE the object from the wrong place with source_ts + offset 1 # timestamp to make sure the change takes effect self.assertEqual(delete_headers.get('X-Timestamp'), - normalize_timestamp('3618.84188')) + Timestamp(3618.84187, offset=1).internal) self.assertEqual( delete_headers.get('X-Backend-Storage-Policy-Index'), '1') # and when we're done, we pop the entry from the queue @@ -1005,18 +1007,18 @@ class TestReconciler(unittest.TestCase): self.assertEqual(self.reconciler.stats['success'], 1) def test_object_delete(self): - q_ts = float(normalize_timestamp(time.time())) + q_ts = time.time() self._mock_listing({ (None, "/.misplaced_objects/3600/1:/AUTH_bob/c/o1"): ( - normalize_timestamp(q_ts), 'application/x-delete'), + Timestamp(q_ts).internal, 'application/x-delete'), # object exists in "correct" storage policy - slightly older - (0, "/AUTH_bob/c/o1"): normalize_timestamp(q_ts - 1), + (0, "/AUTH_bob/c/o1"): Timestamp(q_ts - 1).internal, }) self._mock_oldest_spi({'c': 0}) # the tombstone exists in the enqueued storage policy self.fake_swift.storage_policy[1].register( 'GET', '/v1/AUTH_bob/c/o1', swob.HTTPNotFound, - {'X-Backend-Timestamp': normalize_timestamp(q_ts)}) + {'X-Backend-Timestamp': Timestamp(q_ts).internal}) deleted_container_entries = self._run_once() # found a misplaced object @@ -1044,14 +1046,14 @@ class TestReconciler(unittest.TestCase): reconcile_headers = self.fake_swift.storage_policy[0].headers[1] # we DELETE the object in the right place with q_ts + offset 2 self.assertEqual(reconcile_headers.get('X-Timestamp'), - normalize_timestamp(q_ts + 0.00002)) + Timestamp(q_ts, offset=2).internal) # cleans up the old self.assertEqual(self.reconciler.stats['cleanup_attempt'], 1) self.assertEqual(self.reconciler.stats['cleanup_success'], 1) # we DELETE the object from the wrong place with source_ts + offset 1 # timestamp to make sure the change takes effect self.assertEqual(delete_headers.get('X-Timestamp'), - normalize_timestamp(q_ts + 0.00001)) + Timestamp(q_ts, offset=1)) # and when we're done, we pop the entry from the queue self.assertEqual(self.reconciler.stats['pop_queue'], 1) self.assertEqual(deleted_container_entries, @@ -1117,13 +1119,13 @@ class TestReconciler(unittest.TestCase): # .. with source timestamp + offset 2 put_headers = self.fake_swift.storage_policy[0].headers[1] self.assertEqual(put_headers.get('X-Timestamp'), - normalize_timestamp(3600.234587)) + Timestamp(3600.234567, offset=2)) # src object is cleaned up self.assertEqual(self.reconciler.stats['cleanup_attempt'], 1) self.assertEqual(self.reconciler.stats['cleanup_success'], 1) # ... with q_ts + offset 1 self.assertEqual(delete_headers.get('X-Timestamp'), - normalize_timestamp(3600.123466)) + Timestamp(3600.123456, offset=1)) # and queue is popped self.assertEqual(self.reconciler.stats['pop_queue'], 1) self.assertEqual(deleted_container_entries, @@ -1132,7 +1134,7 @@ class TestReconciler(unittest.TestCase): def test_object_move_src_object_older_than_queue_entry(self): # should be some sort of retry case - q_ts = float(normalize_timestamp(time.time())) + q_ts = time.time() container = str(int(q_ts // 3600 * 3600)) q_path = '.misplaced_objects/%s' % container self._mock_listing({ @@ -1169,7 +1171,7 @@ class TestReconciler(unittest.TestCase): def test_src_object_unavailable_with_slightly_newer_tombstone(self): # should be some sort of retry case - q_ts = float(normalize_timestamp(time.time())) + q_ts = float(Timestamp(time.time())) container = str(int(q_ts // 3600 * 3600)) q_path = '.misplaced_objects/%s' % container self._mock_listing({ @@ -1178,7 +1180,7 @@ class TestReconciler(unittest.TestCase): self._mock_oldest_spi({'c': 0}) self.fake_swift.storage_policy[1].register( 'GET', '/v1/AUTH_bob/c/o1', swob.HTTPNotFound, - {'X-Backend-Timestamp': normalize_timestamp(q_ts + 0.00002)}) + {'X-Backend-Timestamp': Timestamp(q_ts, offset=2).internal}) deleted_container_entries = self._run_once() # found a misplaced object @@ -1208,7 +1210,7 @@ class TestReconciler(unittest.TestCase): def test_src_object_unavailable_server_error(self): # should be some sort of retry case - q_ts = float(normalize_timestamp(time.time())) + q_ts = float(Timestamp(time.time())) container = str(int(q_ts // 3600 * 3600)) q_path = '.misplaced_objects/%s' % container self._mock_listing({ @@ -1248,7 +1250,7 @@ class TestReconciler(unittest.TestCase): # setup the cluster self._mock_listing({ (None, "/.misplaced_objects/3600/1:/AUTH_bob/c/o1"): 3600.123456, - (1, '/AUTH_bob/c/o1'): 3600.234567, # slightly newer + (1, '/AUTH_bob/c/o1'): 3600.123457, # slightly newer }) self._mock_oldest_spi({'c': 0}) # destination @@ -1283,12 +1285,12 @@ class TestReconciler(unittest.TestCase): # .. with source timestamp + offset 2 put_headers = self.fake_swift.storage_policy[0].headers[1] self.assertEqual(put_headers.get('X-Timestamp'), - normalize_timestamp(3600.234587)) + Timestamp(3600.123457, offset=2)) # we try to cleanup self.assertEqual(self.reconciler.stats['cleanup_attempt'], 1) # ... with q_ts + offset 1 self.assertEqual(delete_headers.get('X-Timestamp'), - normalize_timestamp(3600.123466)) + Timestamp(3600.12346, offset=1)) # but cleanup fails! self.assertEqual(self.reconciler.stats['cleanup_failed'], 1) # so the queue is not popped @@ -1366,7 +1368,7 @@ class TestReconciler(unittest.TestCase): self.assertEqual(self.reconciler.stats['cleanup_attempt'], 1) self.assertEqual(self.reconciler.stats['cleanup_success'], 1) self.assertEqual(delete_headers.get('X-Timestamp'), - normalize_timestamp(3679.20191)) + Timestamp(3679.2019, offset=1)) # and wipe our hands of it self.assertEqual(self.reconciler.stats['pop_queue'], 1) self.assertEqual(deleted_container_entries, @@ -1407,7 +1409,7 @@ class TestReconciler(unittest.TestCase): self.assertEqual(self.reconciler.stats['cleanup_success'], 1) self.assertEqual(delete_headers.get('X-Timestamp'), - normalize_timestamp('3679.20191')) + Timestamp(3679.2019, offset=1)) # and since we cleaned up the old object, so this counts as done self.assertEqual(self.reconciler.stats['pop_queue'], 1) self.assertEqual(deleted_container_entries, @@ -1448,13 +1450,13 @@ class TestReconciler(unittest.TestCase): # ... with a q_ts + offset 2 put_headers = self.fake_swift.storage_policy[0].headers[1] self.assertEqual(put_headers.get('X-Timestamp'), - normalize_timestamp(36123.38395)) + Timestamp(36123.38393, offset=2)) # then clean the dark matter self.assertEqual(self.reconciler.stats['cleanup_attempt'], 1) self.assertEqual(self.reconciler.stats['cleanup_success'], 1) # ... with a q_ts + offset 1 self.assertEqual(delete_headers.get('X-Timestamp'), - normalize_timestamp(36123.38394)) + Timestamp(36123.38393, offset=1)) # and pop the queue self.assertEqual(self.reconciler.stats['pop_queue'], 1) @@ -1499,7 +1501,7 @@ class TestReconciler(unittest.TestCase): put_headers = self.fake_swift.storage_policy[0].headers[1] # ...with q_ts + offset 2 (20-microseconds) self.assertEqual(put_headers.get('X-Timestamp'), - normalize_timestamp(36123.383945)) + Timestamp(36123.383925, offset=2)) # but it failed self.assertEqual(self.reconciler.stats['copy_success'], 0) self.assertEqual(self.reconciler.stats['copy_failed'], 1) @@ -1550,7 +1552,7 @@ class TestReconciler(unittest.TestCase): put_headers = self.fake_swift.storage_policy[0].headers[1] # ...with q_ts + offset 2 (20-microseconds) self.assertEqual(put_headers.get('X-Timestamp'), - normalize_timestamp(36123.383945)) + Timestamp(36123.383925, offset=2)) # but it blows up hard self.assertEqual(self.reconciler.stats['unhandled_error'], 1) # so we don't cleanup @@ -1561,7 +1563,7 @@ class TestReconciler(unittest.TestCase): self.assertEqual(self.reconciler.stats['retry'], 1) def test_object_move_no_such_object_no_tombstone_recent(self): - q_ts = float(normalize_timestamp(time.time())) + q_ts = float(Timestamp(time.time())) container = str(int(q_ts // 3600 * 3600)) q_path = '.misplaced_objects/%s' % container @@ -1593,7 +1595,7 @@ class TestReconciler(unittest.TestCase): self.assertEqual(deleted_container_entries, []) def test_object_move_no_such_object_no_tombstone_ancient(self): - queue_ts = float(normalize_timestamp(time.time())) - \ + queue_ts = float(Timestamp(time.time())) - \ self.reconciler.reclaim_age * 1.1 container = str(int(queue_ts // 3600 * 3600)) @@ -1630,8 +1632,7 @@ class TestReconciler(unittest.TestCase): [('.misplaced_objects', container, '1:/AUTH_jeb/c/o1')]) def test_delete_old_empty_queue_containers(self): - ts = float(normalize_timestamp(time.time())) - \ - self.reconciler.reclaim_age * 1.1 + ts = time.time() - self.reconciler.reclaim_age * 1.1 container = str(int(ts // 3600 * 3600)) older_ts = ts - 3600 older_container = str(int(older_ts // 3600 * 3600)) diff --git a/test/unit/container/test_replicator.py b/test/unit/container/test_replicator.py index a1888c4031..74cf7734a4 100644 --- a/test/unit/container/test_replicator.py +++ b/test/unit/container/test_replicator.py @@ -26,7 +26,7 @@ from swift.common import db_replicator from swift.container import replicator, backend, server from swift.container.reconciler import ( MISPLACED_OBJECTS_ACCOUNT, get_reconciler_container_name) -from swift.common.utils import normalize_timestamp +from swift.common.utils import Timestamp from swift.common.storage_policy import POLICIES from test.unit.common import test_db_replicator @@ -44,18 +44,18 @@ class TestReplicator(unittest.TestCase): def test_report_up_to_date(self): repl = replicator.ContainerReplicator({}) - info = {'put_timestamp': normalize_timestamp(1), - 'delete_timestamp': normalize_timestamp(0), + info = {'put_timestamp': Timestamp(1).internal, + 'delete_timestamp': Timestamp(0).internal, 'object_count': 0, 'bytes_used': 0, - 'reported_put_timestamp': normalize_timestamp(1), - 'reported_delete_timestamp': normalize_timestamp(0), + 'reported_put_timestamp': Timestamp(1).internal, + 'reported_delete_timestamp': Timestamp(0).internal, 'reported_object_count': 0, 'reported_bytes_used': 0} self.assertTrue(repl.report_up_to_date(info)) - info['delete_timestamp'] = normalize_timestamp(2) + info['delete_timestamp'] = Timestamp(2).internal self.assertFalse(repl.report_up_to_date(info)) - info['reported_delete_timestamp'] = normalize_timestamp(2) + info['reported_delete_timestamp'] = Timestamp(2).internal self.assertTrue(repl.report_up_to_date(info)) info['object_count'] = 1 self.assertFalse(repl.report_up_to_date(info)) @@ -65,9 +65,9 @@ class TestReplicator(unittest.TestCase): self.assertFalse(repl.report_up_to_date(info)) info['reported_bytes_used'] = 1 self.assertTrue(repl.report_up_to_date(info)) - info['put_timestamp'] = normalize_timestamp(3) + info['put_timestamp'] = Timestamp(3).internal self.assertFalse(repl.report_up_to_date(info)) - info['reported_put_timestamp'] = normalize_timestamp(3) + info['reported_put_timestamp'] = Timestamp(3).internal self.assertTrue(repl.report_up_to_date(info)) @@ -328,21 +328,21 @@ class TestReplicatorSync(test_db_replicator.TestReplicatorSync): self.assertTrue(remote_broker.is_deleted()) info = broker.get_info() remote_info = remote_broker.get_info() - self.assert_(float(remote_info['status_changed_at']) > - float(remote_info['put_timestamp']), + self.assert_(Timestamp(remote_info['status_changed_at']) > + Timestamp(remote_info['put_timestamp']), 'remote status_changed_at (%s) is not ' 'greater than put_timestamp (%s)' % ( remote_info['status_changed_at'], remote_info['put_timestamp'])) - self.assert_(float(remote_info['status_changed_at']) > - float(info['status_changed_at']), + self.assert_(Timestamp(remote_info['status_changed_at']) > + Timestamp(info['status_changed_at']), 'remote status_changed_at (%s) is not ' 'greater than local status_changed_at (%s)' % ( remote_info['status_changed_at'], info['status_changed_at'])) def test_sync_bogus_db_quarantines(self): - ts = (normalize_timestamp(t) for t in + ts = (Timestamp(t).internal for t in itertools.count(int(time.time()))) policy = random.choice(list(POLICIES)) @@ -667,22 +667,21 @@ class TestReplicatorSync(test_db_replicator.TestReplicatorSync): remote_broker.update_status_changed_at(remote_recreate_timestamp) def test_sync_to_remote_with_misplaced(self): - ts = itertools.count(int(time.time())) + ts = (Timestamp(t).internal for t in + itertools.count(int(time.time()))) # create "local" broker policy = random.choice(list(POLICIES)) broker = self._get_broker('a', 'c', node_index=0) - broker.initialize(normalize_timestamp(ts.next()), - policy.idx) + broker.initialize(ts.next(), policy.idx) # create "remote" broker remote_policy = random.choice([p for p in POLICIES if p is not policy]) remote_broker = self._get_broker('a', 'c', node_index=1) - remote_broker.initialize(normalize_timestamp(ts.next()), - remote_policy.idx) + remote_broker.initialize(ts.next(), remote_policy.idx) # add misplaced row to remote_broker remote_broker.put_object( - '/a/c/o', normalize_timestamp(ts.next()), 0, 'content-type', + '/a/c/o', ts.next(), 0, 'content-type', 'etag', storage_policy_index=remote_broker.storage_policy_index) # since this row matches policy index or remote, it shows up in count self.assertEqual(remote_broker.get_info()['object_count'], 1) @@ -716,19 +715,18 @@ class TestReplicatorSync(test_db_replicator.TestReplicatorSync): self.assertEqual(info[key], value) def test_misplaced_rows_replicate_and_enqueue(self): - ts = itertools.count(int(time.time())) + ts = (Timestamp(t).internal for t in + itertools.count(int(time.time()))) policy = random.choice(list(POLICIES)) broker = self._get_broker('a', 'c', node_index=0) - broker.initialize(normalize_timestamp(ts.next()), - policy.idx) + broker.initialize(ts.next(), policy.idx) remote_policy = random.choice([p for p in POLICIES if p is not policy]) remote_broker = self._get_broker('a', 'c', node_index=1) - remote_broker.initialize(normalize_timestamp(ts.next()), - remote_policy.idx) + remote_broker.initialize(ts.next(), remote_policy.idx) # add a misplaced row to *local* broker - obj_put_timestamp = normalize_timestamp(ts.next()) + obj_put_timestamp = ts.next() broker.put_object( 'o', obj_put_timestamp, 0, 'content-type', 'etag', storage_policy_index=remote_policy.idx) @@ -777,22 +775,21 @@ class TestReplicatorSync(test_db_replicator.TestReplicatorSync): self.assertEqual(broker.get_reconciler_sync(), 1) def test_multiple_out_sync_reconciler_enqueue_normalize(self): - ts = itertools.count(int(time.time())) + ts = (Timestamp(t).internal for t in + itertools.count(int(time.time()))) policy = random.choice(list(POLICIES)) broker = self._get_broker('a', 'c', node_index=0) - broker.initialize(normalize_timestamp(ts.next()), policy.idx) + broker.initialize(ts.next(), policy.idx) remote_policy = random.choice([p for p in POLICIES if p is not policy]) remote_broker = self._get_broker('a', 'c', node_index=1) - remote_broker.initialize(normalize_timestamp(ts.next()), - remote_policy.idx) + remote_broker.initialize(ts.next(), remote_policy.idx) # add some rows to brokers for db in (broker, remote_broker): for p in (policy, remote_policy): - db.put_object('o-%s' % p.name, normalize_timestamp(ts.next()), - 0, 'content-type', 'etag', - storage_policy_index=p.idx) + db.put_object('o-%s' % p.name, ts.next(), 0, 'content-type', + 'etag', storage_policy_index=p.idx) db._commit_puts() expected_policy_stats = { diff --git a/test/unit/container/test_server.py b/test/unit/container/test_server.py index 1823196300..757ce5ac97 100644 --- a/test/unit/container/test_server.py +++ b/test/unit/container/test_server.py @@ -35,8 +35,8 @@ from swift.common.swob import Request, HeaderKeyDict import swift.container from swift.container import server as container_server from swift.common import constraints -from swift.common.utils import (normalize_timestamp, mkdirs, public, - replication, lock_parent_directory) +from swift.common.utils import (Timestamp, mkdirs, public, replication, + lock_parent_directory, json) from test.unit import fake_http_connect from swift.common.storage_policy import (POLICY_INDEX, POLICIES, StoragePolicy) @@ -171,9 +171,9 @@ class TestContainerController(unittest.TestCase): def test_HEAD(self): start = int(time.time()) - ts = itertools.count(start) + ts = (Timestamp(t).internal for t in itertools.count(start)) req = Request.blank('/sda1/p/a/c', method='PUT', headers={ - 'x-timestamp': normalize_timestamp(ts.next())}) + 'x-timestamp': ts.next()}) req.get_response(self.controller) req = Request.blank('/sda1/p/a/c', method='HEAD') response = req.get_response(self.controller) @@ -182,7 +182,7 @@ class TestContainerController(unittest.TestCase): self.assertEqual(response.headers['x-container-object-count'], '0') obj_put_request = Request.blank( '/sda1/p/a/c/o', method='PUT', headers={ - 'x-timestamp': normalize_timestamp(ts.next()), + 'x-timestamp': ts.next(), 'x-size': 42, 'x-content-type': 'text/plain', 'x-etag': 'x', @@ -196,20 +196,24 @@ class TestContainerController(unittest.TestCase): self.assertEqual(response.headers['x-container-bytes-used'], '42') self.assertEqual(response.headers['x-container-object-count'], '1') # created at time... - self.assert_(float(response.headers['x-timestamp']) >= start) + created_at_header = Timestamp(response.headers['x-timestamp']) + self.assertEqual(response.headers['x-timestamp'], + created_at_header.normal) + self.assert_(created_at_header >= start) self.assertEqual(response.headers['x-put-timestamp'], - normalize_timestamp(start)) + Timestamp(start).normal) # backend headers self.assertEqual(int(response.headers[POLICY_INDEX]), int(POLICIES.default)) - self.assert_(float(response.headers['x-backend-timestamp']) >= start) + self.assert_( + Timestamp(response.headers['x-backend-timestamp']) >= start) self.assertEqual(response.headers['x-backend-put-timestamp'], - normalize_timestamp(start)) + Timestamp(start).internal) self.assertEqual(response.headers['x-backend-delete-timestamp'], - normalize_timestamp(0)) + Timestamp(0).internal) self.assertEqual(response.headers['x-backend-status-changed-at'], - normalize_timestamp(start)) + Timestamp(start).internal) def test_HEAD_not_found(self): req = Request.blank('/sda1/p/a/c', method='HEAD') @@ -217,22 +221,23 @@ class TestContainerController(unittest.TestCase): self.assertEqual(resp.status_int, 404) self.assertEqual(int(resp.headers[POLICY_INDEX]), 0) self.assertEqual(resp.headers['x-backend-timestamp'], - normalize_timestamp(0)) + Timestamp(0).internal) self.assertEqual(resp.headers['x-backend-put-timestamp'], - normalize_timestamp(0)) + Timestamp(0).internal) self.assertEqual(resp.headers['x-backend-status-changed-at'], - normalize_timestamp(0)) + Timestamp(0).internal) self.assertEqual(resp.headers['x-backend-delete-timestamp'], - normalize_timestamp(0)) + Timestamp(0).internal) for header in ('x-container-object-count', 'x-container-bytes-used', 'x-timestamp', 'x-put-timestamp'): self.assertEqual(resp.headers[header], None) def test_deleted_headers(self): - ts = itertools.count(int(time.time())) + ts = (Timestamp(t).internal for t in + itertools.count(int(time.time()))) request_method_times = { - 'PUT': normalize_timestamp(ts.next()), - 'DELETE': normalize_timestamp(ts.next()), + 'PUT': ts.next(), + 'DELETE': ts.next(), } # setup a deleted container for method in ('PUT', 'DELETE'): @@ -249,8 +254,8 @@ class TestContainerController(unittest.TestCase): # backend headers self.assertEqual(int(resp.headers[POLICY_INDEX]), int(POLICIES.default)) - self.assert_(float(resp.headers['x-backend-timestamp']) >= - float(request_method_times['PUT'])) + self.assert_(Timestamp(resp.headers['x-backend-timestamp']) >= + Timestamp(request_method_times['PUT'])) self.assertEqual(resp.headers['x-backend-put-timestamp'], request_method_times['PUT']) self.assertEqual(resp.headers['x-backend-delete-timestamp'], @@ -357,7 +362,7 @@ class TestContainerController(unittest.TestCase): policy = random.choice(list(POLICIES)) # Set metadata header req = Request.blank('/sda1/p/a/c', method='PUT', - headers={'X-Timestamp': normalize_timestamp(1), + headers={'X-Timestamp': Timestamp(1).internal, POLICY_INDEX: policy.idx}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 201) @@ -370,7 +375,7 @@ class TestContainerController(unittest.TestCase): def test_PUT_no_policy_specified(self): # Set metadata header req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'}, - headers={'X-Timestamp': normalize_timestamp(1)}) + headers={'X-Timestamp': Timestamp(1).internal}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 201) @@ -383,18 +388,18 @@ class TestContainerController(unittest.TestCase): def test_PUT_bad_policy_specified(self): # Set metadata header req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'}, - headers={'X-Timestamp': normalize_timestamp(1), + headers={'X-Timestamp': Timestamp(1).internal, POLICY_INDEX: 'nada'}) resp = req.get_response(self.controller) # make sure we get bad response self.assertEquals(resp.status_int, 400) def test_PUT_no_policy_change(self): - ts = itertools.count(1) + ts = (Timestamp(t).internal for t in itertools.count(time.time())) policy = random.choice(list(POLICIES)) # Set metadata header req = Request.blank('/sda1/p/a/c', method='PUT', headers={ - 'X-Timestamp': normalize_timestamp(ts.next()), + 'X-Timestamp': ts.next(), POLICY_INDEX: policy.idx}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 201) @@ -407,7 +412,7 @@ class TestContainerController(unittest.TestCase): # now try to update w/o changing the policy for method in ('POST', 'PUT'): req = Request.blank('/sda1/p/a/c', method=method, headers={ - 'X-Timestamp': normalize_timestamp(ts.next()), + 'X-Timestamp': ts.next(), POLICY_INDEX: policy.idx }) resp = req.get_response(self.controller) @@ -419,11 +424,11 @@ class TestContainerController(unittest.TestCase): self.assertEquals(resp.headers.get(POLICY_INDEX), str(policy.idx)) def test_PUT_bad_policy_change(self): - ts = itertools.count(1) + ts = (Timestamp(t).internal for t in itertools.count(time.time())) policy = random.choice(list(POLICIES)) # Set metadata header req = Request.blank('/sda1/p/a/c', method='PUT', headers={ - 'X-Timestamp': normalize_timestamp(ts.next()), + 'X-Timestamp': ts.next(), POLICY_INDEX: policy.idx}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 201) @@ -437,7 +442,7 @@ class TestContainerController(unittest.TestCase): for other_policy in other_policies: # now try to change it and make sure we get a conflict req = Request.blank('/sda1/p/a/c', method='PUT', headers={ - 'X-Timestamp': normalize_timestamp(ts.next()), + 'X-Timestamp': ts.next(), POLICY_INDEX: other_policy.idx }) resp = req.get_response(self.controller) @@ -451,10 +456,10 @@ class TestContainerController(unittest.TestCase): self.assertEquals(resp.headers.get(POLICY_INDEX), str(policy.idx)) def test_POST_ignores_policy_change(self): - ts = itertools.count(1) + ts = (Timestamp(t).internal for t in itertools.count(time.time())) policy = random.choice(list(POLICIES)) req = Request.blank('/sda1/p/a/c', method='PUT', headers={ - 'X-Timestamp': normalize_timestamp(ts.next()), + 'X-Timestamp': ts.next(), POLICY_INDEX: policy.idx}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 201) @@ -468,7 +473,7 @@ class TestContainerController(unittest.TestCase): for other_policy in other_policies: # now try to change it and make sure we get a conflict req = Request.blank('/sda1/p/a/c', method='POST', headers={ - 'X-Timestamp': normalize_timestamp(ts.next()), + 'X-Timestamp': ts.next(), POLICY_INDEX: other_policy.idx }) resp = req.get_response(self.controller) @@ -483,10 +488,11 @@ class TestContainerController(unittest.TestCase): self.assertEquals(resp.headers.get(POLICY_INDEX), str(policy.idx)) def test_PUT_no_policy_for_existing_default(self): - ts = itertools.count(1) + ts = (Timestamp(t).internal for t in + itertools.count(int(time.time()))) # create a container with the default storage policy req = Request.blank('/sda1/p/a/c', method='PUT', headers={ - 'X-Timestamp': normalize_timestamp(ts.next()), + 'X-Timestamp': ts.next(), }) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 201) # sanity check @@ -500,7 +506,7 @@ class TestContainerController(unittest.TestCase): # put again without specifying the storage policy req = Request.blank('/sda1/p/a/c', method='PUT', headers={ - 'X-Timestamp': normalize_timestamp(ts.next()), + 'X-Timestamp': ts.next(), }) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 202) # sanity check @@ -517,10 +523,11 @@ class TestContainerController(unittest.TestCase): # during a config change restart across a multi node cluster. proxy_default = random.choice([p for p in POLICIES if not p.is_default]) - ts = itertools.count(1) + ts = (Timestamp(t).internal for t in + itertools.count(int(time.time()))) # create a container with the default storage policy req = Request.blank('/sda1/p/a/c', method='PUT', headers={ - 'X-Timestamp': normalize_timestamp(ts.next()), + 'X-Timestamp': ts.next(), 'X-Backend-Storage-Policy-Default': int(proxy_default), }) resp = req.get_response(self.controller) @@ -535,7 +542,7 @@ class TestContainerController(unittest.TestCase): # put again without proxy specifying the different default req = Request.blank('/sda1/p/a/c', method='PUT', headers={ - 'X-Timestamp': normalize_timestamp(ts.next()), + 'X-Timestamp': ts.next(), 'X-Backend-Storage-Policy-Default': int(POLICIES.default), }) resp = req.get_response(self.controller) @@ -549,11 +556,11 @@ class TestContainerController(unittest.TestCase): int(proxy_default)) def test_PUT_no_policy_for_existing_non_default(self): - ts = itertools.count(1) + ts = (Timestamp(t).internal for t in itertools.count(time.time())) non_default_policy = [p for p in POLICIES if not p.is_default][0] # create a container with the non-default storage policy req = Request.blank('/sda1/p/a/c', method='PUT', headers={ - 'X-Timestamp': normalize_timestamp(ts.next()), + 'X-Timestamp': ts.next(), POLICY_INDEX: non_default_policy.idx, }) resp = req.get_response(self.controller) @@ -568,7 +575,7 @@ class TestContainerController(unittest.TestCase): # put again without specifiying the storage policy req = Request.blank('/sda1/p/a/c', method='PUT', headers={ - 'X-Timestamp': normalize_timestamp(ts.next()), + 'X-Timestamp': ts.next(), }) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 202) # sanity check @@ -584,7 +591,7 @@ class TestContainerController(unittest.TestCase): # Set metadata header req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'}, - headers={'X-Timestamp': normalize_timestamp(1), + headers={'X-Timestamp': Timestamp(1).internal, 'X-Container-Meta-Test': 'Value'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 201) @@ -595,7 +602,7 @@ class TestContainerController(unittest.TestCase): # Set another metadata header, ensuring old one doesn't disappear req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'POST'}, - headers={'X-Timestamp': normalize_timestamp(1), + headers={'X-Timestamp': Timestamp(1).internal, 'X-Container-Meta-Test2': 'Value2'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 204) @@ -607,7 +614,7 @@ class TestContainerController(unittest.TestCase): # Update metadata header req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'}, - headers={'X-Timestamp': normalize_timestamp(3), + headers={'X-Timestamp': Timestamp(3).internal, 'X-Container-Meta-Test': 'New Value'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 202) @@ -619,7 +626,7 @@ class TestContainerController(unittest.TestCase): # Send old update to metadata header req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'}, - headers={'X-Timestamp': normalize_timestamp(2), + headers={'X-Timestamp': Timestamp(2).internal, 'X-Container-Meta-Test': 'Old Value'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 202) @@ -631,7 +638,7 @@ class TestContainerController(unittest.TestCase): # Remove metadata header (by setting it to empty) req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'}, - headers={'X-Timestamp': normalize_timestamp(4), + headers={'X-Timestamp': Timestamp(4).internal, 'X-Container-Meta-Test': ''}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 202) @@ -646,7 +653,7 @@ class TestContainerController(unittest.TestCase): key2 = '%sTest2' % prefix # Set metadata header req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'}, - headers={'X-Timestamp': normalize_timestamp(1), + headers={'X-Timestamp': Timestamp(1).internal, key: 'Value'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 201) @@ -656,7 +663,7 @@ class TestContainerController(unittest.TestCase): self.assertEquals(resp.headers.get(key.lower()), 'Value') # Set another metadata header, ensuring old one doesn't disappear req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'POST'}, - headers={'X-Timestamp': normalize_timestamp(1), + headers={'X-Timestamp': Timestamp(1).internal, key2: 'Value2'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 204) @@ -667,7 +674,7 @@ class TestContainerController(unittest.TestCase): self.assertEquals(resp.headers.get(key2.lower()), 'Value2') # Update metadata header req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'}, - headers={'X-Timestamp': normalize_timestamp(3), + headers={'X-Timestamp': Timestamp(3).internal, key: 'New Value'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 202) @@ -678,7 +685,7 @@ class TestContainerController(unittest.TestCase): 'New Value') # Send old update to metadata header req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'}, - headers={'X-Timestamp': normalize_timestamp(2), + headers={'X-Timestamp': Timestamp(2).internal, key: 'Old Value'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 202) @@ -689,7 +696,7 @@ class TestContainerController(unittest.TestCase): 'New Value') # Remove metadata header (by setting it to empty) req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'}, - headers={'X-Timestamp': normalize_timestamp(4), + headers={'X-Timestamp': Timestamp(4).internal, key: ''}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 202) @@ -725,13 +732,13 @@ class TestContainerController(unittest.TestCase): def test_POST_HEAD_metadata(self): req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'}, - headers={'X-Timestamp': normalize_timestamp(1)}) + headers={'X-Timestamp': Timestamp(1).internal}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 201) # Set metadata header req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'POST'}, - headers={'X-Timestamp': normalize_timestamp(1), + headers={'X-Timestamp': Timestamp(1).internal, 'X-Container-Meta-Test': 'Value'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 204) @@ -742,7 +749,7 @@ class TestContainerController(unittest.TestCase): # Update metadata header req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'POST'}, - headers={'X-Timestamp': normalize_timestamp(3), + headers={'X-Timestamp': Timestamp(3).internal, 'X-Container-Meta-Test': 'New Value'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 204) @@ -754,7 +761,7 @@ class TestContainerController(unittest.TestCase): # Send old update to metadata header req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'POST'}, - headers={'X-Timestamp': normalize_timestamp(2), + headers={'X-Timestamp': Timestamp(2).internal, 'X-Container-Meta-Test': 'Old Value'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 204) @@ -766,7 +773,7 @@ class TestContainerController(unittest.TestCase): # Remove metadata header (by setting it to empty) req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'POST'}, - headers={'X-Timestamp': normalize_timestamp(4), + headers={'X-Timestamp': Timestamp(4).internal, 'X-Container-Meta-Test': ''}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 204) @@ -779,12 +786,12 @@ class TestContainerController(unittest.TestCase): prefix = get_sys_meta_prefix('container') key = '%sTest' % prefix req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'}, - headers={'X-Timestamp': normalize_timestamp(1)}) + headers={'X-Timestamp': Timestamp(1).internal}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 201) # Set metadata header req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'POST'}, - headers={'X-Timestamp': normalize_timestamp(1), + headers={'X-Timestamp': Timestamp(1).internal, key: 'Value'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 204) @@ -794,7 +801,7 @@ class TestContainerController(unittest.TestCase): self.assertEquals(resp.headers.get(key.lower()), 'Value') # Update metadata header req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'POST'}, - headers={'X-Timestamp': normalize_timestamp(3), + headers={'X-Timestamp': Timestamp(3).internal, key: 'New Value'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 204) @@ -805,7 +812,7 @@ class TestContainerController(unittest.TestCase): 'New Value') # Send old update to metadata header req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'POST'}, - headers={'X-Timestamp': normalize_timestamp(2), + headers={'X-Timestamp': Timestamp(2).internal, key: 'Old Value'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 204) @@ -816,7 +823,7 @@ class TestContainerController(unittest.TestCase): 'New Value') # Remove metadata header (by setting it to empty) req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'POST'}, - headers={'X-Timestamp': normalize_timestamp(4), + headers={'X-Timestamp': Timestamp(4).internal, key: ''}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 204) @@ -962,11 +969,11 @@ class TestContainerController(unittest.TestCase): req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'}, - headers={'X-Timestamp': '0000000001.00000', + headers={'X-Timestamp': Timestamp(1).internal, 'X-Account-Host': '%s:%s' % bindsock.getsockname(), 'X-Account-Partition': '123', 'X-Account-Device': 'sda1'}) - event = spawn(accept, 201, '0000000001.00000') + event = spawn(accept, 201, Timestamp(1).internal) try: with Timeout(3): resp = req.get_response(self.controller) @@ -984,11 +991,11 @@ class TestContainerController(unittest.TestCase): req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'}, - headers={'X-Timestamp': '0000000003.00000', + headers={'X-Timestamp': Timestamp(3).internal, 'X-Account-Host': '%s:%s' % bindsock.getsockname(), 'X-Account-Partition': '123', 'X-Account-Device': 'sda1'}) - event = spawn(accept, 404, '0000000003.00000') + event = spawn(accept, 404, Timestamp(3).internal) try: with Timeout(3): resp = req.get_response(self.controller) @@ -1000,11 +1007,11 @@ class TestContainerController(unittest.TestCase): req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'}, - headers={'X-Timestamp': '0000000005.00000', + headers={'X-Timestamp': Timestamp(5).internal, 'X-Account-Host': '%s:%s' % bindsock.getsockname(), 'X-Account-Partition': '123', 'X-Account-Device': 'sda1'}) - event = spawn(accept, 503, '0000000005.00000') + event = spawn(accept, 503, Timestamp(5).internal) got_exc = False try: with Timeout(3): @@ -1125,9 +1132,9 @@ class TestContainerController(unittest.TestCase): self.assertEquals(resp.status_int, 404) # sanity # backend headers expectations = { - 'x-backend-put-timestamp': normalize_timestamp(1), - 'x-backend-delete-timestamp': normalize_timestamp(2), - 'x-backend-status-changed-at': normalize_timestamp(2), + 'x-backend-put-timestamp': Timestamp(1).internal, + 'x-backend-delete-timestamp': Timestamp(2).internal, + 'x-backend-status-changed-at': Timestamp(2).internal, } for header, value in expectations.items(): self.assertEqual(resp.headers[header], value, @@ -1136,9 +1143,9 @@ class TestContainerController(unittest.TestCase): db = self.controller._get_container_broker('sda1', 'p', 'a', 'c') self.assertEqual(True, db.is_deleted()) info = db.get_info() - self.assertEquals(info['put_timestamp'], normalize_timestamp('1')) - self.assertEquals(info['delete_timestamp'], normalize_timestamp('2')) - self.assertEquals(info['status_changed_at'], normalize_timestamp('2')) + self.assertEquals(info['put_timestamp'], Timestamp('1').internal) + self.assertEquals(info['delete_timestamp'], Timestamp('2').internal) + self.assertEquals(info['status_changed_at'], Timestamp('2').internal) # recreate req = Request.blank(path, method='PUT', headers={'X-Timestamp': '4'}) @@ -1147,17 +1154,17 @@ class TestContainerController(unittest.TestCase): db = self.controller._get_container_broker('sda1', 'p', 'a', 'c') self.assertEqual(False, db.is_deleted()) info = db.get_info() - self.assertEquals(info['put_timestamp'], normalize_timestamp('4')) - self.assertEquals(info['delete_timestamp'], normalize_timestamp('2')) - self.assertEquals(info['status_changed_at'], normalize_timestamp('4')) + self.assertEquals(info['put_timestamp'], Timestamp('4').internal) + self.assertEquals(info['delete_timestamp'], Timestamp('2').internal) + self.assertEquals(info['status_changed_at'], Timestamp('4').internal) for method in ('GET', 'HEAD'): req = Request.blank(path) resp = req.get_response(self.controller) expectations = { - 'x-put-timestamp': normalize_timestamp(4), - 'x-backend-put-timestamp': normalize_timestamp(4), - 'x-backend-delete-timestamp': normalize_timestamp(2), - 'x-backend-status-changed-at': normalize_timestamp(4), + 'x-put-timestamp': Timestamp(4).normal, + 'x-backend-put-timestamp': Timestamp(4).internal, + 'x-backend-delete-timestamp': Timestamp(2).internal, + 'x-backend-status-changed-at': Timestamp(4).internal, } for header, expected in expectations.items(): self.assertEqual(resp.headers[header], expected, @@ -1217,8 +1224,8 @@ class TestContainerController(unittest.TestCase): [(exists, db.db_file) for exists in (False, True)]) # info was updated info = db.get_info() - self.assertEquals(info['put_timestamp'], normalize_timestamp('4')) - self.assertEquals(info['delete_timestamp'], normalize_timestamp('2')) + self.assertEquals(info['put_timestamp'], Timestamp('4').internal) + self.assertEquals(info['delete_timestamp'], Timestamp('2').internal) def test_DELETE_not_found(self): # Even if the container wasn't previously heard of, the container @@ -1231,7 +1238,7 @@ class TestContainerController(unittest.TestCase): self.assertEquals(resp.status_int, 404) def test_change_storage_policy_via_DELETE_then_PUT(self): - ts = (normalize_timestamp(t) for t in + ts = (Timestamp(t).internal for t in itertools.count(int(time.time()))) policy = random.choice(list(POLICIES)) req = Request.blank( @@ -1265,7 +1272,7 @@ class TestContainerController(unittest.TestCase): str(other_policy.idx)) def test_change_to_default_storage_policy_via_DELETE_then_PUT(self): - ts = (normalize_timestamp(t) for t in + ts = (Timestamp(t).internal for t in itertools.count(int(time.time()))) non_default_policy = random.choice([p for p in POLICIES if not p.is_default]) @@ -1297,40 +1304,155 @@ class TestContainerController(unittest.TestCase): def test_DELETE_object(self): req = Request.blank( - '/sda1/p/a/c', - environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': '2'}) + '/sda1/p/a/c', method='PUT', headers={ + 'X-Timestamp': Timestamp(2).internal}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 201) req = Request.blank( - '/sda1/p/a/c/o', - environ={'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '0', - 'HTTP_X_SIZE': 1, 'HTTP_X_CONTENT_TYPE': 'text/plain', - 'HTTP_X_ETAG': 'x'}) + '/sda1/p/a/c/o', method='PUT', headers={ + 'X-Timestamp': Timestamp(0).internal, 'X-Size': 1, + 'X-Content-Type': 'text/plain', 'X-Etag': 'x'}) self._update_object_put_headers(req) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 201) - req = Request.blank( - '/sda1/p/a/c', - environ={'REQUEST_METHOD': 'DELETE'}, headers={'X-Timestamp': '3'}) + ts = (Timestamp(t).internal for t in + itertools.count(3)) + req = Request.blank('/sda1/p/a/c', method='DELETE', headers={ + 'X-Timestamp': ts.next()}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 409) - req = Request.blank( - '/sda1/p/a/c/o', - environ={'REQUEST_METHOD': 'DELETE'}, headers={'X-Timestamp': '4'}) + req = Request.blank('/sda1/p/a/c/o', method='DELETE', headers={ + 'X-Timestamp': ts.next()}) self._update_object_put_headers(req) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 204) - req = Request.blank( - '/sda1/p/a/c', - environ={'REQUEST_METHOD': 'DELETE'}, headers={'X-Timestamp': '5'}) + req = Request.blank('/sda1/p/a/c', method='DELETE', headers={ + 'X-Timestamp': ts.next()}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 204) - req = Request.blank( - '/sda1/p/a/c', - environ={'REQUEST_METHOD': 'GET'}, headers={'X-Timestamp': '6'}) + req = Request.blank('/sda1/p/a/c', method='GET', headers={ + 'X-Timestamp': ts.next()}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 404) + def test_object_update_with_offset(self): + ts = (Timestamp(t).internal for t in + itertools.count(int(time.time()))) + # create container + req = Request.blank('/sda1/p/a/c', method='PUT', headers={ + 'X-Timestamp': ts.next()}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 201) + # check status + req = Request.blank('/sda1/p/a/c', method='HEAD') + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 204) + self.assertEqual(int(resp.headers[POLICY_INDEX]), + int(POLICIES.default)) + # create object + obj_timestamp = ts.next() + req = Request.blank( + '/sda1/p/a/c/o', method='PUT', headers={ + 'X-Timestamp': obj_timestamp, 'X-Size': 1, + 'X-Content-Type': 'text/plain', 'X-Etag': 'x'}) + self._update_object_put_headers(req) + resp = req.get_response(self.controller) + self.assertEquals(resp.status_int, 201) + # check listing + req = Request.blank('/sda1/p/a/c', method='GET', + query_string='format=json') + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 200) + self.assertEqual(int(resp.headers['X-Container-Object-Count']), 1) + self.assertEqual(int(resp.headers['X-Container-Bytes-Used']), 1) + listing_data = json.loads(resp.body) + self.assertEqual(1, len(listing_data)) + for obj in listing_data: + self.assertEqual(obj['name'], 'o') + self.assertEqual(obj['bytes'], 1) + self.assertEqual(obj['hash'], 'x') + self.assertEqual(obj['content_type'], 'text/plain') + # send an update with an offset + offset_timestamp = Timestamp(obj_timestamp, offset=1).internal + req = Request.blank( + '/sda1/p/a/c/o', method='PUT', headers={ + 'X-Timestamp': offset_timestamp, 'X-Size': 2, + 'X-Content-Type': 'text/html', 'X-Etag': 'y'}) + self._update_object_put_headers(req) + resp = req.get_response(self.controller) + self.assertEquals(resp.status_int, 201) + # check updated listing + req = Request.blank('/sda1/p/a/c', method='GET', + query_string='format=json') + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 200) + self.assertEqual(int(resp.headers['X-Container-Object-Count']), 1) + self.assertEqual(int(resp.headers['X-Container-Bytes-Used']), 2) + listing_data = json.loads(resp.body) + self.assertEqual(1, len(listing_data)) + for obj in listing_data: + self.assertEqual(obj['name'], 'o') + self.assertEqual(obj['bytes'], 2) + self.assertEqual(obj['hash'], 'y') + self.assertEqual(obj['content_type'], 'text/html') + # now overwrite with a newer time + delete_timestamp = ts.next() + req = Request.blank( + '/sda1/p/a/c/o', method='DELETE', headers={ + 'X-Timestamp': delete_timestamp}) + self._update_object_put_headers(req) + resp = req.get_response(self.controller) + self.assertEquals(resp.status_int, 204) + # check empty listing + req = Request.blank('/sda1/p/a/c', method='GET', + query_string='format=json') + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 200) + self.assertEqual(int(resp.headers['X-Container-Object-Count']), 0) + self.assertEqual(int(resp.headers['X-Container-Bytes-Used']), 0) + listing_data = json.loads(resp.body) + self.assertEqual(0, len(listing_data)) + # recreate with an offset + offset_timestamp = Timestamp(delete_timestamp, offset=1).internal + req = Request.blank( + '/sda1/p/a/c/o', method='PUT', headers={ + 'X-Timestamp': offset_timestamp, 'X-Size': 3, + 'X-Content-Type': 'text/enriched', 'X-Etag': 'z'}) + self._update_object_put_headers(req) + resp = req.get_response(self.controller) + self.assertEquals(resp.status_int, 201) + # check un-deleted listing + req = Request.blank('/sda1/p/a/c', method='GET', + query_string='format=json') + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 200) + self.assertEqual(int(resp.headers['X-Container-Object-Count']), 1) + self.assertEqual(int(resp.headers['X-Container-Bytes-Used']), 3) + listing_data = json.loads(resp.body) + self.assertEqual(1, len(listing_data)) + for obj in listing_data: + self.assertEqual(obj['name'], 'o') + self.assertEqual(obj['bytes'], 3) + self.assertEqual(obj['hash'], 'z') + self.assertEqual(obj['content_type'], 'text/enriched') + # delete offset with newer offset + delete_timestamp = Timestamp(offset_timestamp, offset=1).internal + req = Request.blank( + '/sda1/p/a/c/o', method='DELETE', headers={ + 'X-Timestamp': delete_timestamp}) + self._update_object_put_headers(req) + resp = req.get_response(self.controller) + self.assertEquals(resp.status_int, 204) + # check empty listing + req = Request.blank('/sda1/p/a/c', method='GET', + query_string='format=json') + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 200) + self.assertEqual(int(resp.headers['X-Container-Object-Count']), 0) + self.assertEqual(int(resp.headers['X-Container-Bytes-Used']), 0) + listing_data = json.loads(resp.body) + self.assertEqual(0, len(listing_data)) + def test_DELETE_account_update(self): bindsock = listen(('127.0.0.1', 0)) @@ -1365,11 +1487,11 @@ class TestContainerController(unittest.TestCase): req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'DELETE'}, - headers={'X-Timestamp': '0000000002.00000', + headers={'X-Timestamp': Timestamp(2).internal, 'X-Account-Host': '%s:%s' % bindsock.getsockname(), 'X-Account-Partition': '123', 'X-Account-Device': 'sda1'}) - event = spawn(accept, 204, '0000000002.00000') + event = spawn(accept, 204, Timestamp(2).internal) try: with Timeout(3): resp = req.get_response(self.controller) @@ -1379,18 +1501,18 @@ class TestContainerController(unittest.TestCase): if err: raise Exception(err) req = Request.blank( - '/sda1/p/a/c', - environ={'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '2'}) + '/sda1/p/a/c', method='PUT', headers={ + 'X-Timestamp': Timestamp(2).internal}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 201) req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'DELETE'}, - headers={'X-Timestamp': '0000000003.00000', + headers={'X-Timestamp': Timestamp(3).internal, 'X-Account-Host': '%s:%s' % bindsock.getsockname(), 'X-Account-Partition': '123', 'X-Account-Device': 'sda1'}) - event = spawn(accept, 404, '0000000003.00000') + event = spawn(accept, 404, Timestamp(3).internal) try: with Timeout(3): resp = req.get_response(self.controller) @@ -1400,18 +1522,18 @@ class TestContainerController(unittest.TestCase): if err: raise Exception(err) req = Request.blank( - '/sda1/p/a/c', - environ={'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '4'}) + '/sda1/p/a/c', method='PUT', headers={ + 'X-Timestamp': Timestamp(4).internal}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 201) req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'DELETE'}, - headers={'X-Timestamp': '0000000005.00000', + headers={'X-Timestamp': Timestamp(5).internal, 'X-Account-Host': '%s:%s' % bindsock.getsockname(), 'X-Account-Partition': '123', 'X-Account-Device': 'sda1'}) - event = spawn(accept, 503, '0000000005.00000') + event = spawn(accept, 503, Timestamp(5).internal) got_exc = False try: with Timeout(3): @@ -1785,18 +1907,11 @@ class TestContainerController(unittest.TestCase): self.assertEquals(result, [u'\u2603', 'text/plain;charset="utf-8"']) def test_GET_accept_not_valid(self): - req = Request.blank( - '/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT', - 'HTTP_X_TIMESTAMP': '0'}) - req.get_response(self.controller) - req = Request.blank('/sda1/p/a/c1', environ={'REQUEST_METHOD': 'PUT'}, - headers={'X-Put-Timestamp': '1', - 'X-Delete-Timestamp': '0', - 'X-Object-Count': '0', - 'X-Bytes-Used': '0', - 'X-Timestamp': normalize_timestamp(0)}) - req.get_response(self.controller) - req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'GET'}) + req = Request.blank('/sda1/p/a/c', method='PUT', headers={ + 'X-Timestamp': Timestamp(0).internal}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 201) + req = Request.blank('/sda1/p/a/c', method='GET') req.accept = 'application/xml*' resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 406) @@ -2080,13 +2195,12 @@ class TestContainerController(unittest.TestCase): def test_params_format(self): req = Request.blank( - '/sda1/p/a/c', - headers={'X-Timestamp': normalize_timestamp(1)}, - environ={'REQUEST_METHOD': 'PUT'}) + '/sda1/p/a/c', method='PUT', + headers={'X-Timestamp': Timestamp(1).internal}) req.get_response(self.controller) for format in ('xml', 'json'): req = Request.blank('/sda1/p/a/c?format=%s' % format, - environ={'REQUEST_METHOD': 'GET'}) + method='GET') resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 200) @@ -2105,9 +2219,8 @@ class TestContainerController(unittest.TestCase): resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 412, "%d on param delimiter" % (resp.status_int)) - req = Request.blank('/sda1/p/a/c', - headers={'X-Timestamp': normalize_timestamp(1)}, - environ={'REQUEST_METHOD': 'PUT'}) + req = Request.blank('/sda1/p/a/c', method='PUT', + headers={'X-Timestamp': Timestamp(1).internal}) req.get_response(self.controller) # Good UTF8 sequence, ignored for limit, doesn't affect other queries for param in ('limit', 'marker', 'path', 'prefix', 'end_marker', @@ -2119,7 +2232,7 @@ class TestContainerController(unittest.TestCase): "%d on param %s" % (resp.status_int, param)) def test_put_auto_create(self): - headers = {'x-timestamp': normalize_timestamp(1), + headers = {'x-timestamp': Timestamp(1).internal, 'x-size': '0', 'x-content-type': 'text/plain', 'x-etag': 'd41d8cd98f00b204e9800998ecf8427e'} @@ -2149,7 +2262,7 @@ class TestContainerController(unittest.TestCase): self.assertEquals(resp.status_int, 404) def test_delete_auto_create(self): - headers = {'x-timestamp': normalize_timestamp(1)} + headers = {'x-timestamp': Timestamp(1).internal} req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'DELETE'}, @@ -2177,7 +2290,7 @@ class TestContainerController(unittest.TestCase): def test_content_type_on_HEAD(self): Request.blank('/sda1/p/a/o', - headers={'X-Timestamp': normalize_timestamp(1)}, + headers={'X-Timestamp': Timestamp(1).internal}, environ={'REQUEST_METHOD': 'PUT'}).get_response( self.controller) @@ -2267,7 +2380,7 @@ class TestContainerController(unittest.TestCase): 'x-bytes-used': 0, 'x-delete-timestamp': '0', 'x-object-count': 0, - 'x-put-timestamp': '0000012345.00000', + 'x-put-timestamp': Timestamp(12345).internal, POLICY_INDEX: '%s' % POLICIES.default.idx, 'referer': 'PUT http://localhost/sda1/p/a/c', 'user-agent': 'container-server %d' % os.getpid(), @@ -2285,7 +2398,7 @@ class TestContainerController(unittest.TestCase): 'x-bytes-used': 0, 'x-delete-timestamp': '0', 'x-object-count': 0, - 'x-put-timestamp': '0000012345.00000', + 'x-put-timestamp': Timestamp(12345).internal, POLICY_INDEX: '%s' % POLICIES.default.idx, 'referer': 'PUT http://localhost/sda1/p/a/c', 'user-agent': 'container-server %d' % os.getpid(), diff --git a/test/unit/obj/test_diskfile.py b/test/unit/obj/test_diskfile.py index c5160c11f0..43bd85d55b 100644 --- a/test/unit/obj/test_diskfile.py +++ b/test/unit/obj/test_diskfile.py @@ -38,7 +38,7 @@ from test.unit import (FakeLogger, mock as unit_mock, temptree, from swift.obj import diskfile from swift.common import utils -from swift.common.utils import hash_path, mkdirs, normalize_timestamp +from swift.common.utils import hash_path, mkdirs, Timestamp from swift.common import ring from swift.common.exceptions import DiskFileNotExist, DiskFileQuarantined, \ DiskFileDeviceUnavailable, DiskFileDeleted, DiskFileNotOpen, \ @@ -254,7 +254,7 @@ class TestDiskFileModuleMethods(unittest.TestCase): mkdirs(df._datadir) f = open( os.path.join(df._datadir, - normalize_timestamp(time() - 100) + '.ts'), + Timestamp(time() - 100).internal + '.ts'), 'wb') f.write('1234567890') f.close() @@ -272,7 +272,7 @@ class TestDiskFileModuleMethods(unittest.TestCase): mkdirs(df._datadir) f = open( os.path.join(df._datadir, - normalize_timestamp(time() - 100) + '.ts'), + Timestamp(time() - 100).internal + '.ts'), 'wb') f.write('1234567890') f.close() @@ -304,7 +304,7 @@ class TestDiskFileModuleMethods(unittest.TestCase): f = open( os.path.join( df._datadir, - normalize_timestamp(int(time()) - tdiff) + suff), + Timestamp(int(time()) - tdiff).internal + suff), 'wb') f.write('1234567890') f.close() @@ -330,7 +330,7 @@ class TestDiskFileModuleMethods(unittest.TestCase): f = open( os.path.join( df._datadir, - normalize_timestamp(int(time()) - tdiff) + suff), + Timestamp(int(time()) - tdiff).internal + suff), 'wb') f.write('1234567890') f.close() @@ -412,7 +412,7 @@ class TestDiskFileModuleMethods(unittest.TestCase): mkdirs(df._datadir) with open( os.path.join(df._datadir, - normalize_timestamp(time()) + '.ts'), + Timestamp(time()).internal + '.ts'), 'wb') as f: f.write('1234567890') part = os.path.join(self.objects, '0') @@ -442,7 +442,7 @@ class TestDiskFileModuleMethods(unittest.TestCase): mkdirs(df._datadir) with open( os.path.join(df._datadir, - normalize_timestamp(time()) + '.ts'), + Timestamp(time()).internal + '.ts'), 'wb') as f: f.write('1234567890') part = os.path.join(self.objects, '0') @@ -462,7 +462,7 @@ class TestDiskFileModuleMethods(unittest.TestCase): mkdirs(df._datadir) with open( os.path.join(df._datadir, - normalize_timestamp(time()) + '.ts'), + Timestamp(time()).internal + '.ts'), 'wb') as f: f.write('1234567890') part = os.path.join(self.objects, '0') @@ -479,7 +479,7 @@ class TestDiskFileModuleMethods(unittest.TestCase): mkdirs(df._datadir) with open( os.path.join(df._datadir, - normalize_timestamp(time()) + '.ts'), + Timestamp(time()).internal + '.ts'), 'wb') as f: f.write('1234567890') part = os.path.join(self.objects, '0') @@ -516,7 +516,7 @@ class TestDiskFileModuleMethods(unittest.TestCase): mkdirs(df._datadir) with open( os.path.join(df._datadir, - normalize_timestamp(time()) + '.ts'), + Timestamp(time()).internal + '.ts'), 'wb') as f: f.write('1234567890') part = os.path.join(self.objects, '0') @@ -554,89 +554,89 @@ class TestDiskFileModuleMethods(unittest.TestCase): def test_hash_cleanup_listdir_purge_data_newer_ts(self): # purge .data if there's a newer .ts - file1 = normalize_timestamp(time()) + '.data' - file2 = normalize_timestamp(time() + 1) + '.ts' + file1 = Timestamp(time()).internal + '.data' + file2 = Timestamp(time() + 1).internal + '.ts' file_list = [file1, file2] self.check_hash_cleanup_listdir(file_list, [file2]) def test_hash_cleanup_listdir_purge_ts_newer_data(self): # purge .ts if there's a newer .data - file1 = normalize_timestamp(time()) + '.ts' - file2 = normalize_timestamp(time() + 1) + '.data' + file1 = Timestamp(time()).internal + '.ts' + file2 = Timestamp(time() + 1).internal + '.data' file_list = [file1, file2] self.check_hash_cleanup_listdir(file_list, [file2]) def test_hash_cleanup_listdir_keep_meta_data_purge_ts(self): # keep .meta and .data if meta newer than data and purge .ts - file1 = normalize_timestamp(time()) + '.ts' - file2 = normalize_timestamp(time() + 1) + '.data' - file3 = normalize_timestamp(time() + 2) + '.meta' + file1 = Timestamp(time()).internal + '.ts' + file2 = Timestamp(time() + 1).internal + '.data' + file3 = Timestamp(time() + 2).internal + '.meta' file_list = [file1, file2, file3] self.check_hash_cleanup_listdir(file_list, [file3, file2]) def test_hash_cleanup_listdir_keep_one_ts(self): # keep only latest of multiple .ts files - file1 = normalize_timestamp(time()) + '.ts' - file2 = normalize_timestamp(time() + 1) + '.ts' - file3 = normalize_timestamp(time() + 2) + '.ts' + file1 = Timestamp(time()).internal + '.ts' + file2 = Timestamp(time() + 1).internal + '.ts' + file3 = Timestamp(time() + 2).internal + '.ts' file_list = [file1, file2, file3] self.check_hash_cleanup_listdir(file_list, [file3]) def test_hash_cleanup_listdir_keep_one_data(self): # keep only latest of multiple .data files - file1 = normalize_timestamp(time()) + '.data' - file2 = normalize_timestamp(time() + 1) + '.data' - file3 = normalize_timestamp(time() + 2) + '.data' + file1 = Timestamp(time()).internal + '.data' + file2 = Timestamp(time() + 1).internal + '.data' + file3 = Timestamp(time() + 2).internal + '.data' file_list = [file1, file2, file3] self.check_hash_cleanup_listdir(file_list, [file3]) def test_hash_cleanup_listdir_keep_one_meta(self): # keep only latest of multiple .meta files - file1 = normalize_timestamp(time()) + '.data' - file2 = normalize_timestamp(time() + 1) + '.meta' - file3 = normalize_timestamp(time() + 2) + '.meta' + file1 = Timestamp(time()).internal + '.data' + file2 = Timestamp(time() + 1).internal + '.meta' + file3 = Timestamp(time() + 2).internal + '.meta' file_list = [file1, file2, file3] self.check_hash_cleanup_listdir(file_list, [file3, file1]) def test_hash_cleanup_listdir_ignore_orphaned_ts(self): # A more recent orphaned .meta file will prevent old .ts files # from being cleaned up otherwise - file1 = normalize_timestamp(time()) + '.ts' - file2 = normalize_timestamp(time() + 1) + '.ts' - file3 = normalize_timestamp(time() + 2) + '.meta' + file1 = Timestamp(time()).internal + '.ts' + file2 = Timestamp(time() + 1).internal + '.ts' + file3 = Timestamp(time() + 2).internal + '.meta' file_list = [file1, file2, file3] self.check_hash_cleanup_listdir(file_list, [file3, file2]) def test_hash_cleanup_listdir_purge_old_data_only(self): # Oldest .data will be purge, .meta and .ts won't be touched - file1 = normalize_timestamp(time()) + '.data' - file2 = normalize_timestamp(time() + 1) + '.ts' - file3 = normalize_timestamp(time() + 2) + '.meta' + file1 = Timestamp(time()).internal + '.data' + file2 = Timestamp(time() + 1).internal + '.ts' + file3 = Timestamp(time() + 2).internal + '.meta' file_list = [file1, file2, file3] self.check_hash_cleanup_listdir(file_list, [file3, file2]) def test_hash_cleanup_listdir_purge_old_ts(self): # A single old .ts file will be removed - file1 = normalize_timestamp(time() - (diskfile.ONE_WEEK + 1)) + '.ts' + file1 = Timestamp(time() - (diskfile.ONE_WEEK + 1)).internal + '.ts' file_list = [file1] self.check_hash_cleanup_listdir(file_list, []) def test_hash_cleanup_listdir_meta_keeps_old_ts(self): # An orphaned .meta will not clean up a very old .ts - file1 = normalize_timestamp(time() - (diskfile.ONE_WEEK + 1)) + '.ts' - file2 = normalize_timestamp(time() + 2) + '.meta' + file1 = Timestamp(time() - (diskfile.ONE_WEEK + 1)).internal + '.ts' + file2 = Timestamp(time() + 2).internal + '.meta' file_list = [file1, file2] self.check_hash_cleanup_listdir(file_list, [file2, file1]) def test_hash_cleanup_listdir_keep_single_old_data(self): # A single old .data file will not be removed - file1 = normalize_timestamp(time() - (diskfile.ONE_WEEK + 1)) + '.data' + file1 = Timestamp(time() - (diskfile.ONE_WEEK + 1)).internal + '.data' file_list = [file1] self.check_hash_cleanup_listdir(file_list, [file1]) def test_hash_cleanup_listdir_keep_single_old_meta(self): # A single old .meta file will not be removed - file1 = normalize_timestamp(time() - (diskfile.ONE_WEEK + 1)) + '.meta' + file1 = Timestamp(time() - (diskfile.ONE_WEEK + 1)).internal + '.meta' file_list = [file1] self.check_hash_cleanup_listdir(file_list, [file1]) @@ -865,7 +865,7 @@ class TestDiskFileManager(unittest.TestCase): def test_pickle_async_update(self): self.df_mgr.logger.increment = mock.MagicMock() - ts = normalize_timestamp(10000.0) + ts = Timestamp(10000.0).internal with mock.patch('swift.obj.diskfile.write_pickle') as wp: self.df_mgr.pickle_async_update(self.existing_device1, 'a', 'c', 'o', @@ -981,11 +981,11 @@ class TestDiskFile(unittest.TestCase): mkdirs(df._datadir) if timestamp is None: timestamp = time() - timestamp = normalize_timestamp(timestamp) + timestamp = Timestamp(timestamp).internal if not metadata: metadata = {} if 'X-Timestamp' not in metadata: - metadata['X-Timestamp'] = normalize_timestamp(timestamp) + metadata['X-Timestamp'] = Timestamp(timestamp).internal if 'ETag' not in metadata: etag = md5() etag.update(data) @@ -1038,13 +1038,13 @@ class TestDiskFile(unittest.TestCase): def test_get_metadata(self): df = self._create_test_file('1234567890', timestamp=42) md = df.get_metadata() - self.assertEqual(md['X-Timestamp'], normalize_timestamp(42)) + self.assertEqual(md['X-Timestamp'], Timestamp(42).internal) def test_read_metadata(self): self._create_test_file('1234567890', timestamp=42) df = self._simple_get_diskfile() md = df.read_metadata() - self.assertEqual(md['X-Timestamp'], normalize_timestamp(42)) + self.assertEqual(md['X-Timestamp'], Timestamp(42).internal) def test_get_metadata_not_opened(self): df = self._simple_get_diskfile() @@ -1069,7 +1069,7 @@ class TestDiskFile(unittest.TestCase): self.assertEquals('1024', df._metadata['Content-Length']) # write some new metadata (fast POST, don't send orig meta, ts 42) df = self._simple_get_diskfile() - df.write_metadata({'X-Timestamp': normalize_timestamp(42), + df.write_metadata({'X-Timestamp': Timestamp(42).internal, 'X-Object-Meta-Key2': 'Value2'}) df = self._simple_get_diskfile() with df.open(): @@ -1240,7 +1240,7 @@ class TestDiskFile(unittest.TestCase): if ts: timestamp = ts else: - timestamp = normalize_timestamp(time()) + timestamp = Timestamp(time()).internal if prealloc: prealloc_size = fsize else: @@ -1578,7 +1578,7 @@ class TestDiskFile(unittest.TestCase): def test_write_metadata(self): df = self._create_test_file('1234567890') - timestamp = normalize_timestamp(time()) + timestamp = Timestamp(time()).internal metadata = {'X-Timestamp': timestamp, 'X-Object-Meta-test': 'data'} df.write_metadata(metadata) dl = os.listdir(df._datadir) @@ -1590,7 +1590,7 @@ class TestDiskFile(unittest.TestCase): df = self._get_open_disk_file() ts = time() df.delete(ts) - exp_name = '%s.ts' % str(normalize_timestamp(ts)) + exp_name = '%s.ts' % Timestamp(ts).internal dl = os.listdir(df._datadir) self.assertEquals(len(dl), 1) self.assertTrue(exp_name in set(dl)) @@ -1599,7 +1599,7 @@ class TestDiskFile(unittest.TestCase): df = self._get_open_disk_file() ts = time() df.delete(ts) - exp_name = '%s.ts' % str(normalize_timestamp(ts)) + exp_name = '%s.ts' % str(Timestamp(ts).internal) dl = os.listdir(df._datadir) self.assertEquals(len(dl), 1) self.assertTrue(exp_name in set(dl)) @@ -1610,7 +1610,7 @@ class TestDiskFile(unittest.TestCase): df = self._get_open_disk_file() ts = time() df.delete(ts) - exp_name = '%s.ts' % str(normalize_timestamp(ts)) + exp_name = '%s.ts' % str(Timestamp(ts).internal) dl = os.listdir(df._datadir) self.assertEquals(len(dl), 1) self.assertTrue(exp_name in set(dl)) @@ -1685,7 +1685,7 @@ class TestDiskFile(unittest.TestCase): try: df.open() except DiskFileDeleted as d: - self.assertEquals(d.timestamp, normalize_timestamp(10)) + self.assertEquals(d.timestamp, Timestamp(10).internal) else: self.fail("Expected DiskFileDeleted exception") @@ -1701,7 +1701,7 @@ class TestDiskFile(unittest.TestCase): try: df.open() except DiskFileDeleted as d: - self.assertEquals(d.timestamp, normalize_timestamp(8)) + self.assertEquals(d.timestamp, Timestamp(8).internal) else: self.fail("Expected DiskFileDeleted exception") @@ -1717,7 +1717,7 @@ class TestDiskFile(unittest.TestCase): with df.open(): self.assertTrue('X-Timestamp' in df._metadata) self.assertEquals(df._metadata['X-Timestamp'], - normalize_timestamp(10)) + Timestamp(10).internal) self.assertTrue('deleted' not in df._metadata) def test_ondisk_search_loop_data_meta_ts(self): @@ -1732,7 +1732,7 @@ class TestDiskFile(unittest.TestCase): with df.open(): self.assertTrue('X-Timestamp' in df._metadata) self.assertEquals(df._metadata['X-Timestamp'], - normalize_timestamp(10)) + Timestamp(10).internal) self.assertTrue('deleted' not in df._metadata) def test_ondisk_search_loop_wayward_files_ignored(self): @@ -1748,7 +1748,7 @@ class TestDiskFile(unittest.TestCase): with df.open(): self.assertTrue('X-Timestamp' in df._metadata) self.assertEquals(df._metadata['X-Timestamp'], - normalize_timestamp(10)) + Timestamp(10).internal) self.assertTrue('deleted' not in df._metadata) def test_ondisk_search_loop_listdir_error(self): @@ -1995,8 +1995,8 @@ class TestDiskFile(unittest.TestCase): suffixes=['456'])), []) def test_yield_hashes(self): - fresh_ts = normalize_timestamp(time() - 10) - fresher_ts = normalize_timestamp(time() - 1) + fresh_ts = Timestamp(time() - 10).internal + fresher_ts = Timestamp(time() - 1).internal def _listdir(path): if path.endswith('/dev/objects/9'): @@ -2037,8 +2037,8 @@ class TestDiskFile(unittest.TestCase): '9373a92d072897b136b3fc06595b7456', fresher_ts)]) def test_yield_hashes_suffixes(self): - fresh_ts = normalize_timestamp(time() - 10) - fresher_ts = normalize_timestamp(time() - 1) + fresh_ts = Timestamp(time() - 10).internal + fresher_ts = Timestamp(time() - 1).internal def _listdir(path): if path.endswith('/dev/objects/9'): @@ -2095,7 +2095,7 @@ class TestDiskFile(unittest.TestCase): df = self._get_open_disk_file() ts = time() df.delete(ts) - exp_name = '%s.ts' % str(normalize_timestamp(ts)) + exp_name = '%s.ts' % str(Timestamp(ts).internal) dl = os.listdir(df._datadir) self.assertEquals(len(dl), 1) self.assertTrue(exp_name in set(dl)) @@ -2127,7 +2127,7 @@ class TestDiskFile(unittest.TestCase): df = self._get_open_disk_file() ts = time() df.delete(ts) - exp_name = '%s.ts' % str(normalize_timestamp(ts)) + exp_name = '%s.ts' % str(Timestamp(ts).internal) dl = os.listdir(df._datadir) self.assertEquals(len(dl), 1) self.assertTrue(exp_name in set(dl)) @@ -2159,7 +2159,7 @@ class TestDiskFile(unittest.TestCase): df.delete(ts) except OSError: self.fail("OSError raised when it should have been swallowed") - exp_name = '%s.ts' % str(normalize_timestamp(ts)) + exp_name = '%s.ts' % str(Timestamp(ts).internal) dl = os.listdir(df._datadir) self.assertEquals(len(dl), 2) self.assertTrue(exp_name in set(dl)) diff --git a/test/unit/obj/test_server.py b/test/unit/obj/test_server.py index b9f9345159..a293ae99cf 100755 --- a/test/unit/obj/test_server.py +++ b/test/unit/obj/test_server.py @@ -554,11 +554,11 @@ class TestObjectController(unittest.TestCase): self.testdir, 'sda1', storage_directory(diskfile.get_data_dir(0), 'p', hash_path('a', 'c', 'o')), - timestamp + '.data') + utils.Timestamp(timestamp).internal + '.data') self.assert_(os.path.isfile(objfile)) self.assertEquals(open(objfile).read(), 'VERIFY') self.assertEquals(diskfile.read_metadata(objfile), - {'X-Timestamp': timestamp, + {'X-Timestamp': utils.Timestamp(timestamp).internal, 'Content-Length': '6', 'ETag': '0b4c12d7e0a73840c1c4f148fda3b037', 'Content-Type': 'application/octet-stream', @@ -587,11 +587,11 @@ class TestObjectController(unittest.TestCase): self.testdir, 'sda1', storage_directory(diskfile.get_data_dir(0), 'p', hash_path('a', 'c', 'o')), - timestamp + '.data') + utils.Timestamp(timestamp).internal + '.data') self.assert_(os.path.isfile(objfile)) self.assertEquals(open(objfile).read(), 'VERIFY TWO') self.assertEquals(diskfile.read_metadata(objfile), - {'X-Timestamp': timestamp, + {'X-Timestamp': utils.Timestamp(timestamp).internal, 'Content-Length': '10', 'ETag': 'b381a4c5dab1eaa1eb9711fa647cd039', 'Content-Type': 'text/plain', @@ -622,11 +622,11 @@ class TestObjectController(unittest.TestCase): self.testdir, 'sda1', storage_directory(diskfile.get_data_dir(0), 'p', hash_path('a', 'c', 'o')), - timestamp + '.data') + utils.Timestamp(timestamp).internal + '.data') self.assertTrue(os.path.isfile(objfile)) self.assertEqual(open(objfile).read(), 'VERIFY TWO') self.assertEqual(diskfile.read_metadata(objfile), - {'X-Timestamp': timestamp, + {'X-Timestamp': utils.Timestamp(timestamp).internal, 'Content-Length': '10', 'ETag': 'b381a4c5dab1eaa1eb9711fa647cd039', 'Content-Type': 'text/plain', @@ -696,11 +696,11 @@ class TestObjectController(unittest.TestCase): self.testdir, 'sda1', storage_directory(diskfile.get_data_dir(0), 'p', hash_path('a', 'c', 'o')), - timestamp + '.data') + utils.Timestamp(timestamp).internal + '.data') self.assert_(os.path.isfile(objfile)) self.assertEquals(open(objfile).read(), 'VERIFY THREE') self.assertEquals(diskfile.read_metadata(objfile), - {'X-Timestamp': timestamp, + {'X-Timestamp': utils.Timestamp(timestamp).internal, 'Content-Length': '12', 'ETag': 'b114ab7b90d9ccac4bd5d99cc7ebb568', 'Content-Type': 'text/plain', @@ -843,7 +843,7 @@ class TestObjectController(unittest.TestCase): self.testdir, 'sda1', storage_directory(diskfile.get_data_dir(0), 'p', hash_path('a', 'c', 'o')), - timestamp + '.data') + utils.Timestamp(timestamp).internal + '.data') os.unlink(objfile) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'HEAD'}) @@ -873,7 +873,8 @@ class TestObjectController(unittest.TestCase): environ={'REQUEST_METHOD': 'HEAD'}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 404) - self.assertEquals(resp.headers['X-Backend-Timestamp'], timestamp) + self.assertEquals(resp.headers['X-Backend-Timestamp'], + utils.Timestamp(timestamp).internal) def test_HEAD_quarantine_zbyte(self): # Test swift.obj.server.ObjectController.GET @@ -969,7 +970,7 @@ class TestObjectController(unittest.TestCase): self.testdir, 'sda1', storage_directory(diskfile.get_data_dir(0), 'p', hash_path('a', 'c', 'o')), - timestamp + '.data') + utils.Timestamp(timestamp).internal + '.data') os.unlink(objfile) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'GET'}) resp = req.get_response(self.object_controller) @@ -997,7 +998,8 @@ class TestObjectController(unittest.TestCase): req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'GET'}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 404) - self.assertEquals(resp.headers['X-Backend-Timestamp'], timestamp) + self.assertEquals(resp.headers['X-Backend-Timestamp'], + utils.Timestamp(timestamp).internal) def test_GET_if_match(self): req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, @@ -1581,7 +1583,7 @@ class TestObjectController(unittest.TestCase): self.testdir, 'sda1', storage_directory(diskfile.get_data_dir(0), 'p', hash_path('a', 'c', 'o')), - timestamp + '.ts') + utils.Timestamp(timestamp).internal + '.ts') self.assertTrue(os.path.isfile(ts_1000_file)) # There should now be a 1000 ts file. self.assertEquals(len(os.listdir(os.path.dirname(ts_1000_file))), 1) @@ -1597,7 +1599,7 @@ class TestObjectController(unittest.TestCase): self.testdir, 'sda1', storage_directory(diskfile.get_data_dir(0), 'p', hash_path('a', 'c', 'o')), - timestamp + '.ts') + utils.Timestamp(timestamp).internal + '.ts') self.assertFalse(os.path.isfile(ts_999_file)) self.assertTrue(os.path.isfile(ts_1000_file)) self.assertEquals(len(os.listdir(os.path.dirname(ts_1000_file))), 1) @@ -1617,7 +1619,7 @@ class TestObjectController(unittest.TestCase): self.testdir, 'sda1', storage_directory(diskfile.get_data_dir(0), 'p', hash_path('a', 'c', 'o')), - timestamp + '.data') + utils.Timestamp(timestamp).internal + '.data') self.assertTrue(os.path.isfile(data_1002_file)) self.assertEquals(len(os.listdir(os.path.dirname(data_1002_file))), 1) @@ -1632,7 +1634,7 @@ class TestObjectController(unittest.TestCase): self.testdir, 'sda1', storage_directory(diskfile.get_data_dir(0), 'p', hash_path('a', 'c', 'o')), - timestamp + '.ts') + utils.Timestamp(timestamp).internal + '.ts') self.assertFalse(os.path.isfile(ts_1001_file)) self.assertTrue(os.path.isfile(data_1002_file)) self.assertEquals(len(os.listdir(os.path.dirname(ts_1001_file))), 1) @@ -1647,7 +1649,7 @@ class TestObjectController(unittest.TestCase): self.testdir, 'sda1', storage_directory(diskfile.get_data_dir(0), 'p', hash_path('a', 'c', 'o')), - timestamp + '.ts') + utils.Timestamp(timestamp).internal + '.ts') self.assertTrue(os.path.isfile(ts_1003_file)) self.assertEquals(len(os.listdir(os.path.dirname(ts_1003_file))), 1) @@ -1655,10 +1657,11 @@ class TestObjectController(unittest.TestCase): # Test swift.obj.server.ObjectController.DELETE and container # updates, making sure container update is called in the correct # state. - timestamp = normalize_timestamp(time()) + start = time() + timestamp = utils.Timestamp(start) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={ - 'X-Timestamp': timestamp, + 'X-Timestamp': timestamp.internal, 'Content-Type': 'application/octet-stream', 'Content-Length': '4', }) @@ -1676,17 +1679,17 @@ class TestObjectController(unittest.TestCase): try: # The following request should return 409 (HTTP Conflict). A # tombstone file should not have been created with this timestamp. - timestamp = normalize_timestamp(float(timestamp) - 1) + timestamp = utils.Timestamp(start - 0.00001) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'DELETE'}, - headers={'X-Timestamp': timestamp}) + headers={'X-Timestamp': timestamp.internal}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 409) objfile = os.path.join( self.testdir, 'sda1', storage_directory(diskfile.get_data_dir(0), 'p', hash_path('a', 'c', 'o')), - timestamp + '.ts') + utils.Timestamp(timestamp).internal + '.ts') self.assertFalse(os.path.isfile(objfile)) self.assertEquals(len(os.listdir(os.path.dirname(objfile))), 1) self.assertEquals(0, calls_made[0]) @@ -1695,18 +1698,17 @@ class TestObjectController(unittest.TestCase): # be truly deleted (container update is performed) because this # timestamp is newer. A tombstone file should have been created # with this timestamp. - sleep(.00001) - timestamp = normalize_timestamp(time()) + timestamp = utils.Timestamp(start + 0.00001) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'DELETE'}, - headers={'X-Timestamp': timestamp}) + headers={'X-Timestamp': timestamp.internal}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 204) objfile = os.path.join( self.testdir, 'sda1', storage_directory(diskfile.get_data_dir(0), 'p', hash_path('a', 'c', 'o')), - timestamp + '.ts') + utils.Timestamp(timestamp).internal + '.ts') self.assert_(os.path.isfile(objfile)) self.assertEquals(1, calls_made[0]) self.assertEquals(len(os.listdir(os.path.dirname(objfile))), 1) @@ -1715,18 +1717,17 @@ class TestObjectController(unittest.TestCase): # already have been deleted, but it should have also performed a # container update because the timestamp is newer, and a tombstone # file should also exist with this timestamp. - sleep(.00001) - timestamp = normalize_timestamp(time()) + timestamp = utils.Timestamp(start + 0.00002) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'DELETE'}, - headers={'X-Timestamp': timestamp}) + headers={'X-Timestamp': timestamp.internal}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 404) objfile = os.path.join( self.testdir, 'sda1', storage_directory(diskfile.get_data_dir(0), 'p', hash_path('a', 'c', 'o')), - timestamp + '.ts') + utils.Timestamp(timestamp).internal + '.ts') self.assert_(os.path.isfile(objfile)) self.assertEquals(2, calls_made[0]) self.assertEquals(len(os.listdir(os.path.dirname(objfile))), 1) @@ -1735,23 +1736,209 @@ class TestObjectController(unittest.TestCase): # already have been deleted, and it should not have performed a # container update because the timestamp is older, or created a # tombstone file with this timestamp. - timestamp = normalize_timestamp(float(timestamp) - 1) + timestamp = utils.Timestamp(start + 0.00001) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'DELETE'}, - headers={'X-Timestamp': timestamp}) + headers={'X-Timestamp': timestamp.internal}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 404) objfile = os.path.join( self.testdir, 'sda1', storage_directory(diskfile.get_data_dir(0), 'p', hash_path('a', 'c', 'o')), - timestamp + '.ts') + utils.Timestamp(timestamp).internal + '.ts') self.assertFalse(os.path.isfile(objfile)) self.assertEquals(2, calls_made[0]) self.assertEquals(len(os.listdir(os.path.dirname(objfile))), 1) finally: self.object_controller.container_update = orig_cu + def test_object_update_with_offset(self): + ts = (utils.Timestamp(t).internal for t in + itertools.count(int(time()))) + container_updates = [] + + def capture_updates(ip, port, method, path, headers, *args, **kwargs): + container_updates.append((ip, port, method, path, headers)) + # create a new object + create_timestamp = ts.next() + req = Request.blank('/sda1/p/a/c/o', method='PUT', body='test1', + headers={'X-Timestamp': create_timestamp, + 'X-Container-Host': '10.0.0.1:8080', + 'X-Container-Device': 'sda1', + 'X-Container-Partition': 'p', + 'Content-Type': 'text/plain'}) + with mocked_http_conn( + 200, give_connect=capture_updates) as fake_conn: + resp = req.get_response(self.object_controller) + self.assertRaises(StopIteration, fake_conn.code_iter.next) + self.assertEqual(resp.status_int, 201) + self.assertEquals(1, len(container_updates)) + for update in container_updates: + ip, port, method, path, headers = update + self.assertEqual(ip, '10.0.0.1') + self.assertEqual(port, '8080') + self.assertEqual(method, 'PUT') + self.assertEqual(path, '/sda1/p/a/c/o') + expected = { + 'X-Size': len('test1'), + 'X-Etag': md5('test1').hexdigest(), + 'X-Content-Type': 'text/plain', + 'X-Timestamp': create_timestamp, + } + for key, value in expected.items(): + self.assertEqual(headers[key], str(value)) + container_updates = [] # reset + # read back object + req = Request.blank('/sda1/p/a/c/o', method='GET') + resp = req.get_response(self.object_controller) + self.assertEqual(resp.status_int, 200) + self.assertEqual(resp.headers['X-Timestamp'], + utils.Timestamp(create_timestamp).normal) + self.assertEqual(resp.headers['X-Backend-Timestamp'], + create_timestamp) + self.assertEqual(resp.body, 'test1') + # send an update with an offset + offset_timestamp = utils.Timestamp( + create_timestamp, offset=1).internal + req = Request.blank('/sda1/p/a/c/o', method='PUT', body='test2', + headers={'X-Timestamp': offset_timestamp, + 'X-Container-Host': '10.0.0.1:8080', + 'X-Container-Device': 'sda1', + 'X-Container-Partition': 'p', + 'Content-Type': 'text/html'}) + with mocked_http_conn( + 200, give_connect=capture_updates) as fake_conn: + resp = req.get_response(self.object_controller) + self.assertRaises(StopIteration, fake_conn.code_iter.next) + self.assertEqual(resp.status_int, 201) + self.assertEquals(1, len(container_updates)) + for update in container_updates: + ip, port, method, path, headers = update + self.assertEqual(ip, '10.0.0.1') + self.assertEqual(port, '8080') + self.assertEqual(method, 'PUT') + self.assertEqual(path, '/sda1/p/a/c/o') + expected = { + 'X-Size': len('test2'), + 'X-Etag': md5('test2').hexdigest(), + 'X-Content-Type': 'text/html', + 'X-Timestamp': offset_timestamp, + } + for key, value in expected.items(): + self.assertEqual(headers[key], str(value)) + container_updates = [] # reset + # read back new offset + req = Request.blank('/sda1/p/a/c/o', method='GET') + resp = req.get_response(self.object_controller) + self.assertEqual(resp.status_int, 200) + self.assertEqual(resp.headers['X-Timestamp'], + utils.Timestamp(offset_timestamp).normal) + self.assertEqual(resp.headers['X-Backend-Timestamp'], + offset_timestamp) + self.assertEqual(resp.body, 'test2') + # now overwrite with a newer time + overwrite_timestamp = ts.next() + req = Request.blank('/sda1/p/a/c/o', method='PUT', body='test3', + headers={'X-Timestamp': overwrite_timestamp, + 'X-Container-Host': '10.0.0.1:8080', + 'X-Container-Device': 'sda1', + 'X-Container-Partition': 'p', + 'Content-Type': 'text/enriched'}) + with mocked_http_conn( + 200, give_connect=capture_updates) as fake_conn: + resp = req.get_response(self.object_controller) + self.assertRaises(StopIteration, fake_conn.code_iter.next) + self.assertEqual(resp.status_int, 201) + self.assertEquals(1, len(container_updates)) + for update in container_updates: + ip, port, method, path, headers = update + self.assertEqual(ip, '10.0.0.1') + self.assertEqual(port, '8080') + self.assertEqual(method, 'PUT') + self.assertEqual(path, '/sda1/p/a/c/o') + expected = { + 'X-Size': len('test3'), + 'X-Etag': md5('test3').hexdigest(), + 'X-Content-Type': 'text/enriched', + 'X-Timestamp': overwrite_timestamp, + } + for key, value in expected.items(): + self.assertEqual(headers[key], str(value)) + container_updates = [] # reset + # read back overwrite + req = Request.blank('/sda1/p/a/c/o', method='GET') + resp = req.get_response(self.object_controller) + self.assertEqual(resp.status_int, 200) + self.assertEqual(resp.headers['X-Timestamp'], + utils.Timestamp(overwrite_timestamp).normal) + self.assertEqual(resp.headers['X-Backend-Timestamp'], + overwrite_timestamp) + self.assertEqual(resp.body, 'test3') + # delete with an offset + offset_delete = utils.Timestamp(overwrite_timestamp, + offset=1).internal + req = Request.blank('/sda1/p/a/c/o', method='DELETE', + headers={'X-Timestamp': offset_delete, + 'X-Container-Host': '10.0.0.1:8080', + 'X-Container-Device': 'sda1', + 'X-Container-Partition': 'p'}) + with mocked_http_conn( + 200, give_connect=capture_updates) as fake_conn: + resp = req.get_response(self.object_controller) + self.assertRaises(StopIteration, fake_conn.code_iter.next) + self.assertEqual(resp.status_int, 204) + self.assertEquals(1, len(container_updates)) + for update in container_updates: + ip, port, method, path, headers = update + self.assertEqual(ip, '10.0.0.1') + self.assertEqual(port, '8080') + self.assertEqual(method, 'DELETE') + self.assertEqual(path, '/sda1/p/a/c/o') + expected = { + 'X-Timestamp': offset_delete, + } + for key, value in expected.items(): + self.assertEqual(headers[key], str(value)) + container_updates = [] # reset + # read back offset delete + req = Request.blank('/sda1/p/a/c/o', method='GET') + resp = req.get_response(self.object_controller) + self.assertEqual(resp.status_int, 404) + self.assertEqual(resp.headers['X-Timestamp'], None) + self.assertEqual(resp.headers['X-Backend-Timestamp'], offset_delete) + # and one more delete with a newer timestamp + delete_timestamp = ts.next() + req = Request.blank('/sda1/p/a/c/o', method='DELETE', + headers={'X-Timestamp': delete_timestamp, + 'X-Container-Host': '10.0.0.1:8080', + 'X-Container-Device': 'sda1', + 'X-Container-Partition': 'p'}) + with mocked_http_conn( + 200, give_connect=capture_updates) as fake_conn: + resp = req.get_response(self.object_controller) + self.assertRaises(StopIteration, fake_conn.code_iter.next) + self.assertEqual(resp.status_int, 404) + self.assertEquals(1, len(container_updates)) + for update in container_updates: + ip, port, method, path, headers = update + self.assertEqual(ip, '10.0.0.1') + self.assertEqual(port, '8080') + self.assertEqual(method, 'DELETE') + self.assertEqual(path, '/sda1/p/a/c/o') + expected = { + 'X-Timestamp': delete_timestamp, + } + for key, value in expected.items(): + self.assertEqual(headers[key], str(value)) + container_updates = [] # reset + # read back delete + req = Request.blank('/sda1/p/a/c/o', method='GET') + resp = req.get_response(self.object_controller) + self.assertEqual(resp.status_int, 404) + self.assertEqual(resp.headers['X-Timestamp'], None) + self.assertEqual(resp.headers['X-Backend-Timestamp'], delete_timestamp) + def test_call_bad_request(self): # Test swift.obj.server.ObjectController.__call__ inbuf = StringIO() @@ -2208,7 +2395,7 @@ class TestObjectController(unittest.TestCase): 'x-content-type': 'application/burrito', 'x-etag': 'd41d8cd98f00b204e9800998ecf8427e', 'x-size': '0', - 'x-timestamp': '12345', + 'x-timestamp': utils.Timestamp('12345').internal, POLICY_INDEX: '37', 'referer': 'PUT http://localhost/sda1/p/a/c/o', 'user-agent': 'obj-server %d' % os.getpid(), @@ -2227,7 +2414,7 @@ class TestObjectController(unittest.TestCase): 'x-content-type': 'text/plain', 'x-etag': 'd41d8cd98f00b204e9800998ecf8427e', 'x-size': '0', - 'x-timestamp': '12345', + 'x-timestamp': utils.Timestamp('12345').internal, 'referer': 'PUT http://localhost/sda1/p/a/c/o', 'user-agent': 'obj-server %d' % os.getpid(), POLICY_INDEX: 0, # system account storage policy is 0 @@ -2245,7 +2432,7 @@ class TestObjectController(unittest.TestCase): 'x-content-type': 'text/plain', 'x-etag': 'd41d8cd98f00b204e9800998ecf8427e', 'x-size': '0', - 'x-timestamp': '12345', + 'x-timestamp': utils.Timestamp('12345').internal, 'referer': 'PUT http://localhost/sda1/p/a/c/o', 'user-agent': 'obj-server %d' % os.getpid(), POLICY_INDEX: 0, # system account storage policy is 0 @@ -2314,7 +2501,7 @@ class TestObjectController(unittest.TestCase): 'x-content-type': 'application/burrito', 'x-etag': 'd41d8cd98f00b204e9800998ecf8427e', 'x-size': '0', - 'x-timestamp': '12345', + 'x-timestamp': utils.Timestamp('12345').internal, POLICY_INDEX: '26', 'referer': 'PUT http://localhost/sda1/p/a/c/o', 'user-agent': 'obj-server %d' % os.getpid(), @@ -2332,7 +2519,7 @@ class TestObjectController(unittest.TestCase): 'x-content-type': 'application/burrito', 'x-etag': 'd41d8cd98f00b204e9800998ecf8427e', 'x-size': '0', - 'x-timestamp': '12345', + 'x-timestamp': utils.Timestamp('12345').internal, POLICY_INDEX: '26', 'referer': 'PUT http://localhost/sda1/p/a/c/o', 'user-agent': 'obj-server %d' % os.getpid(), @@ -2340,7 +2527,7 @@ class TestObjectController(unittest.TestCase): def test_object_delete_at_aysnc_update(self): policy = random.choice(list(POLICIES)) - ts = (normalize_timestamp(t) for t in + ts = (utils.Timestamp(t) for t in itertools.count(int(time()))) container_updates = [] @@ -2348,8 +2535,9 @@ class TestObjectController(unittest.TestCase): def capture_updates(ip, port, method, path, headers, *args, **kwargs): container_updates.append((ip, port, method, path, headers)) - put_timestamp = ts.next() - delete_at_timestamp = utils.normalize_delete_at_timestamp(ts.next()) + put_timestamp = ts.next().internal + delete_at_timestamp = utils.normalize_delete_at_timestamp( + ts.next().normal) delete_at_container = ( int(delete_at_timestamp) / self.object_controller.expiring_objects_container_divisor * @@ -2441,7 +2629,8 @@ class TestObjectController(unittest.TestCase): self.assertEquals( pickle.load(open(os.path.join( self.testdir, 'sda1', async_dir, 'a83', - '06fbf0b514e5199dfc4e00f42eb5ea83-0000000001.00000'))), + '06fbf0b514e5199dfc4e00f42eb5ea83-%s' % + utils.Timestamp(1).internal))), {'headers': {'x-timestamp': '1', 'x-out': 'set', 'user-agent': 'obj-server %s' % os.getpid(), POLICY_INDEX: policy.idx}, @@ -2480,7 +2669,8 @@ class TestObjectController(unittest.TestCase): self.assertEquals( pickle.load(open(os.path.join( self.testdir, 'sda1', async_dir, 'a83', - '06fbf0b514e5199dfc4e00f42eb5ea83-0000000001.00000'))), + '06fbf0b514e5199dfc4e00f42eb5ea83-%s' % + utils.Timestamp(1).internal))), {'headers': {'x-timestamp': '1', 'x-out': str(status), 'user-agent': 'obj-server %s' % os.getpid(), POLICY_INDEX: policy.idx}, @@ -2552,7 +2742,8 @@ class TestObjectController(unittest.TestCase): self.assertTrue( os.path.exists(os.path.join( self.testdir, 'sda1', async_dir, 'a83', - '06fbf0b514e5199dfc4e00f42eb5ea83-0000000001.00000'))) + '06fbf0b514e5199dfc4e00f42eb5ea83-%s' % + utils.Timestamp(1).internal))) finally: object_server.http_connect = orig_http_connect utils.HASH_PATH_PREFIX = _prefix @@ -2607,7 +2798,7 @@ class TestObjectController(unittest.TestCase): 'x-size': '0', 'x-etag': 'd41d8cd98f00b204e9800998ecf8427e', 'x-content-type': 'text/plain', - 'x-timestamp': '1', + 'x-timestamp': utils.Timestamp(1).internal, POLICY_INDEX: '0', # default when not given 'x-trans-id': '123', 'referer': 'PUT http://localhost/sda1/0/a/c/o'})) @@ -2639,14 +2830,14 @@ class TestObjectController(unittest.TestCase): self.assertEqual(account, 'a') self.assertEqual(container, 'c') self.assertEqual(obj, 'o') - self.assertEqual(timestamp, '1') + self.assertEqual(timestamp, utils.Timestamp(1).internal) self.assertEqual(policy_index, 0) self.assertEqual(data, { 'headers': HeaderKeyDict({ 'X-Size': '0', 'User-Agent': 'obj-server %s' % os.getpid(), 'X-Content-Type': 'text/plain', - 'X-Timestamp': '1', + 'X-Timestamp': utils.Timestamp(1).internal, 'X-Trans-Id': '123', 'Referer': 'PUT http://localhost/sda1/0/a/c/o', 'X-Backend-Storage-Policy-Index': '0', @@ -2710,7 +2901,7 @@ class TestObjectController(unittest.TestCase): '0000000002-a/c/o', None, None, None, HeaderKeyDict({ POLICY_INDEX: 0, - 'x-timestamp': '1', + 'x-timestamp': utils.Timestamp('1').internal, 'x-trans-id': '123', 'referer': 'PUT http://localhost/v1/a/c/o'}), 'sda1', 0]) @@ -2737,7 +2928,7 @@ class TestObjectController(unittest.TestCase): None, None, None, HeaderKeyDict({ POLICY_INDEX: 0, - 'x-timestamp': '1', + 'x-timestamp': utils.Timestamp('1').internal, 'x-trans-id': '1234', 'referer': 'PUT http://localhost/v1/a/c/o'}), 'sda1', 0]) @@ -2764,7 +2955,7 @@ class TestObjectController(unittest.TestCase): None, None, None, HeaderKeyDict({ POLICY_INDEX: 0, - 'x-timestamp': '1', + 'x-timestamp': utils.Timestamp('1').internal, 'x-trans-id': '1234', 'referer': 'PUT http://localhost/v1/a/c/o'}), 'sda1', 0]) @@ -2799,7 +2990,7 @@ class TestObjectController(unittest.TestCase): 'x-size': '0', 'x-etag': 'd41d8cd98f00b204e9800998ecf8427e', 'x-content-type': 'text/plain', - 'x-timestamp': '1', + 'x-timestamp': utils.Timestamp('1').internal, 'x-trans-id': '1234', 'referer': 'PUT http://localhost/v1/a/c/o'}), 'sda1', 0]) @@ -2850,7 +3041,8 @@ class TestObjectController(unittest.TestCase): '0000000002-a/c/o', None, None, None, HeaderKeyDict({ POLICY_INDEX: 0, - 'x-timestamp': '1', 'x-trans-id': '1234', + 'x-timestamp': utils.Timestamp('1').internal, + 'x-trans-id': '1234', 'referer': 'DELETE http://localhost/v1/a/c/o'}), 'sda1', 0]) @@ -3061,7 +3253,7 @@ class TestObjectController(unittest.TestCase): resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 404) self.assertEquals(resp.headers['X-Backend-Timestamp'], - put_timestamp) + utils.Timestamp(put_timestamp)) finally: object_server.time.time = orig_time @@ -3131,7 +3323,7 @@ class TestObjectController(unittest.TestCase): resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 404) self.assertEquals(resp.headers['X-Backend-Timestamp'], - put_timestamp) + utils.Timestamp(put_timestamp)) finally: object_server.time.time = orig_time @@ -3251,7 +3443,7 @@ class TestObjectController(unittest.TestCase): self.testdir, 'sda1', storage_directory(diskfile.get_data_dir(0), 'p', hash_path('a', 'c', 'o')), - test_timestamp + '.data') + utils.Timestamp(test_timestamp).internal + '.data') self.assert_(os.path.isfile(objfile)) # move time past expirery