openstack/zaqar
Code Issues Proposed changes

feat(mongo): use UNIX timestamp instead of datetime

Browse Source 
Closes-Bug: #1222928

Change-Id: Ia352eb0643bfbc20b10cb965617c2e503e4966f5
This commit is contained in:
Zhihao Yuan 2013-09-09 16:56:17 -04:00
parent 496e6ddd70
commit 9bb249ebeb
4 changed files with 37 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
marconi/storage/mongodb/claims.py
View File

@ -21,8 +21,6 @@ Field Mappings:
letter of their long name. letter of their long name.
""" """
import datetime
from bson import objectid from bson import objectid
from marconi.common import config from marconi.common import config
@ -64,7 +62,7 @@ class ClaimController(storage.ClaimBase):
msg_ctrl = self.driver.message_controller msg_ctrl = self.driver.message_controller
# Base query, always check expire time # Base query, always check expire time
now = timeutils.utcnow() now = timeutils.utcnow_ts()
cid = utils.to_oid(claim_id) cid = utils.to_oid(claim_id)
if cid is None: if cid is None:
raise exceptions.ClaimDoesNotExist(queue, project, claim_id) raise exceptions.ClaimDoesNotExist(queue, project, claim_id)
@ -87,8 +85,8 @@ class ClaimController(storage.ClaimBase):
project=project)) project=project))
claim = next(msgs) claim = next(msgs)
update_time = claim['e'] - datetime.timedelta(seconds=claim['t']) update_time = claim['e'] - claim['t']
age = timeutils.delta_seconds(update_time, now) age = update_time - now
claim = { claim = {
'age': int(age), 'age': int(age),
@ -128,12 +126,10 @@ class ClaimController(storage.ClaimBase):
grace = metadata['grace'] grace = metadata['grace']
oid = objectid.ObjectId() oid = objectid.ObjectId()
now = timeutils.utcnow() now = timeutils.utcnow_ts()
ttl_delta = datetime.timedelta(seconds=ttl) claim_expires = now + ttl
claim_expires = now + ttl_delta
grace_delta = datetime.timedelta(seconds=grace) message_expires = claim_expires + grace
message_expires = claim_expires + grace_delta
message_ttl = ttl + grace message_ttl = ttl + grace
meta = { meta = {
@ -153,7 +149,7 @@ class ClaimController(storage.ClaimBase):
if len(ids) == 0: if len(ids) == 0:
return (None, messages) return (None, messages)
now = timeutils.utcnow() now = timeutils.utcnow_ts()
# Set claim field for messages in ids # Set claim field for messages in ids
updated = msg_ctrl._col.update({'_id': {'$in': ids}, updated = msg_ctrl._col.update({'_id': {'$in': ids},
@ -196,11 +192,9 @@ class ClaimController(storage.ClaimBase):
if cid is None: if cid is None:
raise exceptions.ClaimDoesNotExist(claim_id, queue, project) raise exceptions.ClaimDoesNotExist(claim_id, queue, project)
now = timeutils.utcnow() now = timeutils.utcnow_ts()
ttl = int(metadata.get('ttl', 60)) ttl = int(metadata.get('ttl', 60))
ttl_delta = datetime.timedelta(seconds=ttl) expires = now + ttl
expires = now + ttl_delta
msg_ctrl = self.driver.message_controller msg_ctrl = self.driver.message_controller
claimed = msg_ctrl.claimed(queue, cid, expires=now, claimed = msg_ctrl.claimed(queue, cid, expires=now,

29
marconi/storage/mongodb/messages.py
View File

@ -21,7 +21,6 @@ Field Mappings:
letter of their long name. letter of their long name.
""" """
import datetime
import time import time
import pymongo.errors import pymongo.errors
@ -208,7 +207,7 @@ class MessageController(storage.MessageBase):
'p': project, 'p': project,
'q': queue_name, 'q': queue_name,
'k': {'$ne': head['k']}, 'k': {'$ne': head['k']},
'e': {'$lte': timeutils.utcnow()}, 'e': {'$lte': timeutils.utcnow_ts()},
} }
self._col.remove(query, w=0) self._col.remove(query, w=0)
@ -261,7 +260,7 @@ class MessageController(storage.MessageBase):
raise ValueError(u'sort must be either 1 (ascending) ' raise ValueError(u'sort must be either 1 (ascending) '
u'or -1 (descending)') u'or -1 (descending)')
now = timeutils.utcnow() now = timeutils.utcnow_ts()
query = { query = {
# Messages must belong to this # Messages must belong to this
@ -312,12 +311,12 @@ class MessageController(storage.MessageBase):
'q': queue_name, 'q': queue_name,
# The messages can not be expired # The messages can not be expired
'e': {'$gt': timeutils.utcnow()}, 'e': {'$gt': timeutils.utcnow_ts()},
} }
if not include_claimed: if not include_claimed:
# Exclude messages that are claimed # Exclude messages that are claimed
query['c.e'] = {'$lte': timeutils.utcnow()} query['c.e'] = {'$lte': timeutils.utcnow_ts()}
return self._col.find(query).hint(COUNTING_INDEX_FIELDS).count() return self._col.find(query).hint(COUNTING_INDEX_FIELDS).count()
@ -362,7 +361,7 @@ class MessageController(storage.MessageBase):
'p': project, 'p': project,
'q': queue_name, 'q': queue_name,
'c.id': claim_id, 'c.id': claim_id,
'c.e': {'$gt': expires or timeutils.utcnow()}, 'c.e': {'$gt': expires or timeutils.utcnow_ts()},
} }
# NOTE(kgriffs): Claimed messages bust be queried from # NOTE(kgriffs): Claimed messages bust be queried from
@ -375,7 +374,7 @@ class MessageController(storage.MessageBase):
if limit is not None: if limit is not None:
msgs = msgs.limit(limit) msgs = msgs.limit(limit)
now = timeutils.utcnow() now = timeutils.utcnow_ts()
def denormalizer(msg): def denormalizer(msg):
doc = _basic_message(msg, now) doc = _basic_message(msg, now)
@ -395,7 +394,7 @@ class MessageController(storage.MessageBase):
# NOTE(cpp-cabrera): unclaim by setting the claim ID to None # NOTE(cpp-cabrera): unclaim by setting the claim ID to None
# and the claim expiration time to now # and the claim expiration time to now
now = timeutils.utcnow() now = timeutils.utcnow_ts()
self._col.update({'p': project, 'q': queue_name, 'c.id': cid}, self._col.update({'p': project, 'q': queue_name, 'c.id': cid},
{'$set': {'c': {'id': None, 'e': now}}}, {'$set': {'c': {'id': None, 'e': now}}},
upsert=False, multi=True) upsert=False, multi=True)
@ -443,7 +442,7 @@ class MessageController(storage.MessageBase):
marker_id = {} marker_id = {}
now = timeutils.utcnow() now = timeutils.utcnow_ts()
def denormalizer(msg): def denormalizer(msg):
marker_id['next'] = msg['k'] marker_id['next'] = msg['k']
@ -464,7 +463,7 @@ class MessageController(storage.MessageBase):
raise exceptions.MessageDoesNotExist(message_id, queue_name, raise exceptions.MessageDoesNotExist(message_id, queue_name,
project) project)
now = timeutils.utcnow() now = timeutils.utcnow_ts()
query = { query = {
'_id': mid, '_id': mid,
@ -487,7 +486,7 @@ class MessageController(storage.MessageBase):
if not message_ids: if not message_ids:
return iter([]) return iter([])
now = timeutils.utcnow() now = timeutils.utcnow_ts()
# Base query, always check expire time # Base query, always check expire time
query = { query = {
@ -508,7 +507,7 @@ class MessageController(storage.MessageBase):
@utils.raises_conn_error @utils.raises_conn_error
def post(self, queue_name, messages, client_uuid, project=None): def post(self, queue_name, messages, client_uuid, project=None):
now = timeutils.utcnow() now = timeutils.utcnow_ts()
if not self._queue_controller.exists(queue_name, project): if not self._queue_controller.exists(queue_name, project):
raise exceptions.QueueDoesNotExist(queue_name, project) raise exceptions.QueueDoesNotExist(queue_name, project)
@ -521,7 +520,7 @@ class MessageController(storage.MessageBase):
't': message['ttl'], 't': message['ttl'],
'q': queue_name, 'q': queue_name,
'p': project, 'p': project,
'e': now + datetime.timedelta(seconds=message['ttl']), 'e': now + message['ttl'],
'u': client_uuid, 'u': client_uuid,
'c': {'id': None, 'e': now}, 'c': {'id': None, 'e': now},
'b': message['body'] if 'body' in message else {}, 'b': message['body'] if 'body' in message else {},
@ -647,7 +646,7 @@ class MessageController(storage.MessageBase):
if cid is None: if cid is None:
return return
now = timeutils.utcnow() now = timeutils.utcnow_ts()
query['e'] = {'$gt': now} query['e'] = {'$gt': now}
message = self._col.find_one(query) message = self._col.find_one(query)
@ -681,7 +680,7 @@ class MessageController(storage.MessageBase):
def _basic_message(msg, now): def _basic_message(msg, now):
oid = msg['_id'] oid = msg['_id']
age = timeutils.delta_seconds(utils.oid_utc(oid), now) age = utils.oid_ts(oid) - now
return { return {
'id': str(oid), 'id': str(oid),

2
marconi/storage/mongodb/queues.py
View File

@ -164,7 +164,7 @@ class QueueController(storage.QueueBase):
except exceptions.QueueIsEmpty: except exceptions.QueueIsEmpty:
pass pass
else: else:
now = timeutils.utcnow() now = timeutils.utcnow_ts()
message_stats['oldest'] = utils.stat_message(oldest, now) message_stats['oldest'] = utils.stat_message(oldest, now)
message_stats['newest'] = utils.stat_message(newest, now) message_stats['newest'] = utils.stat_message(newest, now)

20
marconi/storage/mongodb/utils.py
View File

@ -14,12 +14,14 @@
# limitations under the License. # limitations under the License.
import collections import collections
import datetime
import functools import functools
import random import random
import re import re
from bson import errors as berrors from bson import errors as berrors
from bson import objectid from bson import objectid
from bson import tz_util
from pymongo import errors from pymongo import errors
from marconi.common import exceptions from marconi.common import exceptions
@ -30,6 +32,10 @@ from marconi.storage import exceptions as storage_exceptions
DUP_MARKER_REGEX = re.compile(r'\$queue_marker.*?:\s(\d+)') DUP_MARKER_REGEX = re.compile(r'\$queue_marker.*?:\s(\d+)')
# BSON ObjectId gives TZ-aware datetime, so we generate a
# TZ-aware UNIX epoch for convenience.
EPOCH = datetime.datetime.utcfromtimestamp(0).replace(tzinfo=tz_util.utc)
LOG = logging.getLogger(__name__) LOG = logging.getLogger(__name__)
@ -118,7 +124,7 @@ def to_oid(obj):
"""Creates a new ObjectId based on the input. """Creates a new ObjectId based on the input.
Returns None when TypeError or berrors.InvalidId Returns None when TypeError or berrors.InvalidId
is raised by the ObjectID class. is raised by the ObjectId class.
:param obj: Anything that can be passed as an :param obj: Anything that can be passed as an
input to `objectid.ObjectId` input to `objectid.ObjectId`
@ -129,12 +135,12 @@ def to_oid(obj):
return None return None
def oid_utc(oid): def oid_ts(oid):
"""Converts an ObjectId to a non-tz-aware datetime. """Converts an ObjectId to a UNIX timestamp.
:raises: TypeError if oid isn't an ObjectId :raises: TypeError if oid isn't an ObjectId
""" """
try: try:
return timeutils.normalize_time(oid.generation_time) return timeutils.delta_seconds(EPOCH, oid.generation_time)
except AttributeError: except AttributeError:
raise TypeError(u'Expected ObjectId and got %s' % type(oid)) raise TypeError(u'Expected ObjectId and got %s' % type(oid))
@ -142,13 +148,13 @@ def oid_utc(oid):
def stat_message(message, now): def stat_message(message, now):
"""Creates a stat document from the given message, relative to now.""" """Creates a stat document from the given message, relative to now."""
oid = message['_id'] oid = message['_id']
created = oid_utc(oid) created = oid_ts(oid)
age = timeutils.delta_seconds(created, now) age = created - now
return { return {
'id': str(oid), 'id': str(oid),
'age': int(age), 'age': int(age),
'created': timeutils.isotime(created), 'created': timeutils.iso8601_from_timestamp(created),
} }