x/stacktach
Code Issues Proposed changes

Added image exists, usage and delete models for glance

Browse Source
This commit is contained in:
Anuj Mathur 2013-07-01 15:16:10 +05:30 committed by Manali Latkar
parent 3b1e086a26
commit ec5bbf31a2
8 changed files with 378 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
stacktach/db.py
View File

@ -103,4 +103,33 @@ def create_generic_rawdata(**kwargs):
rawdata = models.GenericRawData(**kwargs)
rawdata.save()
return rawdata
return rawdata
def create_image_usage(**kwargs):
usage = models.ImageUsage(**kwargs)
usage.save()
return usage
def create_image_delete(**kwargs):
delete = models.ImageDeletes(**kwargs)
delete.save()
return delete
def create_image_exists(**kwargs):
exists = models.ImageExists(**kwargs)
exists.save()
return exists
def get_image_delete(**kwargs):
return _safe_get(models.ImageDeletes, **kwargs)
def get_image_usage(**kwargs):
return _safe_get(models.ImageUsage, **kwargs)

70
stacktach/models.py
View File

@ -1,18 +1,3 @@
# Copyright 2012 - Dark Secret Software Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from django import forms
from django.db import models
@ -316,5 +301,60 @@ class GlanceRawData(models.Model):
def get_name():
return GlanceRawData.__name__
class ImageUsage(models.Model):
uuid = models.CharField(max_length=50, db_index=True)
created_at = models.DecimalField(max_digits=20,
decimal_places=6, db_index=True)
owner = models.CharField(max_length=50, db_index=True)
size = models.BigIntegerField(max_length=20)
last_raw = models.ForeignKey(GlanceRawData)
class ImageDeletes(models.Model):
uuid = models.CharField(max_length=50, db_index=True)
created_at = models.DecimalField(max_digits=20,
decimal_places=6, db_index=True)
deleted_at = models.DecimalField(max_digits=20,
decimal_places=6, db_index=True)
owner = models.CharField(max_length=50, db_index=True)
size = models.BigIntegerField(max_length=20)
raw = models.ForeignKey(GlanceRawData)
class ImageExists(models.Model):
PENDING = 'pending'
VERIFYING = 'verifying'
VERIFIED = 'verified'
FAILED = 'failed'
STATUS_CHOICES = [
(PENDING, 'Pending Verification'),
(VERIFYING, 'Currently Being Verified'),
(VERIFIED, 'Passed Verification'),
(FAILED, 'Failed Verification'),
]
uuid = models.CharField(max_length=50, db_index=True)
created_at = models.DecimalField(max_digits=20,
decimal_places=6, db_index=True)
deleted_at = models.DecimalField(max_digits=20,
decimal_places=6, db_index=True)
audit_period_beginning = models.DecimalField(max_digits=20,
decimal_places=6,
db_index=True)
audit_period_ending = models.DecimalField(max_digits=20,
decimal_places=6, db_index=True)
status = models.CharField(max_length=50, db_index=True,
choices=STATUS_CHOICES,
default=PENDING)
fail_reason = models.CharField(max_length=300, db_index=True, null=True)
raw = models.ForeignKey(GlanceRawData, related_name='+')
usage = models.ForeignKey(ImageUsage, related_name='+')
delete = models.ForeignKey(ImageDeletes, related_name='+')
send_status = models.IntegerField(default=0, db_index=True)
owner = models.CharField(max_length=255, db_index=True)
size = models.BigIntegerField(max_length=20)
def get_model_fields(model):
return model._meta.fields

48
stacktach/tests.py
View File

@ -22,7 +22,7 @@ from datetime import datetime
from django.test import TransactionTestCase
import db
from stacktach.datetime_to_decimal import dt_to_decimal
from stacktach.models import RawDataImageMeta
from stacktach.models import RawDataImageMeta, ImageUsage, ImageDeletes
from stacktach.models import GenericRawData
from stacktach.models import GlanceRawData
from stacktach.models import RawData
@ -69,8 +69,8 @@ class RawDataImageMetaDbTestCase(TransactionTestCase):
self.assertEquals(raw_image_meta.rax_options, kwargs['rax_options'])
class GlanceRawDataTestCase(TransactionTestCase):
def test_create_rawdata_should_populate_glance_rawdata(self):
class GlanceTestCase(TransactionTestCase):
def _create_glance_rawdata(self):
deployment = db.get_or_create_deployment('deployment1')[0]
kwargs = {
'deployment': deployment,
@ -88,15 +88,53 @@ class GlanceRawDataTestCase(TransactionTestCase):
'uuid': '1234-5678-0912-3456',
'status': 'active',
}
db.create_glance_rawdata(**kwargs)
rawdata = GlanceRawData.objects.all().order_by('-id')[0]
rawdata = GlanceRawData.objects.all()[0]
return kwargs, rawdata
def test_create_rawdata_should_populate_glance_rawdata(self):
kwargs, rawdata = self._create_glance_rawdata()
for field in get_model_fields(GlanceRawData):
if field.name != 'id':
self.assertEquals(getattr(rawdata, field.name),
kwargs[field.name])
def test_create_glance_usage_should_populate_image_usage(self):
_, rawdata = self._create_glance_rawdata()
kwargs = {
'uuid': '1',
'created_at': dt_to_decimal(datetime.utcnow()),
'owner': '1234567',
'size': 12345,
'last_raw': rawdata
}
db.create_image_usage(**kwargs)
usage = ImageUsage.objects.all()[0]
for field in get_model_fields(ImageUsage):
if field.name != 'id':
self.assertEquals(getattr(usage, field.name),
kwargs[field.name])
def test_create_image_delete_should_populate_image_delete(self):
_, rawdata = self._create_glance_rawdata()
kwargs = {
'uuid': '1',
'created_at': dt_to_decimal(datetime.utcnow()),
'owner': 'owner',
'size': 12345,
'raw': rawdata,
'deleted_at': dt_to_decimal(datetime.utcnow())
}
db.create_image_delete(**kwargs)
image_delete = ImageDeletes.objects.all()[0]
for field in get_model_fields(ImageDeletes):
if field.name != 'id':
self.assertEquals(getattr(image_delete, field.name),
kwargs[field.name])
class GenericRawDataTestCase(TransactionTestCase):
def test_create_generic_rawdata_should_populate_generic_rawdata(self):

77
stacktach/views.py
View File

@ -295,6 +295,55 @@ def _process_exists(raw, body):
stacklog.warn("Ignoring exists without launched_at. RawData(%s)" % raw.id)
def _process_glance_usage(raw, notification):
values = {
'uuid': notification.uuid,
'created_at': notification.created_at,
'owner': notification.owner,
'size': notification.size,
'last_raw': raw
}
STACKDB.create_image_usage(**values)
def _process_glance_delete(raw, notification):
values = {
'uuid': notification.uuid,
'created_at': notification.created_at,
'owner': notification.owner,
'size': notification.size,
'raw': raw,
'deleted_at': notification.deleted_at
}
STACKDB.create_image_delete(**values)
def _process_glance_exists(raw, notification):
if notification.created_at:
values = {
'uuid': notification.uuid,
'audit_period_beginning': notification.audit_period_beginning,
'audit_period_ending': notification.audit_period_ending,
'owner': notification.owner,
'size': notification.size,
'raw': raw,
}
created_at_range = (notification.created_at, notification.created_at+1)
usage = STACKDB.get_image_usage(uuid=notification.uuid,
created_at__range=created_at_range)
values['usage'] = usage
values['created_at'] = notification.created_at
if notification.deleted_at:
delete = STACKDB.get_image_delete(uuid=notification.uuid,
created_at__range=created_at_range)
values['delete'] = delete
values['deleted_at'] = notification.deleted_at
STACKDB.create_image_exists(**values)
else:
stacklog.warn("Ignoring exists without created_at. GlanceRawData(%s)"
% raw.id)
USAGE_PROCESS_MAPPING = {
INSTANCE_EVENT['create_start']: _process_usage_for_new_launch,
INSTANCE_EVENT['rebuild_start']: _process_usage_for_new_launch,
@ -306,8 +355,14 @@ USAGE_PROCESS_MAPPING = {
INSTANCE_EVENT['resize_finish_end']: _process_usage_for_updates,
INSTANCE_EVENT['resize_revert_end']: _process_usage_for_updates,
INSTANCE_EVENT['delete_end']: _process_delete,
INSTANCE_EVENT['exists']: _process_exists,
}
INSTANCE_EVENT['exists']: _process_exists
}
GLANCE_USAGE_PROCESS_MAPPING = {
'image.activate': _process_glance_usage,
'image.delete': _process_glance_delete,
'image.exists': _process_glance_exists
}
def aggregate_usage(raw, body):
@ -318,6 +373,11 @@ def aggregate_usage(raw, body):
USAGE_PROCESS_MAPPING[raw.event](raw, body)
def aggregate_glance_usage(raw, body):
if raw.event in GLANCE_USAGE_PROCESS_MAPPING.keys():
GLANCE_USAGE_PROCESS_MAPPING[raw.event](raw, body)
def process_raw_data(deployment, args, json_args, exchange):
"""This is called directly by the worker to add the event to the db."""
db.reset_queries()
@ -325,19 +385,20 @@ def process_raw_data(deployment, args, json_args, exchange):
routing_key, body = args
notif = notification.notification_factory(body, deployment, routing_key,
json_args, exchange)
return notif.save()
raw = notif.save()
return raw, notif
def post_process_rawdata(raw, body):
def post_process_rawdata(raw, notification):
aggregate_lifecycle(raw)
aggregate_usage(raw, body)
aggregate_usage(raw, notification)
def post_process_glancerawdata(raw, body):
pass
def post_process_glancerawdata(raw, notification):
aggregate_glance_usage(raw, notification)
def post_process_genericrawdata(raw, body):
def post_process_genericrawdata(raw, body, notification):
pass

169
tests/unit/test_stacktach.py
View File

@ -26,6 +26,7 @@ import mox
import utils
from utils import INSTANCE_ID_1
from utils import DECIMAL_DUMMY_TIME
from utils import OS_VERSION_1
from utils import OS_ARCH_1
from utils import OS_DISTRO_1
@ -36,6 +37,7 @@ from utils import TENANT_ID_1
from utils import INSTANCE_TYPE_ID_1
from utils import DUMMY_TIME
from utils import INSTANCE_TYPE_ID_2
from utils import IMAGE_UUID_1
from stacktach import stacklog
from stacktach import notification
from stacktach import views
@ -75,7 +77,7 @@ class StacktachRawParsingTestCase(unittest.TestCase):
self.assertEquals(
views.process_raw_data(deployment, args, json_args, exchange),
mock_record)
(mock_record, mock_notification))
self.mox.VerifyAll()
def test_process_raw_data_old_timestamp(self):
@ -710,3 +712,168 @@ class StacktachUsageParsingTestCase(unittest.TestCase):
views._process_exists(raw, notif[1])
self.mox.VerifyAll()
class StacktachImageUsageParsingTestCase(unittest.TestCase):
def setUp(self):
self.mox = mox.Mox()
views.STACKDB = self.mox.CreateMockAnything()
def tearDown(self):
self.mox.UnsetStubs()
def test_process_image_usage_for_new_launch(self):
raw = self.mox.CreateMockAnything()
values = {
'created_at': str(DUMMY_TIME),
'owner': TENANT_ID_1,
'uuid': IMAGE_UUID_1,
'size': 1234,
'last_raw': raw
}
notification = self.mox.CreateMockAnything()
notification.created_at = values['created_at']
notification.owner = values['owner']
notification.uuid = values['uuid']
notification.size = values['size']
notification.raw = values['last_raw']
views.STACKDB.create_image_usage(**values)
self.mox.ReplayAll()
views._process_glance_usage(raw, notification)
self.mox.VerifyAll()
def test_process_image_deletes(self):
raw = self.mox.CreateMockAnything()
values = {
'uuid': IMAGE_UUID_1,
'created_at': str(DUMMY_TIME),
'deleted_at': str(DUMMY_TIME),
'owner': TENANT_ID_1,
'size': 1234,
'raw': raw
}
notification = self.mox.CreateMockAnything()
notification.created_at = values['created_at']
notification.deleted_at = values['deleted_at']
notification.owner = values['owner']
notification.uuid = values['uuid']
notification.size = values['size']
notification.raw = values['raw']
views.STACKDB.create_image_delete(**values)
self.mox.ReplayAll()
views._process_glance_delete(raw, notification)
self.mox.VerifyAll()
def test_process_image_exists(self):
raw = self.mox.CreateMockAnything()
values = {
'uuid': IMAGE_UUID_1,
'created_at': DECIMAL_DUMMY_TIME,
'deleted_at': DECIMAL_DUMMY_TIME,
'audit_period_beginning': DECIMAL_DUMMY_TIME,
'audit_period_ending': DECIMAL_DUMMY_TIME,
'owner': TENANT_ID_1,
'size': 1234,
'raw': raw,
'usage': None,
'delete': None
}
notification = self.mox.CreateMockAnything()
notification.created_at = values['created_at']
notification.deleted_at = values['deleted_at']
notification.owner = values['owner']
notification.uuid = values['uuid']
notification.size = values['size']
notification.raw = values['raw']
notification.audit_period_beginning = values['audit_period_beginning']
notification.audit_period_ending = values['audit_period_ending']
created_at_range = (DECIMAL_DUMMY_TIME, DECIMAL_DUMMY_TIME+1)
views.STACKDB.get_image_usage(created_at__range=created_at_range,
uuid=notification.uuid).AndReturn(None)
views.STACKDB.get_image_delete(created_at__range=created_at_range,
uuid=notification.uuid).AndReturn(None)
views.STACKDB.create_image_exists(**values)
self.mox.ReplayAll()
views._process_glance_exists(raw, notification)
self.mox.VerifyAll()
def test_process_image_exists_with_usage_not_none(self):
raw = self.mox.CreateMockAnything()
usage = self.mox.CreateMockAnything()
values = {
'uuid': IMAGE_UUID_1,
'created_at': DECIMAL_DUMMY_TIME,
'deleted_at': DECIMAL_DUMMY_TIME,
'audit_period_beginning': DECIMAL_DUMMY_TIME,
'audit_period_ending': DECIMAL_DUMMY_TIME,
'owner': TENANT_ID_1,
'size': 1234,
'raw': raw,
'usage': usage,
'delete': None
}
notification = self.mox.CreateMockAnything()
notification.created_at = values['created_at']
notification.deleted_at = values['deleted_at']
notification.owner = values['owner']
notification.uuid = values['uuid']
notification.size = values['size']
notification.raw = values['raw']
notification.audit_period_beginning = values['audit_period_beginning']
notification.audit_period_ending = values['audit_period_ending']
created_at_range = (DECIMAL_DUMMY_TIME, DECIMAL_DUMMY_TIME+1)
views.STACKDB.get_image_usage(created_at__range=created_at_range,
uuid=notification.uuid).AndReturn(usage)
views.STACKDB.get_image_delete(created_at__range=created_at_range,
uuid=notification.uuid).AndReturn(None)
views.STACKDB.create_image_exists(**values)
self.mox.ReplayAll()
views._process_glance_exists(raw, notification)
self.mox.VerifyAll()
def test_process_image_exists_with_delete_not_none(self):
raw = self.mox.CreateMockAnything()
delete = self.mox.CreateMockAnything()
values = {
'uuid': IMAGE_UUID_1,
'created_at': DECIMAL_DUMMY_TIME,
'deleted_at': DECIMAL_DUMMY_TIME,
'audit_period_beginning': DECIMAL_DUMMY_TIME,
'audit_period_ending': DECIMAL_DUMMY_TIME,
'owner': TENANT_ID_1,
'size': 1234,
'raw': raw,
'usage': None,
'delete': delete
}
notification = self.mox.CreateMockAnything()
notification.created_at = values['created_at']
notification.deleted_at = values['deleted_at']
notification.owner = values['owner']
notification.uuid = values['uuid']
notification.size = values['size']
notification.raw = values['raw']
notification.audit_period_beginning = values['audit_period_beginning']
notification.audit_period_ending = values['audit_period_ending']
created_at_range = (DECIMAL_DUMMY_TIME, DECIMAL_DUMMY_TIME+1)
views.STACKDB.get_image_usage(created_at__range=created_at_range,
uuid=notification.uuid).AndReturn(None)
views.STACKDB.get_image_delete(created_at__range=created_at_range,
uuid=notification.uuid).AndReturn(delete)
views.STACKDB.create_image_exists(**values)
self.mox.ReplayAll()
views._process_glance_exists(raw, notification)
self.mox.VerifyAll()
# def test_process_image_exists_should_not_populate_delete_and_deleted_at_when_deleted_at_is_absent(self):

30
tests/unit/test_worker.py
View File

@ -128,8 +128,9 @@ class ConsumerTestCase(unittest.TestCase):
body_dict = {u'key': u'value'}
message.body = json.dumps(body_dict)
mock_notification = self.mox.CreateMockAnything()
mock_post_process_method = self.mox.CreateMockAnything()
mock_post_process_method(raw, body_dict)
mock_post_process_method(raw, mock_notification)
old_handler = worker.POST_PROCESS_METHODS
worker.POST_PROCESS_METHODS["RawData"] = mock_post_process_method
@ -137,7 +138,7 @@ class ConsumerTestCase(unittest.TestCase):
use_mock_anything=True)
args = (routing_key, body_dict)
views.process_raw_data(deployment, args, json.dumps(args), exchange) \
.AndReturn(raw)
.AndReturn((raw, mock_notification))
message.ack()
self.mox.StubOutWithMock(consumer, '_check_memory',
@ -149,31 +150,6 @@ class ConsumerTestCase(unittest.TestCase):
self.mox.VerifyAll()
worker.POST_PROCESS_METHODS["RawData"] = old_handler
def test_process_no_raw_dont_ack(self):
deployment = self.mox.CreateMockAnything()
raw = self.mox.CreateMockAnything()
message = self.mox.CreateMockAnything()
exchange = 'nova'
consumer = worker.Consumer('test', None, deployment, True, {},
exchange, ["monitor.info", "monitor.error"])
routing_key = 'monitor.info'
message.delivery_info = {'routing_key': routing_key}
body_dict = {u'key': u'value'}
message.body = json.dumps(body_dict)
self.mox.StubOutWithMock(views, 'process_raw_data',
use_mock_anything=True)
args = (routing_key, body_dict)
views.process_raw_data(deployment, args, json.dumps(args), exchange) \
.AndReturn(None)
self.mox.StubOutWithMock(consumer, '_check_memory',
use_mock_anything=True)
consumer._check_memory()
self.mox.ReplayAll()
consumer._process(message)
self.assertEqual(consumer.processed, 0)
self.mox.VerifyAll()
def test_run(self):
config = {
'name': 'east_coast.prod.global',

3
tests/unit/utils.py
View File

@ -25,6 +25,8 @@ TENANT_ID_2 = 'testtenantid2'
from stacktach import datetime_to_decimal as dt
IMAGE_UUID_1 = "1"
INSTANCE_ID_1 = "08f685d9-6352-4dbc-8271-96cc54bf14cd"
INSTANCE_ID_2 = "515adf96-41d3-b86d-5467-e584edc61dab"
@ -32,6 +34,7 @@ INSTANCE_TYPE_ID_1 = "12345"
INSTANCE_TYPE_ID_2 = '54321'
DUMMY_TIME = datetime.datetime.utcnow()
DECIMAL_DUMMY_TIME = dt.dt_to_decimal(DUMMY_TIME)
MESSAGE_ID_1 = "7f28f81b-29a2-43f2-9ba1-ccb3e53ab6c8"
MESSAGE_ID_2 = "4d596126-0f04-4329-865f-7b9a7bd69bcf"

12
worker/worker.py
View File

@ -34,7 +34,9 @@ except ImportError:
from pympler.process import ProcessMemoryInfo
from django import db as django_db
from stacktach import db
from stacktach import notification
from stacktach import stacklog
from stacktach import views
@ -84,12 +86,12 @@ class Consumer(kombu.mixins.ConsumerMixin):
args = (routing_key, json.loads(body))
asJson = json.dumps(args)
# save raw and ack the message
raw = views.process_raw_data(self.deployment, args, asJson, self.exchange)
raw, notif = views.process_raw_data(
self.deployment, args, asJson, self.exchange)
if raw:
self.processed += 1
message.ack()
POST_PROCESS_METHODS[raw.get_name()](raw, args[1])
self.processed += 1
message.ack()
POST_PROCESS_METHODS[raw.get_name()](raw, notif)
self._check_memory()