From ec5bbf31a23447ef538bfc4c5016b895fb46d351 Mon Sep 17 00:00:00 2001 From: Anuj Mathur Date: Mon, 1 Jul 2013 15:16:10 +0530 Subject: [PATCH] Added image exists, usage and delete models for glance --- stacktach/db.py | 31 ++++++- stacktach/models.py | 70 +++++++++++---- stacktach/tests.py | 48 ++++++++-- stacktach/views.py | 77 ++++++++++++++-- tests/unit/test_stacktach.py | 169 ++++++++++++++++++++++++++++++++++- tests/unit/test_worker.py | 30 +------ tests/unit/utils.py | 3 + worker/worker.py | 12 +-- 8 files changed, 378 insertions(+), 62 deletions(-) diff --git a/stacktach/db.py b/stacktach/db.py index 0043ec7..d28d723 100644 --- a/stacktach/db.py +++ b/stacktach/db.py @@ -103,4 +103,33 @@ def create_generic_rawdata(**kwargs): rawdata = models.GenericRawData(**kwargs) rawdata.save() - return rawdata \ No newline at end of file + return rawdata + + +def create_image_usage(**kwargs): + usage = models.ImageUsage(**kwargs) + usage.save() + + return usage + + +def create_image_delete(**kwargs): + delete = models.ImageDeletes(**kwargs) + delete.save() + + return delete + + +def create_image_exists(**kwargs): + exists = models.ImageExists(**kwargs) + exists.save() + + return exists + + +def get_image_delete(**kwargs): + return _safe_get(models.ImageDeletes, **kwargs) + + +def get_image_usage(**kwargs): + return _safe_get(models.ImageUsage, **kwargs) \ No newline at end of file diff --git a/stacktach/models.py b/stacktach/models.py index f2d3df4..4876313 100644 --- a/stacktach/models.py +++ b/stacktach/models.py @@ -1,18 +1,3 @@ -# Copyright 2012 - Dark Secret Software Inc. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - from django import forms from django.db import models @@ -316,5 +301,60 @@ class GlanceRawData(models.Model): def get_name(): return GlanceRawData.__name__ + +class ImageUsage(models.Model): + uuid = models.CharField(max_length=50, db_index=True) + created_at = models.DecimalField(max_digits=20, + decimal_places=6, db_index=True) + owner = models.CharField(max_length=50, db_index=True) + size = models.BigIntegerField(max_length=20) + last_raw = models.ForeignKey(GlanceRawData) + + +class ImageDeletes(models.Model): + uuid = models.CharField(max_length=50, db_index=True) + created_at = models.DecimalField(max_digits=20, + decimal_places=6, db_index=True) + deleted_at = models.DecimalField(max_digits=20, + decimal_places=6, db_index=True) + owner = models.CharField(max_length=50, db_index=True) + size = models.BigIntegerField(max_length=20) + raw = models.ForeignKey(GlanceRawData) + + +class ImageExists(models.Model): + PENDING = 'pending' + VERIFYING = 'verifying' + VERIFIED = 'verified' + FAILED = 'failed' + STATUS_CHOICES = [ + (PENDING, 'Pending Verification'), + (VERIFYING, 'Currently Being Verified'), + (VERIFIED, 'Passed Verification'), + (FAILED, 'Failed Verification'), + ] + + uuid = models.CharField(max_length=50, db_index=True) + created_at = models.DecimalField(max_digits=20, + decimal_places=6, db_index=True) + deleted_at = models.DecimalField(max_digits=20, + decimal_places=6, db_index=True) + audit_period_beginning = models.DecimalField(max_digits=20, + decimal_places=6, + db_index=True) + audit_period_ending = models.DecimalField(max_digits=20, + decimal_places=6, db_index=True) + status = models.CharField(max_length=50, db_index=True, + choices=STATUS_CHOICES, + default=PENDING) + fail_reason = models.CharField(max_length=300, db_index=True, null=True) + raw = models.ForeignKey(GlanceRawData, related_name='+') + usage = models.ForeignKey(ImageUsage, related_name='+') + delete = models.ForeignKey(ImageDeletes, related_name='+') + send_status = models.IntegerField(default=0, db_index=True) + owner = models.CharField(max_length=255, db_index=True) + size = models.BigIntegerField(max_length=20) + + def get_model_fields(model): return model._meta.fields diff --git a/stacktach/tests.py b/stacktach/tests.py index 80ec3ad..802a5fb 100644 --- a/stacktach/tests.py +++ b/stacktach/tests.py @@ -22,7 +22,7 @@ from datetime import datetime from django.test import TransactionTestCase import db from stacktach.datetime_to_decimal import dt_to_decimal -from stacktach.models import RawDataImageMeta +from stacktach.models import RawDataImageMeta, ImageUsage, ImageDeletes from stacktach.models import GenericRawData from stacktach.models import GlanceRawData from stacktach.models import RawData @@ -69,8 +69,8 @@ class RawDataImageMetaDbTestCase(TransactionTestCase): self.assertEquals(raw_image_meta.rax_options, kwargs['rax_options']) -class GlanceRawDataTestCase(TransactionTestCase): - def test_create_rawdata_should_populate_glance_rawdata(self): +class GlanceTestCase(TransactionTestCase): + def _create_glance_rawdata(self): deployment = db.get_or_create_deployment('deployment1')[0] kwargs = { 'deployment': deployment, @@ -88,15 +88,53 @@ class GlanceRawDataTestCase(TransactionTestCase): 'uuid': '1234-5678-0912-3456', 'status': 'active', } - db.create_glance_rawdata(**kwargs) - rawdata = GlanceRawData.objects.all().order_by('-id')[0] + rawdata = GlanceRawData.objects.all()[0] + return kwargs, rawdata + + def test_create_rawdata_should_populate_glance_rawdata(self): + kwargs, rawdata = self._create_glance_rawdata() for field in get_model_fields(GlanceRawData): if field.name != 'id': self.assertEquals(getattr(rawdata, field.name), kwargs[field.name]) + def test_create_glance_usage_should_populate_image_usage(self): + _, rawdata = self._create_glance_rawdata() + kwargs = { + 'uuid': '1', + 'created_at': dt_to_decimal(datetime.utcnow()), + 'owner': '1234567', + 'size': 12345, + 'last_raw': rawdata + } + db.create_image_usage(**kwargs) + usage = ImageUsage.objects.all()[0] + + for field in get_model_fields(ImageUsage): + if field.name != 'id': + self.assertEquals(getattr(usage, field.name), + kwargs[field.name]) + + def test_create_image_delete_should_populate_image_delete(self): + _, rawdata = self._create_glance_rawdata() + kwargs = { + 'uuid': '1', + 'created_at': dt_to_decimal(datetime.utcnow()), + 'owner': 'owner', + 'size': 12345, + 'raw': rawdata, + 'deleted_at': dt_to_decimal(datetime.utcnow()) + } + db.create_image_delete(**kwargs) + image_delete = ImageDeletes.objects.all()[0] + + for field in get_model_fields(ImageDeletes): + if field.name != 'id': + self.assertEquals(getattr(image_delete, field.name), + kwargs[field.name]) + class GenericRawDataTestCase(TransactionTestCase): def test_create_generic_rawdata_should_populate_generic_rawdata(self): diff --git a/stacktach/views.py b/stacktach/views.py index 15da309..386eee8 100644 --- a/stacktach/views.py +++ b/stacktach/views.py @@ -295,6 +295,55 @@ def _process_exists(raw, body): stacklog.warn("Ignoring exists without launched_at. RawData(%s)" % raw.id) +def _process_glance_usage(raw, notification): + values = { + 'uuid': notification.uuid, + 'created_at': notification.created_at, + 'owner': notification.owner, + 'size': notification.size, + 'last_raw': raw + } + STACKDB.create_image_usage(**values) + + +def _process_glance_delete(raw, notification): + values = { + 'uuid': notification.uuid, + 'created_at': notification.created_at, + 'owner': notification.owner, + 'size': notification.size, + 'raw': raw, + 'deleted_at': notification.deleted_at + } + STACKDB.create_image_delete(**values) + + +def _process_glance_exists(raw, notification): + if notification.created_at: + values = { + 'uuid': notification.uuid, + 'audit_period_beginning': notification.audit_period_beginning, + 'audit_period_ending': notification.audit_period_ending, + 'owner': notification.owner, + 'size': notification.size, + 'raw': raw, + } + created_at_range = (notification.created_at, notification.created_at+1) + usage = STACKDB.get_image_usage(uuid=notification.uuid, + created_at__range=created_at_range) + values['usage'] = usage + values['created_at'] = notification.created_at + if notification.deleted_at: + delete = STACKDB.get_image_delete(uuid=notification.uuid, + created_at__range=created_at_range) + values['delete'] = delete + values['deleted_at'] = notification.deleted_at + + STACKDB.create_image_exists(**values) + else: + stacklog.warn("Ignoring exists without created_at. GlanceRawData(%s)" + % raw.id) + USAGE_PROCESS_MAPPING = { INSTANCE_EVENT['create_start']: _process_usage_for_new_launch, INSTANCE_EVENT['rebuild_start']: _process_usage_for_new_launch, @@ -306,8 +355,14 @@ USAGE_PROCESS_MAPPING = { INSTANCE_EVENT['resize_finish_end']: _process_usage_for_updates, INSTANCE_EVENT['resize_revert_end']: _process_usage_for_updates, INSTANCE_EVENT['delete_end']: _process_delete, - INSTANCE_EVENT['exists']: _process_exists, -} + INSTANCE_EVENT['exists']: _process_exists +} + +GLANCE_USAGE_PROCESS_MAPPING = { + 'image.activate': _process_glance_usage, + 'image.delete': _process_glance_delete, + 'image.exists': _process_glance_exists +} def aggregate_usage(raw, body): @@ -318,6 +373,11 @@ def aggregate_usage(raw, body): USAGE_PROCESS_MAPPING[raw.event](raw, body) +def aggregate_glance_usage(raw, body): + if raw.event in GLANCE_USAGE_PROCESS_MAPPING.keys(): + GLANCE_USAGE_PROCESS_MAPPING[raw.event](raw, body) + + def process_raw_data(deployment, args, json_args, exchange): """This is called directly by the worker to add the event to the db.""" db.reset_queries() @@ -325,19 +385,20 @@ def process_raw_data(deployment, args, json_args, exchange): routing_key, body = args notif = notification.notification_factory(body, deployment, routing_key, json_args, exchange) - return notif.save() + raw = notif.save() + return raw, notif -def post_process_rawdata(raw, body): +def post_process_rawdata(raw, notification): aggregate_lifecycle(raw) - aggregate_usage(raw, body) + aggregate_usage(raw, notification) -def post_process_glancerawdata(raw, body): - pass +def post_process_glancerawdata(raw, notification): + aggregate_glance_usage(raw, notification) -def post_process_genericrawdata(raw, body): +def post_process_genericrawdata(raw, body, notification): pass diff --git a/tests/unit/test_stacktach.py b/tests/unit/test_stacktach.py index 4138997..055d3d3 100644 --- a/tests/unit/test_stacktach.py +++ b/tests/unit/test_stacktach.py @@ -26,6 +26,7 @@ import mox import utils from utils import INSTANCE_ID_1 +from utils import DECIMAL_DUMMY_TIME from utils import OS_VERSION_1 from utils import OS_ARCH_1 from utils import OS_DISTRO_1 @@ -36,6 +37,7 @@ from utils import TENANT_ID_1 from utils import INSTANCE_TYPE_ID_1 from utils import DUMMY_TIME from utils import INSTANCE_TYPE_ID_2 +from utils import IMAGE_UUID_1 from stacktach import stacklog from stacktach import notification from stacktach import views @@ -75,7 +77,7 @@ class StacktachRawParsingTestCase(unittest.TestCase): self.assertEquals( views.process_raw_data(deployment, args, json_args, exchange), - mock_record) + (mock_record, mock_notification)) self.mox.VerifyAll() def test_process_raw_data_old_timestamp(self): @@ -710,3 +712,168 @@ class StacktachUsageParsingTestCase(unittest.TestCase): views._process_exists(raw, notif[1]) self.mox.VerifyAll() + +class StacktachImageUsageParsingTestCase(unittest.TestCase): + def setUp(self): + self.mox = mox.Mox() + views.STACKDB = self.mox.CreateMockAnything() + + def tearDown(self): + self.mox.UnsetStubs() + + def test_process_image_usage_for_new_launch(self): + raw = self.mox.CreateMockAnything() + values = { + 'created_at': str(DUMMY_TIME), + 'owner': TENANT_ID_1, + 'uuid': IMAGE_UUID_1, + 'size': 1234, + 'last_raw': raw + } + notification = self.mox.CreateMockAnything() + notification.created_at = values['created_at'] + notification.owner = values['owner'] + notification.uuid = values['uuid'] + notification.size = values['size'] + notification.raw = values['last_raw'] + views.STACKDB.create_image_usage(**values) + self.mox.ReplayAll() + views._process_glance_usage(raw, notification) + self.mox.VerifyAll() + + def test_process_image_deletes(self): + raw = self.mox.CreateMockAnything() + values = { + 'uuid': IMAGE_UUID_1, + 'created_at': str(DUMMY_TIME), + 'deleted_at': str(DUMMY_TIME), + 'owner': TENANT_ID_1, + 'size': 1234, + 'raw': raw + } + + notification = self.mox.CreateMockAnything() + notification.created_at = values['created_at'] + notification.deleted_at = values['deleted_at'] + notification.owner = values['owner'] + notification.uuid = values['uuid'] + notification.size = values['size'] + notification.raw = values['raw'] + views.STACKDB.create_image_delete(**values) + self.mox.ReplayAll() + views._process_glance_delete(raw, notification) + self.mox.VerifyAll() + + def test_process_image_exists(self): + raw = self.mox.CreateMockAnything() + values = { + 'uuid': IMAGE_UUID_1, + 'created_at': DECIMAL_DUMMY_TIME, + 'deleted_at': DECIMAL_DUMMY_TIME, + 'audit_period_beginning': DECIMAL_DUMMY_TIME, + 'audit_period_ending': DECIMAL_DUMMY_TIME, + 'owner': TENANT_ID_1, + 'size': 1234, + 'raw': raw, + 'usage': None, + 'delete': None + } + + notification = self.mox.CreateMockAnything() + notification.created_at = values['created_at'] + notification.deleted_at = values['deleted_at'] + notification.owner = values['owner'] + notification.uuid = values['uuid'] + notification.size = values['size'] + notification.raw = values['raw'] + notification.audit_period_beginning = values['audit_period_beginning'] + notification.audit_period_ending = values['audit_period_ending'] + created_at_range = (DECIMAL_DUMMY_TIME, DECIMAL_DUMMY_TIME+1) + views.STACKDB.get_image_usage(created_at__range=created_at_range, + uuid=notification.uuid).AndReturn(None) + views.STACKDB.get_image_delete(created_at__range=created_at_range, + uuid=notification.uuid).AndReturn(None) + views.STACKDB.create_image_exists(**values) + + self.mox.ReplayAll() + + views._process_glance_exists(raw, notification) + self.mox.VerifyAll() + + def test_process_image_exists_with_usage_not_none(self): + raw = self.mox.CreateMockAnything() + usage = self.mox.CreateMockAnything() + values = { + 'uuid': IMAGE_UUID_1, + 'created_at': DECIMAL_DUMMY_TIME, + 'deleted_at': DECIMAL_DUMMY_TIME, + 'audit_period_beginning': DECIMAL_DUMMY_TIME, + 'audit_period_ending': DECIMAL_DUMMY_TIME, + 'owner': TENANT_ID_1, + 'size': 1234, + 'raw': raw, + 'usage': usage, + 'delete': None + } + + notification = self.mox.CreateMockAnything() + notification.created_at = values['created_at'] + notification.deleted_at = values['deleted_at'] + notification.owner = values['owner'] + notification.uuid = values['uuid'] + notification.size = values['size'] + notification.raw = values['raw'] + notification.audit_period_beginning = values['audit_period_beginning'] + notification.audit_period_ending = values['audit_period_ending'] + created_at_range = (DECIMAL_DUMMY_TIME, DECIMAL_DUMMY_TIME+1) + views.STACKDB.get_image_usage(created_at__range=created_at_range, + uuid=notification.uuid).AndReturn(usage) + views.STACKDB.get_image_delete(created_at__range=created_at_range, + uuid=notification.uuid).AndReturn(None) + views.STACKDB.create_image_exists(**values) + + self.mox.ReplayAll() + + views._process_glance_exists(raw, notification) + self.mox.VerifyAll() + + + def test_process_image_exists_with_delete_not_none(self): + raw = self.mox.CreateMockAnything() + delete = self.mox.CreateMockAnything() + values = { + 'uuid': IMAGE_UUID_1, + 'created_at': DECIMAL_DUMMY_TIME, + 'deleted_at': DECIMAL_DUMMY_TIME, + 'audit_period_beginning': DECIMAL_DUMMY_TIME, + 'audit_period_ending': DECIMAL_DUMMY_TIME, + 'owner': TENANT_ID_1, + 'size': 1234, + 'raw': raw, + 'usage': None, + 'delete': delete + } + + notification = self.mox.CreateMockAnything() + notification.created_at = values['created_at'] + notification.deleted_at = values['deleted_at'] + notification.owner = values['owner'] + notification.uuid = values['uuid'] + notification.size = values['size'] + notification.raw = values['raw'] + notification.audit_period_beginning = values['audit_period_beginning'] + notification.audit_period_ending = values['audit_period_ending'] + created_at_range = (DECIMAL_DUMMY_TIME, DECIMAL_DUMMY_TIME+1) + views.STACKDB.get_image_usage(created_at__range=created_at_range, + uuid=notification.uuid).AndReturn(None) + views.STACKDB.get_image_delete(created_at__range=created_at_range, + uuid=notification.uuid).AndReturn(delete) + views.STACKDB.create_image_exists(**values) + + self.mox.ReplayAll() + + views._process_glance_exists(raw, notification) + self.mox.VerifyAll() + + # def test_process_image_exists_should_not_populate_delete_and_deleted_at_when_deleted_at_is_absent(self): + diff --git a/tests/unit/test_worker.py b/tests/unit/test_worker.py index f21c227..a7db64b 100644 --- a/tests/unit/test_worker.py +++ b/tests/unit/test_worker.py @@ -128,8 +128,9 @@ class ConsumerTestCase(unittest.TestCase): body_dict = {u'key': u'value'} message.body = json.dumps(body_dict) + mock_notification = self.mox.CreateMockAnything() mock_post_process_method = self.mox.CreateMockAnything() - mock_post_process_method(raw, body_dict) + mock_post_process_method(raw, mock_notification) old_handler = worker.POST_PROCESS_METHODS worker.POST_PROCESS_METHODS["RawData"] = mock_post_process_method @@ -137,7 +138,7 @@ class ConsumerTestCase(unittest.TestCase): use_mock_anything=True) args = (routing_key, body_dict) views.process_raw_data(deployment, args, json.dumps(args), exchange) \ - .AndReturn(raw) + .AndReturn((raw, mock_notification)) message.ack() self.mox.StubOutWithMock(consumer, '_check_memory', @@ -149,31 +150,6 @@ class ConsumerTestCase(unittest.TestCase): self.mox.VerifyAll() worker.POST_PROCESS_METHODS["RawData"] = old_handler - def test_process_no_raw_dont_ack(self): - deployment = self.mox.CreateMockAnything() - raw = self.mox.CreateMockAnything() - message = self.mox.CreateMockAnything() - - exchange = 'nova' - consumer = worker.Consumer('test', None, deployment, True, {}, - exchange, ["monitor.info", "monitor.error"]) - routing_key = 'monitor.info' - message.delivery_info = {'routing_key': routing_key} - body_dict = {u'key': u'value'} - message.body = json.dumps(body_dict) - self.mox.StubOutWithMock(views, 'process_raw_data', - use_mock_anything=True) - args = (routing_key, body_dict) - views.process_raw_data(deployment, args, json.dumps(args), exchange) \ - .AndReturn(None) - self.mox.StubOutWithMock(consumer, '_check_memory', - use_mock_anything=True) - consumer._check_memory() - self.mox.ReplayAll() - consumer._process(message) - self.assertEqual(consumer.processed, 0) - self.mox.VerifyAll() - def test_run(self): config = { 'name': 'east_coast.prod.global', diff --git a/tests/unit/utils.py b/tests/unit/utils.py index 437e169..222b427 100644 --- a/tests/unit/utils.py +++ b/tests/unit/utils.py @@ -25,6 +25,8 @@ TENANT_ID_2 = 'testtenantid2' from stacktach import datetime_to_decimal as dt +IMAGE_UUID_1 = "1" + INSTANCE_ID_1 = "08f685d9-6352-4dbc-8271-96cc54bf14cd" INSTANCE_ID_2 = "515adf96-41d3-b86d-5467-e584edc61dab" @@ -32,6 +34,7 @@ INSTANCE_TYPE_ID_1 = "12345" INSTANCE_TYPE_ID_2 = '54321' DUMMY_TIME = datetime.datetime.utcnow() +DECIMAL_DUMMY_TIME = dt.dt_to_decimal(DUMMY_TIME) MESSAGE_ID_1 = "7f28f81b-29a2-43f2-9ba1-ccb3e53ab6c8" MESSAGE_ID_2 = "4d596126-0f04-4329-865f-7b9a7bd69bcf" diff --git a/worker/worker.py b/worker/worker.py index 11efb98..e542b9c 100644 --- a/worker/worker.py +++ b/worker/worker.py @@ -34,7 +34,9 @@ except ImportError: from pympler.process import ProcessMemoryInfo +from django import db as django_db from stacktach import db +from stacktach import notification from stacktach import stacklog from stacktach import views @@ -84,12 +86,12 @@ class Consumer(kombu.mixins.ConsumerMixin): args = (routing_key, json.loads(body)) asJson = json.dumps(args) # save raw and ack the message - raw = views.process_raw_data(self.deployment, args, asJson, self.exchange) + raw, notif = views.process_raw_data( + self.deployment, args, asJson, self.exchange) - if raw: - self.processed += 1 - message.ack() - POST_PROCESS_METHODS[raw.get_name()](raw, args[1]) + self.processed += 1 + message.ack() + POST_PROCESS_METHODS[raw.get_name()](raw, notif) self._check_memory()