Added image exists, usage and delete models for glance

stacktach/db.py

@@ -103,4 +103,33 @@ def create_generic_rawdata(**kwargs):
     rawdata = models.GenericRawData(**kwargs)
     rawdata.save()
 
-    return rawdata
+    return rawdata
+
+
+def create_image_usage(**kwargs):
+    usage = models.ImageUsage(**kwargs)
+    usage.save()
+
+    return usage
+
+
+def create_image_delete(**kwargs):
+    delete = models.ImageDeletes(**kwargs)
+    delete.save()
+
+    return delete
+
+
+def create_image_exists(**kwargs):
+    exists = models.ImageExists(**kwargs)
+    exists.save()
+
+    return exists
+
+
+def get_image_delete(**kwargs):
+    return _safe_get(models.ImageDeletes, **kwargs)
+
+
+def get_image_usage(**kwargs):
+    return _safe_get(models.ImageUsage, **kwargs)

stacktach/models.py

@@ -1,18 +1,3 @@
-# Copyright 2012 - Dark Secret Software Inc.
-# All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
 from django import forms
 from django.db import models
 
@@ -316,5 +301,60 @@ class GlanceRawData(models.Model):
     def get_name():
         return GlanceRawData.__name__
 
+
+class ImageUsage(models.Model):
+    uuid = models.CharField(max_length=50, db_index=True)
+    created_at = models.DecimalField(max_digits=20,
+                                     decimal_places=6, db_index=True)
+    owner = models.CharField(max_length=50, db_index=True)
+    size = models.BigIntegerField(max_length=20)
+    last_raw = models.ForeignKey(GlanceRawData)
+
+
+class ImageDeletes(models.Model):
+    uuid = models.CharField(max_length=50, db_index=True)
+    created_at = models.DecimalField(max_digits=20,
+                                     decimal_places=6, db_index=True)
+    deleted_at = models.DecimalField(max_digits=20,
+                                     decimal_places=6, db_index=True)
+    owner = models.CharField(max_length=50, db_index=True)
+    size = models.BigIntegerField(max_length=20)
+    raw = models.ForeignKey(GlanceRawData)
+
+
+class ImageExists(models.Model):
+    PENDING = 'pending'
+    VERIFYING = 'verifying'
+    VERIFIED = 'verified'
+    FAILED = 'failed'
+    STATUS_CHOICES = [
+        (PENDING, 'Pending Verification'),
+        (VERIFYING, 'Currently Being Verified'),
+        (VERIFIED, 'Passed Verification'),
+        (FAILED, 'Failed Verification'),
+    ]
+
+    uuid = models.CharField(max_length=50, db_index=True)
+    created_at = models.DecimalField(max_digits=20,
+                                     decimal_places=6, db_index=True)
+    deleted_at = models.DecimalField(max_digits=20,
+                                     decimal_places=6, db_index=True)
+    audit_period_beginning = models.DecimalField(max_digits=20,
+                                                 decimal_places=6,
+                                                 db_index=True)
+    audit_period_ending = models.DecimalField(max_digits=20,
+                                              decimal_places=6, db_index=True)
+    status = models.CharField(max_length=50, db_index=True,
+                              choices=STATUS_CHOICES,
+                              default=PENDING)
+    fail_reason = models.CharField(max_length=300, db_index=True, null=True)
+    raw = models.ForeignKey(GlanceRawData, related_name='+')
+    usage = models.ForeignKey(ImageUsage, related_name='+')
+    delete = models.ForeignKey(ImageDeletes, related_name='+')
+    send_status = models.IntegerField(default=0, db_index=True)
+    owner = models.CharField(max_length=255, db_index=True)
+    size = models.BigIntegerField(max_length=20)
+
+
 def get_model_fields(model):
     return model._meta.fields

stacktach/tests.py

@@ -22,7 +22,7 @@ from datetime import datetime
 from django.test import TransactionTestCase
 import db
 from stacktach.datetime_to_decimal import dt_to_decimal
-from stacktach.models import RawDataImageMeta
+from stacktach.models import RawDataImageMeta, ImageUsage, ImageDeletes
 from stacktach.models import GenericRawData
 from stacktach.models import GlanceRawData
 from stacktach.models import RawData
@@ -69,8 +69,8 @@ class RawDataImageMetaDbTestCase(TransactionTestCase):
         self.assertEquals(raw_image_meta.rax_options, kwargs['rax_options'])
 
 
-class GlanceRawDataTestCase(TransactionTestCase):
+class GlanceTestCase(TransactionTestCase):
-    def test_create_rawdata_should_populate_glance_rawdata(self):
+    def _create_glance_rawdata(self):
         deployment = db.get_or_create_deployment('deployment1')[0]
         kwargs = {
             'deployment': deployment,
@@ -88,15 +88,53 @@ class GlanceRawDataTestCase(TransactionTestCase):
             'uuid': '1234-5678-0912-3456',
             'status': 'active',
         }
-
         db.create_glance_rawdata(**kwargs)
-        rawdata = GlanceRawData.objects.all().order_by('-id')[0]
+        rawdata = GlanceRawData.objects.all()[0]
+        return kwargs, rawdata
+
+    def test_create_rawdata_should_populate_glance_rawdata(self):
+        kwargs, rawdata = self._create_glance_rawdata()
 
         for field in get_model_fields(GlanceRawData):
             if field.name != 'id':
                 self.assertEquals(getattr(rawdata, field.name),
                                   kwargs[field.name])
 
+    def test_create_glance_usage_should_populate_image_usage(self):
+        _, rawdata = self._create_glance_rawdata()
+        kwargs = {
+            'uuid': '1',
+            'created_at': dt_to_decimal(datetime.utcnow()),
+            'owner': '1234567',
+            'size': 12345,
+            'last_raw': rawdata
+        }
+        db.create_image_usage(**kwargs)
+        usage = ImageUsage.objects.all()[0]
+
+        for field in get_model_fields(ImageUsage):
+            if field.name != 'id':
+                self.assertEquals(getattr(usage, field.name),
+                                  kwargs[field.name])
+
+    def test_create_image_delete_should_populate_image_delete(self):
+        _, rawdata = self._create_glance_rawdata()
+        kwargs = {
+            'uuid': '1',
+            'created_at': dt_to_decimal(datetime.utcnow()),
+            'owner': 'owner',
+            'size': 12345,
+            'raw': rawdata,
+            'deleted_at': dt_to_decimal(datetime.utcnow())
+        }
+        db.create_image_delete(**kwargs)
+        image_delete = ImageDeletes.objects.all()[0]
+
+        for field in get_model_fields(ImageDeletes):
+            if field.name != 'id':
+                self.assertEquals(getattr(image_delete, field.name),
+                                  kwargs[field.name])
+
 
 class GenericRawDataTestCase(TransactionTestCase):
     def test_create_generic_rawdata_should_populate_generic_rawdata(self):

stacktach/views.py

@@ -295,6 +295,55 @@ def _process_exists(raw, body):
         stacklog.warn("Ignoring exists without launched_at. RawData(%s)" % raw.id)
 
 
+def _process_glance_usage(raw, notification):
+    values = {
+        'uuid': notification.uuid,
+        'created_at': notification.created_at,
+        'owner': notification.owner,
+        'size': notification.size,
+        'last_raw': raw
+    }
+    STACKDB.create_image_usage(**values)
+
+
+def _process_glance_delete(raw, notification):
+    values = {
+        'uuid': notification.uuid,
+        'created_at': notification.created_at,
+        'owner': notification.owner,
+        'size': notification.size,
+        'raw': raw,
+        'deleted_at': notification.deleted_at
+    }
+    STACKDB.create_image_delete(**values)
+
+
+def _process_glance_exists(raw, notification):
+    if notification.created_at:
+        values = {
+            'uuid': notification.uuid,
+            'audit_period_beginning': notification.audit_period_beginning,
+            'audit_period_ending': notification.audit_period_ending,
+            'owner': notification.owner,
+            'size': notification.size,
+            'raw': raw,
+        }
+        created_at_range = (notification.created_at, notification.created_at+1)
+        usage = STACKDB.get_image_usage(uuid=notification.uuid,
+                                         created_at__range=created_at_range)
+        values['usage'] = usage
+        values['created_at'] = notification.created_at
+        if notification.deleted_at:
+            delete = STACKDB.get_image_delete(uuid=notification.uuid,
+                                              created_at__range=created_at_range)
+            values['delete'] = delete
+            values['deleted_at'] = notification.deleted_at
+
+        STACKDB.create_image_exists(**values)
+    else:
+        stacklog.warn("Ignoring exists without created_at. GlanceRawData(%s)"
+                      % raw.id)
+
 USAGE_PROCESS_MAPPING = {
     INSTANCE_EVENT['create_start']: _process_usage_for_new_launch,
     INSTANCE_EVENT['rebuild_start']: _process_usage_for_new_launch,
@@ -306,8 +355,14 @@ USAGE_PROCESS_MAPPING = {
     INSTANCE_EVENT['resize_finish_end']: _process_usage_for_updates,
     INSTANCE_EVENT['resize_revert_end']: _process_usage_for_updates,
     INSTANCE_EVENT['delete_end']: _process_delete,
-    INSTANCE_EVENT['exists']: _process_exists,
+    INSTANCE_EVENT['exists']: _process_exists
-} 
+}
+
+GLANCE_USAGE_PROCESS_MAPPING = {
+    'image.activate': _process_glance_usage,
+    'image.delete': _process_glance_delete,
+    'image.exists': _process_glance_exists
+}
 
 
 def aggregate_usage(raw, body):
@@ -318,6 +373,11 @@ def aggregate_usage(raw, body):
         USAGE_PROCESS_MAPPING[raw.event](raw, body)
 
 
+def aggregate_glance_usage(raw, body):
+    if raw.event in GLANCE_USAGE_PROCESS_MAPPING.keys():
+        GLANCE_USAGE_PROCESS_MAPPING[raw.event](raw, body)
+
+
 def process_raw_data(deployment, args, json_args, exchange):
     """This is called directly by the worker to add the event to the db."""
     db.reset_queries()
@@ -325,19 +385,20 @@ def process_raw_data(deployment, args, json_args, exchange):
     routing_key, body = args
     notif = notification.notification_factory(body, deployment, routing_key,
                                               json_args, exchange)
-    return notif.save()
+    raw = notif.save()
+    return raw, notif
 
 
-def post_process_rawdata(raw, body):
+def post_process_rawdata(raw, notification):
     aggregate_lifecycle(raw)
-    aggregate_usage(raw, body)
+    aggregate_usage(raw, notification)
 
 
-def post_process_glancerawdata(raw, body):
+def post_process_glancerawdata(raw, notification):
-    pass
+    aggregate_glance_usage(raw, notification)
 
 
-def post_process_genericrawdata(raw, body):
+def post_process_genericrawdata(raw, body, notification):
     pass
 
 

tests/unit/test_stacktach.py

@@ -26,6 +26,7 @@ import mox
 
 import utils
 from utils import INSTANCE_ID_1
+from utils import DECIMAL_DUMMY_TIME
 from utils import OS_VERSION_1
 from utils import OS_ARCH_1
 from utils import OS_DISTRO_1
@@ -36,6 +37,7 @@ from utils import TENANT_ID_1
 from utils import INSTANCE_TYPE_ID_1
 from utils import DUMMY_TIME
 from utils import INSTANCE_TYPE_ID_2
+from utils import IMAGE_UUID_1
 from stacktach import stacklog
 from stacktach import notification
 from stacktach import views
@@ -75,7 +77,7 @@ class StacktachRawParsingTestCase(unittest.TestCase):
 
         self.assertEquals(
             views.process_raw_data(deployment, args, json_args, exchange),
-            mock_record)
+                                  (mock_record, mock_notification))
         self.mox.VerifyAll()
 
     def test_process_raw_data_old_timestamp(self):
@@ -710,3 +712,168 @@ class StacktachUsageParsingTestCase(unittest.TestCase):
         views._process_exists(raw, notif[1])
         self.mox.VerifyAll()
 
+
+class StacktachImageUsageParsingTestCase(unittest.TestCase):
+    def setUp(self):
+        self.mox = mox.Mox()
+        views.STACKDB = self.mox.CreateMockAnything()
+
+    def tearDown(self):
+        self.mox.UnsetStubs()
+
+    def test_process_image_usage_for_new_launch(self):
+        raw = self.mox.CreateMockAnything()
+        values = {
+            'created_at': str(DUMMY_TIME),
+            'owner': TENANT_ID_1,
+            'uuid': IMAGE_UUID_1,
+            'size': 1234,
+            'last_raw': raw
+        }
+        notification = self.mox.CreateMockAnything()
+        notification.created_at = values['created_at']
+        notification.owner = values['owner']
+        notification.uuid = values['uuid']
+        notification.size = values['size']
+        notification.raw = values['last_raw']
+        views.STACKDB.create_image_usage(**values)
+        self.mox.ReplayAll()
+        views._process_glance_usage(raw, notification)
+        self.mox.VerifyAll()
+
+    def test_process_image_deletes(self):
+        raw = self.mox.CreateMockAnything()
+        values = {
+            'uuid': IMAGE_UUID_1,
+            'created_at': str(DUMMY_TIME),
+            'deleted_at': str(DUMMY_TIME),
+            'owner': TENANT_ID_1,
+            'size': 1234,
+            'raw': raw
+        }
+
+        notification = self.mox.CreateMockAnything()
+        notification.created_at = values['created_at']
+        notification.deleted_at = values['deleted_at']
+        notification.owner = values['owner']
+        notification.uuid = values['uuid']
+        notification.size = values['size']
+        notification.raw = values['raw']
+        views.STACKDB.create_image_delete(**values)
+        self.mox.ReplayAll()
+        views._process_glance_delete(raw, notification)
+        self.mox.VerifyAll()
+
+    def test_process_image_exists(self):
+        raw = self.mox.CreateMockAnything()
+        values = {
+            'uuid': IMAGE_UUID_1,
+            'created_at': DECIMAL_DUMMY_TIME,
+            'deleted_at': DECIMAL_DUMMY_TIME,
+            'audit_period_beginning': DECIMAL_DUMMY_TIME,
+            'audit_period_ending': DECIMAL_DUMMY_TIME,
+            'owner': TENANT_ID_1,
+            'size': 1234,
+            'raw': raw,
+            'usage': None,
+            'delete': None
+        }
+
+        notification = self.mox.CreateMockAnything()
+        notification.created_at = values['created_at']
+        notification.deleted_at = values['deleted_at']
+        notification.owner = values['owner']
+        notification.uuid = values['uuid']
+        notification.size = values['size']
+        notification.raw = values['raw']
+        notification.audit_period_beginning = values['audit_period_beginning']
+        notification.audit_period_ending = values['audit_period_ending']
+        created_at_range = (DECIMAL_DUMMY_TIME, DECIMAL_DUMMY_TIME+1)
+        views.STACKDB.get_image_usage(created_at__range=created_at_range,
+                                      uuid=notification.uuid).AndReturn(None)
+        views.STACKDB.get_image_delete(created_at__range=created_at_range,
+                                       uuid=notification.uuid).AndReturn(None)
+        views.STACKDB.create_image_exists(**values)
+
+        self.mox.ReplayAll()
+
+        views._process_glance_exists(raw, notification)
+        self.mox.VerifyAll()
+
+    def test_process_image_exists_with_usage_not_none(self):
+        raw = self.mox.CreateMockAnything()
+        usage = self.mox.CreateMockAnything()
+        values = {
+            'uuid': IMAGE_UUID_1,
+            'created_at': DECIMAL_DUMMY_TIME,
+            'deleted_at': DECIMAL_DUMMY_TIME,
+            'audit_period_beginning': DECIMAL_DUMMY_TIME,
+            'audit_period_ending': DECIMAL_DUMMY_TIME,
+            'owner': TENANT_ID_1,
+            'size': 1234,
+            'raw': raw,
+            'usage': usage,
+            'delete': None
+        }
+
+        notification = self.mox.CreateMockAnything()
+        notification.created_at = values['created_at']
+        notification.deleted_at = values['deleted_at']
+        notification.owner = values['owner']
+        notification.uuid = values['uuid']
+        notification.size = values['size']
+        notification.raw = values['raw']
+        notification.audit_period_beginning = values['audit_period_beginning']
+        notification.audit_period_ending = values['audit_period_ending']
+        created_at_range = (DECIMAL_DUMMY_TIME, DECIMAL_DUMMY_TIME+1)
+        views.STACKDB.get_image_usage(created_at__range=created_at_range,
+                                      uuid=notification.uuid).AndReturn(usage)
+        views.STACKDB.get_image_delete(created_at__range=created_at_range,
+                                       uuid=notification.uuid).AndReturn(None)
+        views.STACKDB.create_image_exists(**values)
+
+        self.mox.ReplayAll()
+
+        views._process_glance_exists(raw, notification)
+        self.mox.VerifyAll()
+
+
+    def test_process_image_exists_with_delete_not_none(self):
+        raw = self.mox.CreateMockAnything()
+        delete = self.mox.CreateMockAnything()
+        values = {
+            'uuid': IMAGE_UUID_1,
+            'created_at': DECIMAL_DUMMY_TIME,
+            'deleted_at': DECIMAL_DUMMY_TIME,
+            'audit_period_beginning': DECIMAL_DUMMY_TIME,
+            'audit_period_ending': DECIMAL_DUMMY_TIME,
+            'owner': TENANT_ID_1,
+            'size': 1234,
+            'raw': raw,
+            'usage': None,
+            'delete': delete
+        }
+
+        notification = self.mox.CreateMockAnything()
+        notification.created_at = values['created_at']
+        notification.deleted_at = values['deleted_at']
+        notification.owner = values['owner']
+        notification.uuid = values['uuid']
+        notification.size = values['size']
+        notification.raw = values['raw']
+        notification.audit_period_beginning = values['audit_period_beginning']
+        notification.audit_period_ending = values['audit_period_ending']
+        created_at_range = (DECIMAL_DUMMY_TIME, DECIMAL_DUMMY_TIME+1)
+        views.STACKDB.get_image_usage(created_at__range=created_at_range,
+                                      uuid=notification.uuid).AndReturn(None)
+        views.STACKDB.get_image_delete(created_at__range=created_at_range,
+                                       uuid=notification.uuid).AndReturn(delete)
+        views.STACKDB.create_image_exists(**values)
+
+        self.mox.ReplayAll()
+
+        views._process_glance_exists(raw, notification)
+        self.mox.VerifyAll()
+
+    # def test_process_image_exists_should_not_populate_delete_and_deleted_at_when_deleted_at_is_absent(self):
+

tests/unit/test_worker.py

@@ -128,8 +128,9 @@ class ConsumerTestCase(unittest.TestCase):
         body_dict = {u'key': u'value'}
         message.body = json.dumps(body_dict)
 
+        mock_notification = self.mox.CreateMockAnything()
         mock_post_process_method = self.mox.CreateMockAnything()
-        mock_post_process_method(raw, body_dict)
+        mock_post_process_method(raw, mock_notification)
         old_handler = worker.POST_PROCESS_METHODS
         worker.POST_PROCESS_METHODS["RawData"] = mock_post_process_method
 
@@ -137,7 +138,7 @@ class ConsumerTestCase(unittest.TestCase):
                                  use_mock_anything=True)
         args = (routing_key, body_dict)
         views.process_raw_data(deployment, args, json.dumps(args), exchange) \
-            .AndReturn(raw)
+            .AndReturn((raw, mock_notification))
         message.ack()
 
         self.mox.StubOutWithMock(consumer, '_check_memory',
@@ -149,31 +150,6 @@ class ConsumerTestCase(unittest.TestCase):
         self.mox.VerifyAll()
         worker.POST_PROCESS_METHODS["RawData"] = old_handler
 
-    def test_process_no_raw_dont_ack(self):
-        deployment = self.mox.CreateMockAnything()
-        raw = self.mox.CreateMockAnything()
-        message = self.mox.CreateMockAnything()
-
-        exchange = 'nova'
-        consumer = worker.Consumer('test', None, deployment, True, {},
-                                   exchange, ["monitor.info", "monitor.error"])
-        routing_key = 'monitor.info'
-        message.delivery_info = {'routing_key': routing_key}
-        body_dict = {u'key': u'value'}
-        message.body = json.dumps(body_dict)
-        self.mox.StubOutWithMock(views, 'process_raw_data',
-                                 use_mock_anything=True)
-        args = (routing_key, body_dict)
-        views.process_raw_data(deployment, args, json.dumps(args), exchange) \
-            .AndReturn(None)
-        self.mox.StubOutWithMock(consumer, '_check_memory',
-                                 use_mock_anything=True)
-        consumer._check_memory()
-        self.mox.ReplayAll()
-        consumer._process(message)
-        self.assertEqual(consumer.processed, 0)
-        self.mox.VerifyAll()
-
     def test_run(self):
         config = {
             'name': 'east_coast.prod.global',

tests/unit/utils.py

@@ -25,6 +25,8 @@ TENANT_ID_2 = 'testtenantid2'
 
 from stacktach import datetime_to_decimal as dt
 
+IMAGE_UUID_1 = "1"
+
 INSTANCE_ID_1 = "08f685d9-6352-4dbc-8271-96cc54bf14cd"
 INSTANCE_ID_2 = "515adf96-41d3-b86d-5467-e584edc61dab"
 
@@ -32,6 +34,7 @@ INSTANCE_TYPE_ID_1 = "12345"
 INSTANCE_TYPE_ID_2 = '54321'
 
 DUMMY_TIME = datetime.datetime.utcnow()
+DECIMAL_DUMMY_TIME = dt.dt_to_decimal(DUMMY_TIME)
 
 MESSAGE_ID_1 = "7f28f81b-29a2-43f2-9ba1-ccb3e53ab6c8"
 MESSAGE_ID_2 = "4d596126-0f04-4329-865f-7b9a7bd69bcf"

worker/worker.py

@@ -34,7 +34,9 @@ except ImportError:
 
 from pympler.process import ProcessMemoryInfo
 
+from django import db as django_db
 from stacktach import db
+from stacktach import notification
 from stacktach import stacklog
 from stacktach import views
 
@@ -84,12 +86,12 @@ class Consumer(kombu.mixins.ConsumerMixin):
         args = (routing_key, json.loads(body))
         asJson = json.dumps(args)
         # save raw and ack the message
-        raw = views.process_raw_data(self.deployment, args, asJson, self.exchange)
+        raw, notif = views.process_raw_data(
+            self.deployment, args, asJson, self.exchange)
 
-        if raw:
+        self.processed += 1
-            self.processed += 1
+        message.ack()
-            message.ack()
+        POST_PROCESS_METHODS[raw.get_name()](raw, notif)
-            POST_PROCESS_METHODS[raw.get_name()](raw, args[1])
 
         self._check_memory()