diff --git a/stacktach/migrations/0005_create_glancerawdata_and_genericrawdata.py b/stacktach/migrations/0005_create_glance_usage_verification_tables.py similarity index 73% rename from stacktach/migrations/0005_create_glancerawdata_and_genericrawdata.py rename to stacktach/migrations/0005_create_glance_usage_verification_tables.py index 172c5bc..34aafeb 100644 --- a/stacktach/migrations/0005_create_glancerawdata_and_genericrawdata.py +++ b/stacktach/migrations/0005_create_glance_usage_verification_tables.py @@ -8,6 +8,18 @@ from django.db import models class Migration(SchemaMigration): def forwards(self, orm): + # Adding model 'ImageDeletes' + db.create_table(u'stacktach_imagedeletes', ( + (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('uuid', self.gf('django.db.models.fields.CharField')(max_length=50, db_index=True)), + ('created_at', self.gf('django.db.models.fields.DecimalField')(max_digits=20, decimal_places=6, db_index=True)), + ('deleted_at', self.gf('django.db.models.fields.DecimalField')(null=True, max_digits=20, decimal_places=6, db_index=True)), + ('owner', self.gf('django.db.models.fields.CharField')(max_length=50, db_index=True)), + ('size', self.gf('django.db.models.fields.BigIntegerField')(max_length=20)), + ('raw', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['stacktach.GlanceRawData'])), + )) + db.send_create_signal(u'stacktach', ['ImageDeletes']) + # Adding model 'GlanceRawData' db.create_table(u'stacktach_glancerawdata', ( (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), @@ -28,6 +40,17 @@ class Migration(SchemaMigration): )) db.send_create_signal(u'stacktach', ['GlanceRawData']) + # Adding model 'ImageUsage' + db.create_table(u'stacktach_imageusage', ( + (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('uuid', self.gf('django.db.models.fields.CharField')(max_length=50, db_index=True)), + ('created_at', self.gf('django.db.models.fields.DecimalField')(max_digits=20, decimal_places=6, db_index=True)), + ('owner', self.gf('django.db.models.fields.CharField')(max_length=50, db_index=True)), + ('size', self.gf('django.db.models.fields.BigIntegerField')(max_length=20)), + ('last_raw', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['stacktach.GlanceRawData'])), + )) + db.send_create_signal(u'stacktach', ['ImageUsage']) + # Adding model 'GenericRawData' db.create_table(u'stacktach_genericrawdata', ( (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), @@ -46,14 +69,42 @@ class Migration(SchemaMigration): )) db.send_create_signal(u'stacktach', ['GenericRawData']) + # Adding model 'ImageExists' + db.create_table(u'stacktach_imageexists', ( + (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('uuid', self.gf('django.db.models.fields.CharField')(max_length=50, db_index=True)), + ('created_at', self.gf('django.db.models.fields.DecimalField')(null=True, max_digits=20, decimal_places=6, db_index=True)), + ('deleted_at', self.gf('django.db.models.fields.DecimalField')(null=True, max_digits=20, decimal_places=6, db_index=True)), + ('audit_period_beginning', self.gf('django.db.models.fields.DecimalField')(max_digits=20, decimal_places=6, db_index=True)), + ('audit_period_ending', self.gf('django.db.models.fields.DecimalField')(max_digits=20, decimal_places=6, db_index=True)), + ('status', self.gf('django.db.models.fields.CharField')(default='pending', max_length=50, db_index=True)), + ('fail_reason', self.gf('django.db.models.fields.CharField')(max_length=300, null=True)), + ('raw', self.gf('django.db.models.fields.related.ForeignKey')(related_name='+', to=orm['stacktach.GlanceRawData'])), + ('usage', self.gf('django.db.models.fields.related.ForeignKey')(related_name='+', null=True, to=orm['stacktach.ImageUsage'])), + ('delete', self.gf('django.db.models.fields.related.ForeignKey')(related_name='+', null=True, to=orm['stacktach.ImageDeletes'])), + ('send_status', self.gf('django.db.models.fields.IntegerField')(default=0, db_index=True)), + ('owner', self.gf('django.db.models.fields.CharField')(max_length=255, db_index=True)), + ('size', self.gf('django.db.models.fields.BigIntegerField')(max_length=20)), + )) + db.send_create_signal(u'stacktach', ['ImageExists']) + def backwards(self, orm): + # Deleting model 'ImageDeletes' + db.delete_table(u'stacktach_imagedeletes') + # Deleting model 'GlanceRawData' db.delete_table(u'stacktach_glancerawdata') + # Deleting model 'ImageUsage' + db.delete_table(u'stacktach_imageusage') + # Deleting model 'GenericRawData' db.delete_table(u'stacktach_genericrawdata') + # Deleting model 'ImageExists' + db.delete_table(u'stacktach_imageexists') + models = { u'stacktach.deployment': { @@ -95,6 +146,42 @@ class Migration(SchemaMigration): 'uuid': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '36', 'null': 'True', 'blank': 'True'}), 'when': ('django.db.models.fields.DecimalField', [], {'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}) }, + u'stacktach.imagedeletes': { + 'Meta': {'object_name': 'ImageDeletes'}, + 'created_at': ('django.db.models.fields.DecimalField', [], {'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}), + 'deleted_at': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'owner': ('django.db.models.fields.CharField', [], {'max_length': '50', 'db_index': 'True'}), + 'raw': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['stacktach.GlanceRawData']"}), + 'size': ('django.db.models.fields.BigIntegerField', [], {'max_length': '20'}), + 'uuid': ('django.db.models.fields.CharField', [], {'max_length': '50', 'db_index': 'True'}) + }, + u'stacktach.imageexists': { + 'Meta': {'object_name': 'ImageExists'}, + 'audit_period_beginning': ('django.db.models.fields.DecimalField', [], {'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}), + 'audit_period_ending': ('django.db.models.fields.DecimalField', [], {'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}), + 'created_at': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}), + 'delete': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'null': 'True', 'to': u"orm['stacktach.ImageDeletes']"}), + 'deleted_at': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}), + 'fail_reason': ('django.db.models.fields.CharField', [], {'max_length': '300', 'null': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'owner': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}), + 'raw': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'to': u"orm['stacktach.GlanceRawData']"}), + 'send_status': ('django.db.models.fields.IntegerField', [], {'default': '0', 'db_index': 'True'}), + 'size': ('django.db.models.fields.BigIntegerField', [], {'max_length': '20'}), + 'status': ('django.db.models.fields.CharField', [], {'default': "'pending'", 'max_length': '50', 'db_index': 'True'}), + 'usage': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'null': 'True', 'to': u"orm['stacktach.ImageUsage']"}), + 'uuid': ('django.db.models.fields.CharField', [], {'max_length': '50', 'db_index': 'True'}) + }, + u'stacktach.imageusage': { + 'Meta': {'object_name': 'ImageUsage'}, + 'created_at': ('django.db.models.fields.DecimalField', [], {'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'last_raw': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['stacktach.GlanceRawData']"}), + 'owner': ('django.db.models.fields.CharField', [], {'max_length': '50', 'db_index': 'True'}), + 'size': ('django.db.models.fields.BigIntegerField', [], {'max_length': '20'}), + 'uuid': ('django.db.models.fields.CharField', [], {'max_length': '50', 'db_index': 'True'}) + }, u'stacktach.instancedeletes': { 'Meta': {'object_name': 'InstanceDeletes'}, 'deleted_at': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}), diff --git a/stacktach/models.py b/stacktach/models.py index 4876313..5119db3 100644 --- a/stacktach/models.py +++ b/stacktach/models.py @@ -316,7 +316,8 @@ class ImageDeletes(models.Model): created_at = models.DecimalField(max_digits=20, decimal_places=6, db_index=True) deleted_at = models.DecimalField(max_digits=20, - decimal_places=6, db_index=True) + decimal_places=6, db_index=True, + null=True) owner = models.CharField(max_length=50, db_index=True) size = models.BigIntegerField(max_length=20) raw = models.ForeignKey(GlanceRawData) @@ -336,9 +337,11 @@ class ImageExists(models.Model): uuid = models.CharField(max_length=50, db_index=True) created_at = models.DecimalField(max_digits=20, - decimal_places=6, db_index=True) + decimal_places=6, db_index=True, + null=True) deleted_at = models.DecimalField(max_digits=20, - decimal_places=6, db_index=True) + decimal_places=6, db_index=True, + null=True) audit_period_beginning = models.DecimalField(max_digits=20, decimal_places=6, db_index=True) @@ -347,10 +350,10 @@ class ImageExists(models.Model): status = models.CharField(max_length=50, db_index=True, choices=STATUS_CHOICES, default=PENDING) - fail_reason = models.CharField(max_length=300, db_index=True, null=True) + fail_reason = models.CharField(max_length=300, null=True) raw = models.ForeignKey(GlanceRawData, related_name='+') - usage = models.ForeignKey(ImageUsage, related_name='+') - delete = models.ForeignKey(ImageDeletes, related_name='+') + usage = models.ForeignKey(ImageUsage, related_name='+', null=True) + delete = models.ForeignKey(ImageDeletes, related_name='+', null=True) send_status = models.IntegerField(default=0, db_index=True) owner = models.CharField(max_length=255, db_index=True) size = models.BigIntegerField(max_length=20) diff --git a/stacktach/notification.py b/stacktach/notification.py index b69f76a..656fb5f 100644 --- a/stacktach/notification.py +++ b/stacktach/notification.py @@ -18,6 +18,7 @@ # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. from stacktach import utils +from stacktach import stacklog from stacktach import image_type from stacktach import db @@ -93,7 +94,17 @@ class GlanceNotification(Notification): self.image_type = image_type.get_numeric_code(self.payload) self.status = self.payload.get('status', None) self.uuid = self.payload.get('id', None) - + self.size = self.payload.get('size', None) + created_at = self.payload.get('created_at', None) + self.created_at = created_at and utils.str_time_to_unix(created_at) + audit_period_beginning = self.payload.get( + 'audit_period_beginning', None) + self.audit_period_beginning = audit_period_beginning and\ + utils.str_time_to_unix(audit_period_beginning) + audit_period_ending = self.payload.get( + 'audit_period_ending', None) + self.audit_period_ending = audit_period_ending and \ + utils.str_time_to_unix(audit_period_ending) @property def owner(self): @@ -102,39 +113,81 @@ class GlanceNotification(Notification): @property def instance(self): return self.properties.get('instance_uuid', None) + @property + def deleted_at(self): + deleted_at = self.body.get('deleted_at', None) + deleted_at = deleted_at or self.payload.get('deleted_at', None) + return deleted_at and utils.str_time_to_unix(deleted_at) def save(self): - db.create_glance_rawdata(deployment=self.deployment, - routing_key=self.routing_key, - owner=self.owner, - json=self.json, - when=self.when, - publisher=self.publisher, - event=self.event, - service=self.service, - host=self.host, - instance=self.instance, - request_id=self.request_id, - image_type=self.image_type, - status=self.status, - uuid=self.uuid) + return db.create_glance_rawdata(deployment=self.deployment, + routing_key=self.routing_key, + owner=self.owner, + json=self.json, + when=self.when, + publisher=self.publisher, + event=self.event, + service=self.service, + host=self.host, + instance=self.instance, + request_id=self.request_id, + image_type=self.image_type, + status=self.status, + uuid=self.uuid) + + def save_exists(self, raw): + if self.created_at: + values = { + 'uuid': self.uuid, + 'audit_period_beginning': self.audit_period_beginning, + 'audit_period_ending': self.audit_period_ending, + 'owner': self.owner, + 'size': self.size, + 'raw': raw + } + created_at_range = (self.created_at, self.created_at+1) + usage = db.get_image_usage( + uuid=self.uuid, created_at__range=created_at_range) + values['usage'] = usage + values['created_at'] = self.created_at + if self.deleted_at: + delete = db.get_image_delete( + uuid=self.uuid, created_at__range=created_at_range) + values['delete'] = delete + values['deleted_at'] = self.deleted_at + + db.create_image_exists(**values) + else: + stacklog.warn("Ignoring exists without created_at. GlanceRawData(%s)" + % raw.id) class NovaNotification(Notification): def __init__(self, body, deployment, routing_key, json): super(NovaNotification, self).__init__(body, deployment, routing_key, json) - self.state = self.payload.get('state', "") - self.old_state = self.payload.get('old_state', "") - self.old_task = self.payload.get('old_task_state', "") - self.task = self.payload.get('new_task_state', "") + self.state = self.payload.get('state', '') + self.old_state = self.payload.get('old_state', '') + self.old_task = self.payload.get('old_task_state', '') + self.task = self.payload.get('new_task_state', '') self.image_type = image_type.get_numeric_code(self.payload) image_meta = self.payload.get('image_meta', {}) - self.os_architecture = image_meta.get('org.openstack__1__architecture', - '') + self.os_architecture = \ + image_meta.get('org.openstack__1__architecture', '') self.os_distro = image_meta.get('org.openstack__1__os_distro', '') self.os_version = image_meta.get('org.openstack__1__os_version', '') self.rax_options = image_meta.get('com.rackspace__1__options', '') + self.instance_type_id = self.payload.get('instance_type_id', None) + self.new_instance_type_id = \ + self.payload.get('new_instance_type_id', None) + self.launched_at = self.payload.get('launched_at', None) + self.deleted_at = self.payload.get('deleted_at', None) + self.audit_period_beginning = self.payload.get( + 'audit_period_beginning', None) + self.audit_period_ending = self.payload.get( + 'audit_period_ending', None) + self.message = self.payload.get('message', None) + self.message_id = self.body.get('message_id', None) @property def host(self): diff --git a/stacktach/views.py b/stacktach/views.py index 386eee8..8e419bc 100644 --- a/stacktach/views.py +++ b/stacktach/views.py @@ -163,17 +163,16 @@ INSTANCE_EVENT = { } -def _process_usage_for_new_launch(raw, body): - payload = body['payload'] +def _process_usage_for_new_launch(raw, notification): values = {} - values['instance'] = payload['instance_id'] - values['request_id'] = body['_context_request_id'] + values['instance'] = notification.instance + values['request_id'] = notification.request_id (usage, new) = STACKDB.get_or_create_instance_usage(**values) if raw.event in [INSTANCE_EVENT['create_start'], INSTANCE_EVENT['rebuild_start']]: - usage.instance_type_id = payload['instance_type_id'] + usage.instance_type_id = notification.instance_type_id if raw.event in [INSTANCE_EVENT['rebuild_start'], INSTANCE_EVENT['resize_prep_start'], @@ -183,27 +182,23 @@ def _process_usage_for_new_launch(raw, body): # we will have a launch record corresponding to the exists. # We don't want to override a launched_at if it is already set # though, because we may have already received the end event - usage.launched_at = utils.str_time_to_unix(payload['launched_at']) + usage.launched_at = utils.str_time_to_unix(notification.launched_at) - usage.tenant = payload['tenant_id'] - image_meta = payload.get('image_meta', {}) - usage.rax_options = image_meta.get('com.rackspace__1__options', '') - usage.os_architecture = image_meta.get('org.openstack__1__architecture', - '') - usage.os_version = image_meta.get('org.openstack__1__os_version', '') - usage.os_distro = image_meta.get('org.openstack__1__os_distro', '') + usage.tenant = notification.tenant + usage.rax_options = notification.rax_options + usage.os_architecture = notification.os_architecture + usage.os_version = notification.os_version + usage.os_distro = notification.os_distro STACKDB.save(usage) -def _process_usage_for_updates(raw, body): - payload = body['payload'] - +def _process_usage_for_updates(raw, notification): if raw.event == INSTANCE_EVENT['create_end']: - if 'message' in payload and payload['message'] != 'Success': + if notification.message and notification.message != 'Success': return - instance_id = payload['instance_id'] - request_id = body['_context_request_id'] + instance_id = notification.instance + request_id = notification.request_id (usage, new) = STACKDB.get_or_create_instance_usage(instance=instance_id, request_id=request_id) @@ -211,28 +206,25 @@ def _process_usage_for_updates(raw, body): INSTANCE_EVENT['rebuild_end'], INSTANCE_EVENT['resize_finish_end'], INSTANCE_EVENT['resize_revert_end']]: - usage.launched_at = utils.str_time_to_unix(payload['launched_at']) + usage.launched_at = utils.str_time_to_unix(notification.launched_at) if raw.event == INSTANCE_EVENT['resize_revert_end']: - usage.instance_type_id = payload['instance_type_id'] + usage.instance_type_id = notification.instance_type_id elif raw.event == INSTANCE_EVENT['resize_prep_end']: - usage.instance_type_id = payload['new_instance_type_id'] + usage.instance_type_id = notification.new_instance_type_id - usage.tenant = payload['tenant_id'] - image_meta = payload.get('image_meta', {}) - usage.rax_options = image_meta.get('com.rackspace__1__options', '') - usage.os_architecture = image_meta.get('org.openstack__1__architecture', - '') - usage.os_version = image_meta.get('org.openstack__1__os_version', '') - usage.os_distro = image_meta.get('org.openstack__1__os_distro', '') + usage.tenant = notification.tenant + usage.rax_options = notification.rax_options + usage.os_architecture = notification.os_architecture + usage.os_version = notification.os_version + usage.os_distro = notification.os_distro STACKDB.save(usage) -def _process_delete(raw, body): - payload = body['payload'] - instance_id = payload['instance_id'] - deleted_at = utils.str_time_to_unix(payload['deleted_at']) +def _process_delete(raw, notification): + instance_id = notification.instance + deleted_at = utils.str_time_to_unix(notification.deleted_at) values = { 'instance': instance_id, 'deleted_at': deleted_at, @@ -240,7 +232,7 @@ def _process_delete(raw, body): (delete, new) = STACKDB.get_or_create_instance_delete(**values) delete.raw = raw - launched_at = payload.get('launched_at') + launched_at = notification.launched_at if launched_at and launched_at != '': launched_at = utils.str_time_to_unix(launched_at) delete.launched_at = launched_at @@ -248,35 +240,33 @@ def _process_delete(raw, body): STACKDB.save(delete) -def _process_exists(raw, body): - payload = body['payload'] - instance_id = payload['instance_id'] - launched_at_str = payload.get('launched_at') +def _process_exists(raw, notification): + instance_id = notification.instance + launched_at_str = notification.launched_at if launched_at_str is not None and launched_at_str != '': - launched_at = utils.str_time_to_unix(payload['launched_at']) + launched_at = utils.str_time_to_unix(notification.launched_at) launched_range = (launched_at, launched_at+1) usage = STACKDB.get_instance_usage(instance=instance_id, launched_at__range=launched_range) values = {} - values['message_id'] = body['message_id'] + values['message_id'] = notification.message_id values['instance'] = instance_id values['launched_at'] = launched_at - beginning = utils.str_time_to_unix(payload['audit_period_beginning']) + beginning = utils.str_time_to_unix(notification.audit_period_beginning) values['audit_period_beginning'] = beginning - ending = utils.str_time_to_unix(payload['audit_period_ending']) + ending = utils.str_time_to_unix(notification.audit_period_ending) values['audit_period_ending'] = ending - values['instance_type_id'] = payload['instance_type_id'] + values['instance_type_id'] = notification.instance_type_id if usage: values['usage'] = usage values['raw'] = raw - values['tenant'] = payload['tenant_id'] - image_meta = payload.get('image_meta', {}) - values['rax_options'] = image_meta.get('com.rackspace__1__options', '') - values['os_architecture'] = image_meta.get('org.openstack__1__architecture', '') - values['os_version'] = image_meta.get('org.openstack__1__os_version', '') - values['os_distro'] = image_meta.get('org.openstack__1__os_distro', '') + values['tenant'] = notification.tenant + values['rax_options'] = notification.rax_options + values['os_architecture'] = notification.os_architecture + values['os_version'] = notification.os_version + values['os_distro'] = notification.os_distro - deleted_at = payload.get('deleted_at') + deleted_at = notification.deleted_at if deleted_at and deleted_at != '': # We only want to pre-populate the 'delete' if we know this is in # fact an exist event for a deleted instance. Otherwise, there @@ -319,30 +309,7 @@ def _process_glance_delete(raw, notification): def _process_glance_exists(raw, notification): - if notification.created_at: - values = { - 'uuid': notification.uuid, - 'audit_period_beginning': notification.audit_period_beginning, - 'audit_period_ending': notification.audit_period_ending, - 'owner': notification.owner, - 'size': notification.size, - 'raw': raw, - } - created_at_range = (notification.created_at, notification.created_at+1) - usage = STACKDB.get_image_usage(uuid=notification.uuid, - created_at__range=created_at_range) - values['usage'] = usage - values['created_at'] = notification.created_at - if notification.deleted_at: - delete = STACKDB.get_image_delete(uuid=notification.uuid, - created_at__range=created_at_range) - values['delete'] = delete - values['deleted_at'] = notification.deleted_at - - STACKDB.create_image_exists(**values) - else: - stacklog.warn("Ignoring exists without created_at. GlanceRawData(%s)" - % raw.id) + notification.save_exists(raw) USAGE_PROCESS_MAPPING = { INSTANCE_EVENT['create_start']: _process_usage_for_new_launch, @@ -359,18 +326,18 @@ USAGE_PROCESS_MAPPING = { } GLANCE_USAGE_PROCESS_MAPPING = { - 'image.activate': _process_glance_usage, + 'image.upload': _process_glance_usage, 'image.delete': _process_glance_delete, 'image.exists': _process_glance_exists } -def aggregate_usage(raw, body): +def aggregate_usage(raw, notification): if not raw.instance: return if raw.event in USAGE_PROCESS_MAPPING: - USAGE_PROCESS_MAPPING[raw.event](raw, body) + USAGE_PROCESS_MAPPING[raw.event](raw, notification) def aggregate_glance_usage(raw, body): diff --git a/tests/unit/test_notification.py b/tests/unit/test_notification.py index b1dffe3..0347cbf 100644 --- a/tests/unit/test_notification.py +++ b/tests/unit/test_notification.py @@ -22,12 +22,16 @@ import unittest import mox -from stacktach import notification, utils +from stacktach import notification +from stacktach import utils from stacktach.notification import Notification +from stacktach.notification import NovaNotification from stacktach.notification import GlanceNotification from stacktach import db from tests.unit.utils import REQUEST_ID_1 +from tests.unit.utils import DECIMAL_DUMMY_TIME +from tests.unit.utils import DUMMY_TIME from tests.unit.utils import TIMESTAMP_1 from tests.unit.utils import TENANT_ID_1 from tests.unit.utils import INSTANCE_ID_1 @@ -85,6 +89,65 @@ class NovaNotificationTestCase(unittest.TestCase): 'unknown_exchange') self.mox.VerifyAll() + def test_save_should_persist_nova_rawdata_to_database(self): + body = { + "event_type": "compute.instance.exists", + '_context_request_id': REQUEST_ID_1, + '_context_project_id': TENANT_ID_1, + "timestamp": TIMESTAMP_1, + "publisher_id": "compute.global.preprod-ord.ohthree.com", + "payload": { + 'instance_id': INSTANCE_ID_1, + "status": "saving", + "container_format": "ovf", + "properties": { + "image_type": "snapshot", + }, + "tenant": "5877054", + "old_state": 'old_state', + "old_task_state": 'old_task', + "image_meta": { + "org.openstack__1__architecture": 'os_arch', + "org.openstack__1__os_distro": 'os_distro', + "org.openstack__1__os_version": 'os_version', + "com.rackspace__1__options": 'rax_opt', + }, + "state": 'state', + "new_task_state": 'task' + } + } + deployment = "1" + routing_key = "monitor.info" + json = '{["routing_key", {%s}]}' % body + raw = self.mox.CreateMockAnything() + self.mox.StubOutWithMock(db, 'create_nova_rawdata') + db.create_nova_rawdata( + deployment="1", + tenant=TENANT_ID_1, + json=json, + routing_key=routing_key, + when=utils.str_time_to_unix(TIMESTAMP_1), + publisher="compute.global.preprod-ord.ohthree.com", + event="compute.instance.exists", + service="compute", + host="global.preprod-ord.ohthree.com", + instance=INSTANCE_ID_1, + request_id=REQUEST_ID_1, + old_state='old_state', + old_task='old_task', + os_architecture='os_arch', + os_distro='os_distro', + os_version='os_version', + rax_options='rax_opt', + state='state', + task='task').AndReturn(raw) + + self.mox.ReplayAll() + + notification = NovaNotification(body, deployment, routing_key, json) + self.assertEquals(notification.save(), raw) + self.mox.VerifyAll() + class GlanceNotificationTestCase(unittest.TestCase): def setUp(self): @@ -111,6 +174,7 @@ class GlanceNotificationTestCase(unittest.TestCase): deployment = "1" routing_key = "glance_monitor.info" json = '{["routing_key", {%s}]}' % body + raw = self.mox.CreateMockAnything() self.mox.StubOutWithMock(db, 'create_glance_rawdata') db.create_glance_rawdata( deployment="1", @@ -126,13 +190,178 @@ class GlanceNotificationTestCase(unittest.TestCase): request_id=None, image_type=0, status="saving", - uuid="2df2ccf6-bc1b-4853-aab0-25fda346b3bb") + uuid="2df2ccf6-bc1b-4853-aab0-25fda346b3bb").AndReturn(raw) self.mox.ReplayAll() notification = GlanceNotification(body, deployment, routing_key, json) - notification.save() + self.assertEquals(notification.save(), raw) + self.mox.VerifyAll() + + def test_process_image_exists(self): + usage = self.mox.CreateMockAnything() + raw = self.mox.CreateMockAnything() + audit_period_beginning = "2013-05-20 17:31:57.939614" + audit_period_ending = "2013-06-20 17:31:57.939614" + size = 123 + uuid = "2df2ccf6-bc1b-4853-aab0-25fda346b3bb" + body = { + "event_type": "image.upload", + "timestamp": "2013-06-20 18:31:57.939614", + "publisher_id": "glance-api01-r2961.global.preprod-ord.ohthree.com", + "payload": { + "created_at": str(DUMMY_TIME), + "status": "saving", + "audit_period_beginning": audit_period_beginning, + "audit_period_ending": audit_period_ending, + "properties": { + "image_type": "snapshot", + "instance_uuid": INSTANCE_ID_1, + }, + "size": size, + "owner": TENANT_ID_1, + "id": uuid + } + } + deployment = "1" + routing_key = "glance_monitor.info" + json = '{["routing_key", {%s}]}' % body + + self.mox.StubOutWithMock(db, 'create_image_exists') + self.mox.StubOutWithMock(db, 'get_image_usage') + + created_at_range = (DECIMAL_DUMMY_TIME, DECIMAL_DUMMY_TIME+1) + db.get_image_usage(created_at__range=created_at_range, + uuid=uuid).AndReturn(None) + db.create_image_exists( + created_at=utils.str_time_to_unix(str(DUMMY_TIME)), + owner=TENANT_ID_1, + raw=raw, + audit_period_beginning=utils.str_time_to_unix(audit_period_beginning), + audit_period_ending=utils.str_time_to_unix(audit_period_ending), + size=size, + uuid=uuid, + usage=None).AndReturn(raw) + + self.mox.ReplayAll() + + notification = GlanceNotification(body, deployment, routing_key, + json) + notification.save_exists(raw) + self.mox.VerifyAll() + + def test_process_image_exists_with_delete_not_none(self): + raw = self.mox.CreateMockAnything() + usage = self.mox.CreateMockAnything() + delete = self.mox.CreateMockAnything() + audit_period_beginning = "2013-05-20 17:31:57.939614" + audit_period_ending = "2013-06-20 17:31:57.939614" + size = 123 + uuid = "2df2ccf6-bc1b-4853-aab0-25fda346b3bb" + deleted_at = "2013-06-20 14:31:57.939614" + body = { + "event_type": "image.upload", + "timestamp": "2013-06-20 18:31:57.939614", + "publisher_id": "glance-api01-r2961.global.preprod-ord.ohthree.com", + "payload": { + "created_at": str(DUMMY_TIME), + "status": "saving", + "audit_period_beginning": audit_period_beginning, + "audit_period_ending": audit_period_ending, + "properties": { + "image_type": "snapshot", + "instance_uuid": INSTANCE_ID_1, + }, + "deleted_at": deleted_at, + "size": size, + "owner": TENANT_ID_1, + "id": "2df2ccf6-bc1b-4853-aab0-25fda346b3bb", + } + } + deployment = "1" + routing_key = "glance_monitor.info" + json = '{["routing_key", {%s}]}' % body + + self.mox.StubOutWithMock(db, 'create_image_exists') + self.mox.StubOutWithMock(db, 'get_image_usage') + self.mox.StubOutWithMock(db, 'get_image_delete') + + created_at_range = (DECIMAL_DUMMY_TIME, DECIMAL_DUMMY_TIME+1) + db.get_image_usage(created_at__range=created_at_range, + uuid=uuid).AndReturn(None) + db.get_image_delete(created_at__range=created_at_range, + uuid=uuid).AndReturn(delete) + db.create_image_exists( + created_at=utils.str_time_to_unix(str(DUMMY_TIME)), + owner=TENANT_ID_1, + raw=raw, + audit_period_beginning=utils.str_time_to_unix(audit_period_beginning), + audit_period_ending=utils.str_time_to_unix(audit_period_ending), + size=size, + uuid=uuid, + usage=None, + delete=delete, + deleted_at=utils.str_time_to_unix(str(deleted_at))).AndReturn(raw) + + self.mox.ReplayAll() + + notification = GlanceNotification(body, deployment, routing_key, + json) + notification.save_exists(raw) + self.mox.VerifyAll() + + def test_process_image_exists_with_usage_not_none(self): + raw = self.mox.CreateMockAnything() + usage = self.mox.CreateMockAnything() + audit_period_beginning = "2013-05-20 17:31:57.939614" + audit_period_ending = "2013-06-20 17:31:57.939614" + size = 123 + uuid = "2df2ccf6-bc1b-4853-aab0-25fda346b3bb" + body = { + "event_type": "image.upload", + "timestamp": "2013-06-20 18:31:57.939614", + "publisher_id": "glance-api01-r2961.global.preprod-ord.ohthree.com", + "payload": { + "created_at": str(DUMMY_TIME), + "status": "saving", + "audit_period_beginning": audit_period_beginning, + "audit_period_ending": audit_period_ending, + "properties": { + "image_type": "snapshot", + "instance_uuid": INSTANCE_ID_1, + }, + "size": size, + "owner": TENANT_ID_1, + "id": "2df2ccf6-bc1b-4853-aab0-25fda346b3bb", + } + } + deployment = "1" + routing_key = "glance_monitor.info" + json = '{["routing_key", {%s}]}' % body + + self.mox.StubOutWithMock(db, 'create_image_exists') + self.mox.StubOutWithMock(db, 'get_image_usage') + self.mox.StubOutWithMock(db, 'get_image_delete') + + created_at_range = (DECIMAL_DUMMY_TIME, DECIMAL_DUMMY_TIME+1) + db.get_image_usage(created_at__range=created_at_range, + uuid=uuid).AndReturn(usage) + db.create_image_exists( + created_at=utils.str_time_to_unix(str(DUMMY_TIME)), + owner=TENANT_ID_1, + raw=raw, + audit_period_beginning=utils.str_time_to_unix(audit_period_beginning), + audit_period_ending=utils.str_time_to_unix(audit_period_ending), + size=size, + uuid=uuid, + usage=usage).AndReturn(raw) + + self.mox.ReplayAll() + + notification = GlanceNotification(body, deployment, routing_key, + json) + notification.save_exists(raw) self.mox.VerifyAll() @@ -163,6 +392,7 @@ class NotificationTestCase(unittest.TestCase): deployment = "1" routing_key = "generic_monitor.info" json = '{["routing_key", {%s}]}' % body + raw = self.mox.CreateMockAnything() self.mox.StubOutWithMock(db, 'create_generic_rawdata') db.create_generic_rawdata( deployment="1", @@ -175,10 +405,10 @@ class NotificationTestCase(unittest.TestCase): service="glance-api01-r2961", host="global.preprod-ord.ohthree.com", instance=INSTANCE_ID_1, - request_id=REQUEST_ID_1) + request_id=REQUEST_ID_1).AndReturn(raw) self.mox.ReplayAll() notification = Notification(body, deployment, routing_key, json) - notification.save() + self.assertEquals(notification.save(), raw) self.mox.VerifyAll() diff --git a/tests/unit/test_stacktach.py b/tests/unit/test_stacktach.py index 055d3d3..b493b65 100644 --- a/tests/unit/test_stacktach.py +++ b/tests/unit/test_stacktach.py @@ -26,7 +26,6 @@ import mox import utils from utils import INSTANCE_ID_1 -from utils import DECIMAL_DUMMY_TIME from utils import OS_VERSION_1 from utils import OS_ARCH_1 from utils import OS_DISTRO_1 @@ -305,15 +304,30 @@ class StacktachUsageParsingTestCase(unittest.TestCase): stacklog.get_logger(name=name).AndReturn(self.log) def test_process_usage_for_new_launch_create_start(self): - kwargs = {'launched': str(DUMMY_TIME), 'tenant_id': TENANT_ID_1, 'rax_options': RAX_OPTIONS_1, - 'os_architecture': OS_ARCH_1, 'os_version': OS_VERSION_1, 'os_distro': OS_DISTRO_1 } - notification = utils.create_nova_notif(request_id=REQUEST_ID_1, **kwargs) - event = 'compute.instance.create.start' - raw, usage = self._setup_process_usage_mocks(event, notification) + notification = self.mox.CreateMockAnything() + notification.launched_at = str(DUMMY_TIME) + notification.tenant = TENANT_ID_1 + notification.rax_options = RAX_OPTIONS_1 + notification.os_architecture = OS_ARCH_1 + notification.os_version = OS_VERSION_1 + notification.os_distro = OS_DISTRO_1 + notification.instance = INSTANCE_ID_1 + notification.request_id = REQUEST_ID_1 + notification.instance_type_id = INSTANCE_TYPE_ID_1 - views._process_usage_for_new_launch(raw, notification[1]) + raw = self.mox.CreateMockAnything() + raw.event = 'compute.instance.create.start' - self.assertEquals(usage.instance_type_id, '1') + usage = self.mox.CreateMockAnything() + views.STACKDB.get_or_create_instance_usage(instance=INSTANCE_ID_1, + request_id=REQUEST_ID_1) \ + .AndReturn((usage, True)) + views.STACKDB.save(usage) + self.mox.ReplayAll() + + views._process_usage_for_new_launch(raw, notification) + + self.assertEquals(usage.instance_type_id, INSTANCE_TYPE_ID_1) self.assertEquals(usage.tenant, TENANT_ID_1) self.assertEquals(usage.os_architecture, OS_ARCH_1) self.assertEquals(usage.os_version, OS_VERSION_1) @@ -323,15 +337,29 @@ class StacktachUsageParsingTestCase(unittest.TestCase): self.mox.VerifyAll() def test_process_usage_for_new_launch_rebuild_start(self): - kwargs = {'launched': str(DUMMY_TIME), 'tenant_id': TENANT_ID_1, 'rax_options': RAX_OPTIONS_1, - 'os_architecture': OS_ARCH_1, 'os_version': OS_VERSION_1, 'os_distro': OS_DISTRO_1 } - notification = utils.create_nova_notif(request_id=REQUEST_ID_1, **kwargs) - event = 'compute.instance.rebuild.start' - raw, usage = self._setup_process_usage_mocks(event, notification) + notification = self.mox.CreateMockAnything() + notification.launched_at = str(DUMMY_TIME) + notification.tenant = TENANT_ID_1 + notification.rax_options = RAX_OPTIONS_1 + notification.os_architecture = OS_ARCH_1 + notification.os_version = OS_VERSION_1 + notification.os_distro = OS_DISTRO_1 + notification.instance = INSTANCE_ID_1 + notification.request_id = REQUEST_ID_1 + notification.instance_type_id = INSTANCE_TYPE_ID_1 - views._process_usage_for_new_launch(raw, notification[1]) + raw = self.mox.CreateMockAnything() + raw.event = 'compute.instance.rebuild.start' + usage = self.mox.CreateMockAnything() + views.STACKDB.get_or_create_instance_usage(instance=INSTANCE_ID_1, + request_id=REQUEST_ID_1) \ + .AndReturn((usage, True)) + views.STACKDB.save(usage) + self.mox.ReplayAll() - self.assertEquals(usage.instance_type_id, '1') + views._process_usage_for_new_launch(raw, notification) + + self.assertEquals(usage.instance_type_id, INSTANCE_TYPE_ID_1) self.assertEquals(usage.tenant, TENANT_ID_1) self.assertEquals(usage.os_architecture, OS_ARCH_1) self.assertEquals(usage.os_version, OS_VERSION_1) @@ -340,14 +368,29 @@ class StacktachUsageParsingTestCase(unittest.TestCase): self.mox.VerifyAll() def test_process_usage_for_new_launch_rebuild_start_when_no_launched_at_in_db(self): - kwargs = {'launched': str(DUMMY_TIME), 'tenant_id': TENANT_ID_1, 'rax_options': RAX_OPTIONS_1, - 'os_architecture': OS_ARCH_1, 'os_version': OS_VERSION_1, 'os_distro': OS_DISTRO_1 } - notification = utils.create_nova_notif(request_id=REQUEST_ID_1, **kwargs) - event = 'compute.instance.rebuild.start' - raw, usage = self._setup_process_usage_mocks(event, notification) - usage.launched_at = None + notification = self.mox.CreateMockAnything() + notification.launched_at = str(DUMMY_TIME) + notification.tenant = TENANT_ID_1 + notification.rax_options = RAX_OPTIONS_1 + notification.os_architecture = OS_ARCH_1 + notification.os_version = OS_VERSION_1 + notification.os_distro = OS_DISTRO_1 + notification.instance = INSTANCE_ID_1 + notification.request_id = REQUEST_ID_1 + notification.instance_type_id = INSTANCE_TYPE_ID_1 - views._process_usage_for_new_launch(raw, notification[1]) + raw = self.mox.CreateMockAnything() + raw.event = 'compute.instance.rebuild.start' + + usage = self.mox.CreateMockAnything() + usage.launched_at = None + views.STACKDB.get_or_create_instance_usage(instance=INSTANCE_ID_1, + request_id=REQUEST_ID_1) \ + .AndReturn((usage, True)) + views.STACKDB.save(usage) + self.mox.ReplayAll() + + views._process_usage_for_new_launch(raw, notification) self.assertEqual(usage.launched_at, utils.decimal_utc(DUMMY_TIME)) self.assertEquals(usage.tenant, TENANT_ID_1) @@ -359,14 +402,31 @@ class StacktachUsageParsingTestCase(unittest.TestCase): self.mox.VerifyAll() def test_process_usage_for_new_launch_resize_prep_start_when_no_launched_at_in_db(self): - kwargs = {'launched': str(DUMMY_TIME), 'tenant_id': TENANT_ID_1, 'rax_options': RAX_OPTIONS_1, - 'os_architecture': OS_ARCH_1, 'os_version': OS_VERSION_1, 'os_distro': OS_DISTRO_1 } - notification = utils.create_nova_notif(request_id=REQUEST_ID_1, **kwargs) - event = 'compute.instance.resize.prep.start' - raw, usage = self._setup_process_usage_mocks(event, notification) + notification = self.mox.CreateMockAnything() + notification.launched_at = str(DUMMY_TIME) + notification.tenant = TENANT_ID_1 + notification.rax_options = RAX_OPTIONS_1 + notification.os_architecture = OS_ARCH_1 + notification.os_version = OS_VERSION_1 + notification.os_distro = OS_DISTRO_1 + notification.instance = INSTANCE_ID_1 + notification.request_id = REQUEST_ID_1 + notification.instance_type_id = INSTANCE_TYPE_ID_1 + + raw = self.mox.CreateMockAnything() + raw.event = 'compute.instance.resize.prep.start' + + usage = self.mox.CreateMockAnything() + usage.launched_at = None + views.STACKDB.get_or_create_instance_usage(instance=INSTANCE_ID_1, + request_id=REQUEST_ID_1) \ + .AndReturn((usage, True)) + views.STACKDB.save(usage) + self.mox.ReplayAll() + usage.launched_at = None - views._process_usage_for_new_launch(raw, notification[1]) + views._process_usage_for_new_launch(raw, notification) self.assertEqual(usage.launched_at, utils.decimal_utc(DUMMY_TIME)) self.assertEquals(usage.tenant, TENANT_ID_1) @@ -378,14 +438,29 @@ class StacktachUsageParsingTestCase(unittest.TestCase): self.mox.VerifyAll() def test_process_usage_for_new_launch_resize_revert_start_when_no_launched_at_in_db(self): - kwargs = {'launched': str(DUMMY_TIME), 'tenant_id': TENANT_ID_1,'rax_options': RAX_OPTIONS_1, - 'os_architecture': OS_ARCH_1, 'os_version': OS_VERSION_1, 'os_distro': OS_DISTRO_1 } - notification = utils.create_nova_notif(request_id=REQUEST_ID_1, **kwargs) - event = 'compute.instance.resize.revert.start' - raw, usage = self._setup_process_usage_mocks(event, notification) - usage.launched_at = None + notification = self.mox.CreateMockAnything() + notification.launched_at = str(DUMMY_TIME) + notification.tenant = TENANT_ID_1 + notification.rax_options = RAX_OPTIONS_1 + notification.os_architecture = OS_ARCH_1 + notification.os_version = OS_VERSION_1 + notification.os_distro = OS_DISTRO_1 + notification.instance = INSTANCE_ID_1 + notification.request_id = REQUEST_ID_1 + notification.instance_type_id = INSTANCE_TYPE_ID_1 - views._process_usage_for_new_launch(raw, notification[1]) + raw = self.mox.CreateMockAnything() + raw.event = 'compute.instance.resize.revert.start' + + usage = self.mox.CreateMockAnything() + usage.launched_at = None + views.STACKDB.get_or_create_instance_usage(instance=INSTANCE_ID_1, + request_id=REQUEST_ID_1) \ + .AndReturn((usage, True)) + views.STACKDB.save(usage) + self.mox.ReplayAll() + + views._process_usage_for_new_launch(raw, notification) self.assertEquals(usage.tenant, TENANT_ID_1) self.assertEqual(usage.launched_at, utils.decimal_utc(DUMMY_TIME)) @@ -397,17 +472,30 @@ class StacktachUsageParsingTestCase(unittest.TestCase): self.mox.VerifyAll() def test_process_usage_for_new_launch_resize_prep_start_when_launched_at_in_db(self): - kwargs = {'launched': str(DUMMY_TIME), 'tenant_id': TENANT_ID_1, - 'rax_options': RAX_OPTIONS_1, 'os_architecture': OS_ARCH_1, - 'os_version': OS_VERSION_1, 'os_distro': OS_DISTRO_1 } - notification = utils.create_nova_notif(request_id=REQUEST_ID_1, - **kwargs) - event = 'compute.instance.resize.prep.start' - raw, usage = self._setup_process_usage_mocks(event, notification) - orig_launched_at = utils.decimal_utc(DUMMY_TIME - datetime.timedelta(days=1)) - usage.launched_at = orig_launched_at + notification = self.mox.CreateMockAnything() + notification.launched_at = str(DUMMY_TIME) + notification.tenant = TENANT_ID_1 + notification.rax_options = RAX_OPTIONS_1 + notification.os_architecture = OS_ARCH_1 + notification.os_version = OS_VERSION_1 + notification.os_distro = OS_DISTRO_1 + notification.instance = INSTANCE_ID_1 + notification.request_id = REQUEST_ID_1 + notification.instance_type_id = INSTANCE_TYPE_ID_1 - views._process_usage_for_new_launch(raw, notification[1]) + raw = self.mox.CreateMockAnything() + raw.event = 'compute.instance.resize.prep.start' + + orig_launched_at = utils.decimal_utc(DUMMY_TIME - datetime.timedelta(days=1)) + usage = self.mox.CreateMockAnything() + usage.launched_at = orig_launched_at + views.STACKDB.get_or_create_instance_usage(instance=INSTANCE_ID_1, + request_id=REQUEST_ID_1) \ + .AndReturn((usage, True)) + views.STACKDB.save(usage) + self.mox.ReplayAll() + + views._process_usage_for_new_launch(raw, notification) self.assertEqual(usage.launched_at, orig_launched_at) self.assertEqual(usage.tenant, TENANT_ID_1) @@ -419,16 +507,30 @@ class StacktachUsageParsingTestCase(unittest.TestCase): self.mox.VerifyAll() def test_process_usage_for_updates_create_end(self): - kwargs = {'launched': str(DUMMY_TIME), - 'tenant_id': TENANT_ID_1, 'rax_options': RAX_OPTIONS_1, - 'os_architecture': OS_ARCH_1, 'os_version': OS_VERSION_1, - 'os_distro': OS_DISTRO_1 } - notification = utils.create_nova_notif(request_id=REQUEST_ID_1, - **kwargs) - event = 'compute.instance.create.end' - raw, usage = self._setup_process_usage_mocks(event, notification) + notification = self.mox.CreateMockAnything() + notification.launched_at = str(DUMMY_TIME) + notification.tenant = TENANT_ID_1 + notification.rax_options = RAX_OPTIONS_1 + notification.os_architecture = OS_ARCH_1 + notification.os_version = OS_VERSION_1 + notification.os_distro = OS_DISTRO_1 + notification.instance = INSTANCE_ID_1 + notification.request_id = REQUEST_ID_1 + notification.instance_type_id = INSTANCE_TYPE_ID_1 + notification.message = None - views._process_usage_for_updates(raw, notification[1]) + raw = self.mox.CreateMockAnything() + raw.event = 'compute.instance.create.end' + + usage = self.mox.CreateMockAnything() + usage.launched_at = None + views.STACKDB.get_or_create_instance_usage(instance=INSTANCE_ID_1, + request_id=REQUEST_ID_1) \ + .AndReturn((usage, True)) + views.STACKDB.save(usage) + self.mox.ReplayAll() + + views._process_usage_for_updates(raw, notification) self.assertEqual(usage.launched_at, utils.decimal_utc(DUMMY_TIME)) self.assertEqual(usage.tenant, TENANT_ID_1) @@ -440,17 +542,30 @@ class StacktachUsageParsingTestCase(unittest.TestCase): self.mox.VerifyAll() def test_process_usage_for_updates_create_end_success_message(self): - kwargs = {'launched': str(DUMMY_TIME), - 'tenant_id': TENANT_ID_1, 'rax_options': RAX_OPTIONS_1, - 'os_architecture': OS_ARCH_1, 'os_version': OS_VERSION_1, - 'os_distro': OS_DISTRO_1 } - notification = utils.create_nova_notif(request_id=REQUEST_ID_1, - **kwargs) - notification[1]['payload']['message'] = "Success" - event = 'compute.instance.create.end' - raw, usage = self._setup_process_usage_mocks(event, notification) + notification = self.mox.CreateMockAnything() + notification.launched_at = str(DUMMY_TIME) + notification.tenant = TENANT_ID_1 + notification.rax_options = RAX_OPTIONS_1 + notification.os_architecture = OS_ARCH_1 + notification.os_version = OS_VERSION_1 + notification.os_distro = OS_DISTRO_1 + notification.instance = INSTANCE_ID_1 + notification.request_id = REQUEST_ID_1 + notification.instance_type_id = INSTANCE_TYPE_ID_1 + notification.message = 'Success' - views._process_usage_for_updates(raw, notification[1]) + raw = self.mox.CreateMockAnything() + raw.event = 'compute.instance.create.end' + + usage = self.mox.CreateMockAnything() + usage.launched_at = None + views.STACKDB.get_or_create_instance_usage(instance=INSTANCE_ID_1, + request_id=REQUEST_ID_1) \ + .AndReturn((usage, True)) + views.STACKDB.save(usage) + self.mox.ReplayAll() + + views._process_usage_for_updates(raw, notification) self.assertEqual(usage.launched_at, utils.decimal_utc(DUMMY_TIME)) self.assertEqual(usage.tenant, TENANT_ID_1) @@ -462,37 +577,42 @@ class StacktachUsageParsingTestCase(unittest.TestCase): self.mox.VerifyAll() def test_process_usage_for_updates_create_end_error_message(self): - kwargs = {'launched': str(DUMMY_TIME), - 'tenant_id': TENANT_ID_1, 'rax_options': RAX_OPTIONS_1, - 'os_architecture': OS_ARCH_1, 'os_version': OS_VERSION_1, - 'os_distro': OS_DISTRO_1 } - notification = utils.create_nova_notif(request_id=REQUEST_ID_1, - **kwargs) - notification[1]['payload']['message'] = "Error" - event = 'compute.instance.create.end' - when_time = DUMMY_TIME - when_decimal = utils.decimal_utc(when_time) - json_str = json.dumps(notification) - raw = utils.create_raw(self.mox, when_decimal, event=event, - json_str=json_str) + notification = self.mox.CreateMockAnything() + notification.message = 'Error' + + raw = self.mox.CreateMockAnything() + raw.event = 'compute.instance.create.end' self.mox.ReplayAll() - views._process_usage_for_updates(raw, notification[1]) + views._process_usage_for_updates(raw, notification) self.mox.VerifyAll() def test_process_usage_for_updates_revert_end(self): - kwargs = {'launched': str(DUMMY_TIME), - 'type_id': INSTANCE_TYPE_ID_1, - 'tenant_id': TENANT_ID_1, 'rax_options': RAX_OPTIONS_1, - 'os_architecture': OS_ARCH_1, 'os_version': OS_VERSION_1, - 'os_distro': OS_DISTRO_1 } - notification = utils.create_nova_notif(request_id=REQUEST_ID_1, - **kwargs) - event = 'compute.instance.resize.revert.end' - raw, usage = self._setup_process_usage_mocks(event, notification) + notification = self.mox.CreateMockAnything() + notification.launched_at = str(DUMMY_TIME) + notification.tenant = TENANT_ID_1 + notification.rax_options = RAX_OPTIONS_1 + notification.os_architecture = OS_ARCH_1 + notification.os_version = OS_VERSION_1 + notification.os_distro = OS_DISTRO_1 + notification.instance = INSTANCE_ID_1 + notification.request_id = REQUEST_ID_1 + notification.instance_type_id = INSTANCE_TYPE_ID_1 + notification.message = None - views._process_usage_for_updates(raw, notification[1]) + raw = self.mox.CreateMockAnything() + raw.event = 'compute.instance.resize.revert.end' + + usage = self.mox.CreateMockAnything() + usage.launched_at = None + views.STACKDB.get_or_create_instance_usage(instance=INSTANCE_ID_1, + request_id=REQUEST_ID_1) \ + .AndReturn((usage, True)) + views.STACKDB.save(usage) + self.mox.ReplayAll() + + views._process_usage_for_updates(raw, notification) self.assertEqual(usage.instance_type_id, INSTANCE_TYPE_ID_1) self.assertEqual(usage.launched_at, utils.decimal_utc(DUMMY_TIME)) @@ -505,17 +625,30 @@ class StacktachUsageParsingTestCase(unittest.TestCase): self.mox.VerifyAll() def test_process_usage_for_updates_prep_end(self): - kwargs = {'launched': str(DUMMY_TIME), - 'new_type_id': INSTANCE_TYPE_ID_2, - 'tenant_id': TENANT_ID_1, 'rax_options': RAX_OPTIONS_1, - 'os_architecture': OS_ARCH_1, 'os_version': OS_VERSION_1, - 'os_distro': OS_DISTRO_1 } - notification = utils.create_nova_notif(request_id=REQUEST_ID_1, - **kwargs) - event = 'compute.instance.resize.prep.end' - raw, usage = self._setup_process_usage_mocks(event, notification) + notification = self.mox.CreateMockAnything() + notification.launched_at = str(DUMMY_TIME) + notification.tenant = TENANT_ID_1 + notification.rax_options = RAX_OPTIONS_1 + notification.os_architecture = OS_ARCH_1 + notification.os_version = OS_VERSION_1 + notification.os_distro = OS_DISTRO_1 + notification.instance = INSTANCE_ID_1 + notification.request_id = REQUEST_ID_1 + notification.new_instance_type_id = INSTANCE_TYPE_ID_2 + notification.message = None - views._process_usage_for_updates(raw, notification[1]) + raw = self.mox.CreateMockAnything() + raw.event = 'compute.instance.resize.prep.end' + + usage = self.mox.CreateMockAnything() + usage.launched_at = None + views.STACKDB.get_or_create_instance_usage(instance=INSTANCE_ID_1, + request_id=REQUEST_ID_1) \ + .AndReturn((usage, True)) + views.STACKDB.save(usage) + self.mox.ReplayAll() + + views._process_usage_for_updates(raw, notification) self.assertEqual(usage.instance_type_id, INSTANCE_TYPE_ID_2) self.assertEquals(usage.tenant, TENANT_ID_1) @@ -526,43 +659,29 @@ class StacktachUsageParsingTestCase(unittest.TestCase): self.mox.VerifyAll() - def _setup_process_usage_mocks(self, event, notification): - when_time = DUMMY_TIME - when_decimal = utils.decimal_utc(when_time) - json_str = json.dumps(notification) - raw = utils.create_raw(self.mox, when_decimal, event=event, - json_str=json_str) - usage = self.mox.CreateMockAnything() - views.STACKDB.get_or_create_instance_usage(instance=INSTANCE_ID_1, - request_id=REQUEST_ID_1) \ - .AndReturn((usage, True)) - views.STACKDB.save(usage) - self.mox.ReplayAll() - return raw, usage - def test_process_delete(self): delete_time = datetime.datetime.utcnow() launch_time = delete_time-datetime.timedelta(days=1) launch_decimal = utils.decimal_utc(launch_time) delete_decimal = utils.decimal_utc(delete_time) - notif = utils.create_nova_notif(request_id=REQUEST_ID_1, - launched=str(launch_time), - deleted=str(delete_time)) - json_str = json.dumps(notif) - event = 'compute.instance.delete.end' - raw = utils.create_raw(self.mox, delete_decimal, event=event, - json_str=json_str) + notification = self.mox.CreateMockAnything() + notification.instance = INSTANCE_ID_1 + notification.deleted_at = str(delete_time) + notification.launched_at = str(launch_time) + + raw = self.mox.CreateMockAnything() delete = self.mox.CreateMockAnything() delete.instance = INSTANCE_ID_1 delete.launched_at = launch_decimal delete.deleted_at = delete_decimal - views.STACKDB.get_or_create_instance_delete(instance=INSTANCE_ID_1, - deleted_at=delete_decimal)\ - .AndReturn((delete, True)) + views.STACKDB.get_or_create_instance_delete( + instance=INSTANCE_ID_1, deleted_at=delete_decimal)\ + .AndReturn((delete, True)) views.STACKDB.save(delete) self.mox.ReplayAll() - views._process_delete(raw, notif[1]) + views._process_delete(raw, notification) + self.assertEqual(delete.instance, INSTANCE_ID_1) self.assertEqual(delete.launched_at, launch_decimal) self.assertEqual(delete.deleted_at, delete_decimal) @@ -570,47 +689,50 @@ class StacktachUsageParsingTestCase(unittest.TestCase): def test_process_delete_no_launch(self): delete_time = datetime.datetime.utcnow() + launch_time = delete_time-datetime.timedelta(days=1) delete_decimal = utils.decimal_utc(delete_time) - notif = utils.create_nova_notif(request_id=REQUEST_ID_1, - deleted=str(delete_time)) - json_str = json.dumps(notif) - event = 'compute.instance.delete.end' - raw = utils.create_raw(self.mox, delete_decimal, event=event, - json_str=json_str) + notification = self.mox.CreateMockAnything() + notification.instance = INSTANCE_ID_1 + notification.deleted_at = str(delete_time) + notification.launched_at = str(launch_time) + + raw = self.mox.CreateMockAnything() delete = self.mox.CreateMockAnything() delete.instance = INSTANCE_ID_1 delete.deleted_at = delete_decimal - views.STACKDB.get_or_create_instance_delete(instance=INSTANCE_ID_1, - deleted_at=delete_decimal)\ - .AndReturn((delete, True)) + views.STACKDB.get_or_create_instance_delete( + instance=INSTANCE_ID_1, deleted_at=delete_decimal)\ + .AndReturn((delete, True)) views.STACKDB.save(delete) self.mox.ReplayAll() - views._process_delete(raw, notif[1]) + views._process_delete(raw, notification) + self.assertEqual(delete.instance, INSTANCE_ID_1) self.assertEqual(delete.deleted_at, delete_decimal) self.mox.VerifyAll() def test_process_exists(self): + notification = self.mox.CreateMockAnything() current_time = datetime.datetime.utcnow() launch_time = current_time - datetime.timedelta(hours=23) launch_decimal = utils.decimal_utc(launch_time) - current_decimal = utils.decimal_utc(current_time) audit_beginning = current_time - datetime.timedelta(hours=20) audit_beginning_decimal = utils.decimal_utc(audit_beginning) audit_ending_decimal = utils.decimal_utc(current_time) - notif = utils.create_nova_notif(launched=str(launch_time), - audit_period_beginning=str(audit_beginning), - audit_period_ending=str(current_time), - tenant_id=TENANT_ID_1, - os_architecture=OS_ARCH_1, - os_version=OS_VERSION_1, - os_distro=OS_DISTRO_1, - rax_options=RAX_OPTIONS_1) - json_str = json.dumps(notif) - event = 'compute.instance.exists' - raw = utils.create_raw(self.mox, current_decimal, event=event, - json_str=json_str) + notification.launched_at = str(launch_time) + notification.audit_period_beginning = str(audit_beginning) + notification.audit_period_ending = str(current_time) + notification.tenant = TENANT_ID_1 + notification.os_architecture = OS_ARCH_1 + notification.os_version = OS_VERSION_1 + notification.os_distro = OS_DISTRO_1 + notification.rax_options = RAX_OPTIONS_1 + notification.instance = INSTANCE_ID_1 + notification.deleted_at = '' + notification.instance_type_id = INSTANCE_TYPE_ID_1 + notification.message_id = MESSAGE_ID_1 + raw = self.mox.CreateMockAnything() usage = self.mox.CreateMockAnything() launched_range = (launch_decimal, launch_decimal+1) views.STACKDB.get_instance_usage(instance=INSTANCE_ID_1, @@ -622,7 +744,7 @@ class StacktachUsageParsingTestCase(unittest.TestCase): 'launched_at': launch_decimal, 'audit_period_beginning': audit_beginning_decimal, 'audit_period_ending': audit_ending_decimal, - 'instance_type_id': '1', + 'instance_type_id': INSTANCE_TYPE_ID_1, 'usage': usage, 'raw': raw, 'tenant': TENANT_ID_1, @@ -635,50 +757,45 @@ class StacktachUsageParsingTestCase(unittest.TestCase): views.STACKDB.create_instance_exists(**exists_values).AndReturn(exists) views.STACKDB.save(exists) self.mox.ReplayAll() - views._process_exists(raw, notif[1]) + views._process_exists(raw, notification) self.mox.VerifyAll() def test_process_exists_no_launched_at(self): - current_time = datetime.datetime.utcnow() - current_decimal = utils.decimal_utc(current_time) - audit_beginning = current_time - datetime.timedelta(hours=20) - notif = utils.create_nova_notif(audit_period_beginning=str(audit_beginning), - audit_period_ending=str(current_time), - tenant_id=TENANT_ID_1) - json_str = json.dumps(notif) - event = 'compute.instance.exists' - raw = utils.create_raw(self.mox, current_decimal, event=event, - json_str=json_str) - raw.id = 1 + notification = self.mox.CreateMockAnything() + notification.instance = INSTANCE_ID_1 + notification.launched_at = None + raw = self.mox.CreateMockAnything() + raw.id = '1' self.setup_mock_log() self.log.warn('Ignoring exists without launched_at. RawData(1)') self.mox.ReplayAll() - views._process_exists(raw, notif[1]) + views._process_exists(raw, notification) self.mox.VerifyAll() def test_process_exists_with_deleted_at(self): + notification = self.mox.CreateMockAnything() current_time = datetime.datetime.utcnow() launch_time = current_time - datetime.timedelta(hours=23) launch_decimal = utils.decimal_utc(launch_time) - deleted_time = current_time - datetime.timedelta(hours=12) - deleted_decimal = utils.decimal_utc(deleted_time) - current_decimal = utils.decimal_utc(current_time) + delete_time = datetime.datetime.utcnow() + deleted_decimal = utils.decimal_utc(delete_time) audit_beginning = current_time - datetime.timedelta(hours=20) audit_beginning_decimal = utils.decimal_utc(audit_beginning) audit_ending_decimal = utils.decimal_utc(current_time) - notif = utils.create_nova_notif(launched=str(launch_time), - deleted=str(deleted_time), - audit_period_beginning=str(audit_beginning), - audit_period_ending=str(current_time), - tenant_id=TENANT_ID_1, - os_architecture=OS_ARCH_1, - os_version=OS_VERSION_1, - os_distro=OS_DISTRO_1, - rax_options=RAX_OPTIONS_1) - json_str = json.dumps(notif) - event = 'compute.instance.exists' - raw = utils.create_raw(self.mox, current_decimal, event=event, - json_str=json_str) + + notification.launched_at = str(launch_time) + notification.audit_period_beginning = str(audit_beginning) + notification.audit_period_ending = str(current_time) + notification.tenant = TENANT_ID_1 + notification.os_architecture = OS_ARCH_1 + notification.os_version = OS_VERSION_1 + notification.os_distro = OS_DISTRO_1 + notification.rax_options = RAX_OPTIONS_1 + notification.instance = INSTANCE_ID_1 + notification.instance_type_id = INSTANCE_TYPE_ID_1 + notification.message_id = MESSAGE_ID_1 + notification.deleted_at = str(delete_time) + raw = self.mox.CreateMockAnything() usage = self.mox.CreateMockAnything() launched_range = (launch_decimal, launch_decimal+1) views.STACKDB.get_instance_usage(instance=INSTANCE_ID_1, @@ -695,7 +812,7 @@ class StacktachUsageParsingTestCase(unittest.TestCase): 'deleted_at': deleted_decimal, 'audit_period_beginning': audit_beginning_decimal, 'audit_period_ending': audit_ending_decimal, - 'instance_type_id': '1', + 'instance_type_id': INSTANCE_TYPE_ID_1, 'usage': usage, 'delete': delete, 'raw': raw, @@ -709,7 +826,7 @@ class StacktachUsageParsingTestCase(unittest.TestCase): views.STACKDB.create_instance_exists(**exists_values).AndReturn(exists) views.STACKDB.save(exists) self.mox.ReplayAll() - views._process_exists(raw, notif[1]) + views._process_exists(raw, notification) self.mox.VerifyAll() @@ -766,114 +883,9 @@ class StacktachImageUsageParsingTestCase(unittest.TestCase): def test_process_image_exists(self): raw = self.mox.CreateMockAnything() - values = { - 'uuid': IMAGE_UUID_1, - 'created_at': DECIMAL_DUMMY_TIME, - 'deleted_at': DECIMAL_DUMMY_TIME, - 'audit_period_beginning': DECIMAL_DUMMY_TIME, - 'audit_period_ending': DECIMAL_DUMMY_TIME, - 'owner': TENANT_ID_1, - 'size': 1234, - 'raw': raw, - 'usage': None, - 'delete': None - } - notification = self.mox.CreateMockAnything() - notification.created_at = values['created_at'] - notification.deleted_at = values['deleted_at'] - notification.owner = values['owner'] - notification.uuid = values['uuid'] - notification.size = values['size'] - notification.raw = values['raw'] - notification.audit_period_beginning = values['audit_period_beginning'] - notification.audit_period_ending = values['audit_period_ending'] - created_at_range = (DECIMAL_DUMMY_TIME, DECIMAL_DUMMY_TIME+1) - views.STACKDB.get_image_usage(created_at__range=created_at_range, - uuid=notification.uuid).AndReturn(None) - views.STACKDB.get_image_delete(created_at__range=created_at_range, - uuid=notification.uuid).AndReturn(None) - views.STACKDB.create_image_exists(**values) - + notification.save_exists(raw) self.mox.ReplayAll() views._process_glance_exists(raw, notification) self.mox.VerifyAll() - - def test_process_image_exists_with_usage_not_none(self): - raw = self.mox.CreateMockAnything() - usage = self.mox.CreateMockAnything() - values = { - 'uuid': IMAGE_UUID_1, - 'created_at': DECIMAL_DUMMY_TIME, - 'deleted_at': DECIMAL_DUMMY_TIME, - 'audit_period_beginning': DECIMAL_DUMMY_TIME, - 'audit_period_ending': DECIMAL_DUMMY_TIME, - 'owner': TENANT_ID_1, - 'size': 1234, - 'raw': raw, - 'usage': usage, - 'delete': None - } - - notification = self.mox.CreateMockAnything() - notification.created_at = values['created_at'] - notification.deleted_at = values['deleted_at'] - notification.owner = values['owner'] - notification.uuid = values['uuid'] - notification.size = values['size'] - notification.raw = values['raw'] - notification.audit_period_beginning = values['audit_period_beginning'] - notification.audit_period_ending = values['audit_period_ending'] - created_at_range = (DECIMAL_DUMMY_TIME, DECIMAL_DUMMY_TIME+1) - views.STACKDB.get_image_usage(created_at__range=created_at_range, - uuid=notification.uuid).AndReturn(usage) - views.STACKDB.get_image_delete(created_at__range=created_at_range, - uuid=notification.uuid).AndReturn(None) - views.STACKDB.create_image_exists(**values) - - self.mox.ReplayAll() - - views._process_glance_exists(raw, notification) - self.mox.VerifyAll() - - - def test_process_image_exists_with_delete_not_none(self): - raw = self.mox.CreateMockAnything() - delete = self.mox.CreateMockAnything() - values = { - 'uuid': IMAGE_UUID_1, - 'created_at': DECIMAL_DUMMY_TIME, - 'deleted_at': DECIMAL_DUMMY_TIME, - 'audit_period_beginning': DECIMAL_DUMMY_TIME, - 'audit_period_ending': DECIMAL_DUMMY_TIME, - 'owner': TENANT_ID_1, - 'size': 1234, - 'raw': raw, - 'usage': None, - 'delete': delete - } - - notification = self.mox.CreateMockAnything() - notification.created_at = values['created_at'] - notification.deleted_at = values['deleted_at'] - notification.owner = values['owner'] - notification.uuid = values['uuid'] - notification.size = values['size'] - notification.raw = values['raw'] - notification.audit_period_beginning = values['audit_period_beginning'] - notification.audit_period_ending = values['audit_period_ending'] - created_at_range = (DECIMAL_DUMMY_TIME, DECIMAL_DUMMY_TIME+1) - views.STACKDB.get_image_usage(created_at__range=created_at_range, - uuid=notification.uuid).AndReturn(None) - views.STACKDB.get_image_delete(created_at__range=created_at_range, - uuid=notification.uuid).AndReturn(delete) - views.STACKDB.create_image_exists(**values) - - self.mox.ReplayAll() - - views._process_glance_exists(raw, notification) - self.mox.VerifyAll() - - # def test_process_image_exists_should_not_populate_delete_and_deleted_at_when_deleted_at_is_absent(self): - diff --git a/worker/worker.py b/worker/worker.py index e542b9c..615b693 100644 --- a/worker/worker.py +++ b/worker/worker.py @@ -34,9 +34,7 @@ except ImportError: from pympler.process import ProcessMemoryInfo -from django import db as django_db from stacktach import db -from stacktach import notification from stacktach import stacklog from stacktach import views