From 3f6542f049f7046bb3c10015221a8c4266836e96 Mon Sep 17 00:00:00 2001 From: Manali Latkar Date: Mon, 24 Jun 2013 08:52:04 +0530 Subject: [PATCH 01/11] - Added logic to populate generic rawdata and glance rawdata - Moved the responsibilty to save rawdata to the notification classes - Notification are now created based on exchange instead of routing_key since routing_keys may not be unique across services - Separate consumers are now created for every exchange specified in the config - Each consumer is started in a separate process - Introduced notification factory and the config module --- etc/sample_stacktach_worker_config.json | 12 +- settings.py | 2 + stacktach/db.py | 19 +- ...eexists_and_instanceusages_from_rawdata.py | 8 +- ...create_glancerawdata_and_genericrawdata.py | 211 +++++++++++++ stacktach/models.py | 92 +++++- stacktach/notification.py | 176 ++++++++--- stacktach/tests.py | 91 +++++- stacktach/views.py | 38 +-- tests/unit/test_models.py | 33 ++ tests/unit/test_notification.py | 298 +++++++++--------- tests/unit/test_stacktach.py | 56 ++-- tests/unit/test_worker.py | 82 +++-- tests/unit/utils.py | 1 + worker/config.py | 43 +++ worker/start_workers.py | 27 +- worker/worker.py | 62 ++-- 17 files changed, 899 insertions(+), 352 deletions(-) create mode 100644 stacktach/migrations/0004_create_glancerawdata_and_genericrawdata.py create mode 100644 tests/unit/test_models.py create mode 100644 worker/config.py diff --git a/etc/sample_stacktach_worker_config.json b/etc/sample_stacktach_worker_config.json index e33ad9e..f15892e 100644 --- a/etc/sample_stacktach_worker_config.json +++ b/etc/sample_stacktach_worker_config.json @@ -7,7 +7,11 @@ "rabbit_userid": "rabbit", "rabbit_password": "rabbit", "rabbit_virtual_host": "/", - "exit_on_exception": true + "exit_on_exception": true, + "topics": { + "nova": ["monitor.info", "monitor.error"], + "glance": ["monitor_glance.info", "monitor_glance.error"] + } }, { "name": "east_coast.prod.cell1", @@ -17,6 +21,10 @@ "rabbit_userid": "rabbit", "rabbit_password": "rabbit", "rabbit_virtual_host": "/", - "exit_on_exception": false + "exit_on_exception": false, + "topics": { + "nova": ["monitor.info", "monitor.error"], + "glance": ["monitor_glance.info", "monitor_glance.error"] + } }] } diff --git a/settings.py b/settings.py index ea859b1..c9106be 100644 --- a/settings.py +++ b/settings.py @@ -140,6 +140,8 @@ INSTALLED_APPS = ( 'south' ) +SOUTH_TESTS_MIGRATE = False + ALLOWED_HOSTS = ['*'] # A sample logging configuration. The only tangible logging diff --git a/stacktach/db.py b/stacktach/db.py index ae3672d..0043ec7 100644 --- a/stacktach/db.py +++ b/stacktach/db.py @@ -20,7 +20,7 @@ def get_or_create_deployment(name): return models.Deployment.objects.get_or_create(name=name) -def create_rawdata(**kwargs): +def create_nova_rawdata(**kwargs): imagemeta_fields = ['os_architecture', 'os_version', 'os_distro', 'rax_options'] imagemeta_kwargs = \ @@ -35,6 +35,7 @@ def create_rawdata(**kwargs): return rawdata + def create_lifecycle(**kwargs): return models.Lifecycle(**kwargs) @@ -88,4 +89,18 @@ def create_instance_exists(**kwargs): def save(obj): - obj.save() \ No newline at end of file + obj.save() + + +def create_glance_rawdata(**kwargs): + rawdata = models.GlanceRawData(**kwargs) + rawdata.save() + + return rawdata + + +def create_generic_rawdata(**kwargs): + rawdata = models.GenericRawData(**kwargs) + rawdata.save() + + return rawdata \ No newline at end of file diff --git a/stacktach/migrations/0003_populate_usage_related_fields_in_rawdataimagemeta_instanceexists_and_instanceusages_from_rawdata.py b/stacktach/migrations/0003_populate_usage_related_fields_in_rawdataimagemeta_instanceexists_and_instanceusages_from_rawdata.py index 67edfa4..86ae331 100644 --- a/stacktach/migrations/0003_populate_usage_related_fields_in_rawdataimagemeta_instanceexists_and_instanceusages_from_rawdata.py +++ b/stacktach/migrations/0003_populate_usage_related_fields_in_rawdataimagemeta_instanceexists_and_instanceusages_from_rawdata.py @@ -1,8 +1,7 @@ # -*- coding: utf-8 -*- import copy from south.v2 import DataMigration -from stacktach.notification import Notification -from stacktach.views import NOTIFICATIONS +from stacktach.notification import notification_factory try: import ujson as json @@ -43,8 +42,7 @@ class Migration(DataMigration): json_dict = json.loads(json_message) routing_key = json_dict[0] body = json_dict[1] - notification = NOTIFICATIONS[routing_key](body) - return notification + return notification_factory(body, None, routing_key, json_message, None) def forwards(self, orm): # Note: Don't use "from appname.models import ModelName". @@ -79,7 +77,7 @@ class Migration(DataMigration): exists_update_count += 1 print "Updated %s records in InstanceExists" % exists_update_count - print "\nStarted updating records in InstacnceUsages" + print "\nStarted updating records in InstanceUsages" usages = orm.InstanceUsage.objects.all().values('request_id') usages_update_count = 0 for usage in usages: diff --git a/stacktach/migrations/0004_create_glancerawdata_and_genericrawdata.py b/stacktach/migrations/0004_create_glancerawdata_and_genericrawdata.py new file mode 100644 index 0000000..277c5ca --- /dev/null +++ b/stacktach/migrations/0004_create_glancerawdata_and_genericrawdata.py @@ -0,0 +1,211 @@ +# -*- coding: utf-8 -*- +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + # Adding model 'GlanceRawData' + db.create_table(u'stacktach_glancerawdata', ( + (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('deployment', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['stacktach.Deployment'])), + ('owner', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=50, null=True, blank=True)), + ('json', self.gf('django.db.models.fields.TextField')()), + ('routing_key', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=50, null=True, blank=True)), + ('when', self.gf('django.db.models.fields.DecimalField')(max_digits=20, decimal_places=6, db_index=True)), + ('publisher', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=100, null=True, blank=True)), + ('event', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=50, null=True, blank=True)), + ('service', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=50, null=True, blank=True)), + ('host', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=100, null=True, blank=True)), + ('instance', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=50, null=True, blank=True)), + ('request_id', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=50, null=True, blank=True)), + ('uuid', self.gf('django.db.models.fields.CharField')(max_length=50)), + ('status', self.gf('django.db.models.fields.CharField')(default='queued', max_length=50, db_index=True)), + ('image_type', self.gf('django.db.models.fields.IntegerField')(default=0, null=True, db_index=True)), + )) + db.send_create_signal(u'stacktach', ['GlanceRawData']) + + # Adding model 'GenericRawData' + db.create_table(u'stacktach_genericrawdata', ( + (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('deployment', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['stacktach.Deployment'])), + ('tenant', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=50, null=True, blank=True)), + ('json', self.gf('django.db.models.fields.TextField')()), + ('routing_key', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=50, null=True, blank=True)), + ('image_type', self.gf('django.db.models.fields.IntegerField')(default=0, null=True, db_index=True)), + ('when', self.gf('django.db.models.fields.DecimalField')(max_digits=20, decimal_places=6, db_index=True)), + ('publisher', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=100, null=True, blank=True)), + ('event', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=50, null=True, blank=True)), + ('service', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=50, null=True, blank=True)), + ('host', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=100, null=True, blank=True)), + ('instance', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=50, null=True, blank=True)), + ('request_id', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=50, null=True, blank=True)), + )) + db.send_create_signal(u'stacktach', ['GenericRawData']) + + + def backwards(self, orm): + # Deleting model 'GlanceRawData' + db.delete_table(u'stacktach_glancerawdata') + + # Deleting model 'GenericRawData' + db.delete_table(u'stacktach_genericrawdata') + + + models = { + u'stacktach.deployment': { + 'Meta': {'object_name': 'Deployment'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + u'stacktach.genericrawdata': { + 'Meta': {'object_name': 'GenericRawData'}, + 'deployment': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['stacktach.Deployment']"}), + 'event': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'host': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '100', 'null': 'True', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'image_type': ('django.db.models.fields.IntegerField', [], {'default': '0', 'null': 'True', 'db_index': 'True'}), + 'instance': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'json': ('django.db.models.fields.TextField', [], {}), + 'publisher': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '100', 'null': 'True', 'blank': 'True'}), + 'request_id': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'routing_key': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'service': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'tenant': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'when': ('django.db.models.fields.DecimalField', [], {'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}) + }, + u'stacktach.glancerawdata': { + 'Meta': {'object_name': 'GlanceRawData'}, + 'deployment': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['stacktach.Deployment']"}), + 'event': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'host': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '100', 'null': 'True', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'image_type': ('django.db.models.fields.IntegerField', [], {'default': '0', 'null': 'True', 'db_index': 'True'}), + 'instance': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'json': ('django.db.models.fields.TextField', [], {}), + 'owner': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'publisher': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '100', 'null': 'True', 'blank': 'True'}), + 'request_id': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'routing_key': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'service': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'status': ('django.db.models.fields.CharField', [], {'default': "'queued'", 'max_length': '50', 'db_index': 'True'}), + 'uuid': ('django.db.models.fields.CharField', [], {'max_length': '50'}), + 'when': ('django.db.models.fields.DecimalField', [], {'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}) + }, + u'stacktach.instancedeletes': { + 'Meta': {'object_name': 'InstanceDeletes'}, + 'deleted_at': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'instance': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'launched_at': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}), + 'raw': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['stacktach.RawData']", 'null': 'True'}) + }, + u'stacktach.instanceexists': { + 'Meta': {'object_name': 'InstanceExists'}, + 'audit_period_beginning': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}), + 'audit_period_ending': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}), + 'delete': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'null': 'True', 'to': u"orm['stacktach.InstanceDeletes']"}), + 'deleted_at': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}), + 'fail_reason': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '300', 'null': 'True', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'instance': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'instance_type_id': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'launched_at': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}), + 'message_id': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'os_architecture': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'os_distro': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'os_version': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'raw': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'null': 'True', 'to': u"orm['stacktach.RawData']"}), + 'rax_options': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'send_status': ('django.db.models.fields.IntegerField', [], {'default': '0', 'null': 'True', 'db_index': 'True'}), + 'status': ('django.db.models.fields.CharField', [], {'default': "'pending'", 'max_length': '50', 'db_index': 'True'}), + 'tenant': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'usage': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'null': 'True', 'to': u"orm['stacktach.InstanceUsage']"}) + }, + u'stacktach.instanceusage': { + 'Meta': {'object_name': 'InstanceUsage'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'instance': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'instance_type_id': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'launched_at': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}), + 'os_architecture': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'os_distro': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'os_version': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'rax_options': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'request_id': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'tenant': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}) + }, + u'stacktach.jsonreport': { + 'Meta': {'object_name': 'JsonReport'}, + 'created': ('django.db.models.fields.DecimalField', [], {'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'json': ('django.db.models.fields.TextField', [], {}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50', 'db_index': 'True'}), + 'period_end': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'period_start': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'version': ('django.db.models.fields.IntegerField', [], {'default': '1'}) + }, + u'stacktach.lifecycle': { + 'Meta': {'object_name': 'Lifecycle'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'instance': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'last_raw': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['stacktach.RawData']", 'null': 'True'}), + 'last_state': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'last_task_state': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}) + }, + u'stacktach.rawdata': { + 'Meta': {'object_name': 'RawData'}, + 'deployment': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['stacktach.Deployment']"}), + 'event': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'host': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '100', 'null': 'True', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'image_type': ('django.db.models.fields.IntegerField', [], {'default': '0', 'null': 'True', 'db_index': 'True'}), + 'instance': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'json': ('django.db.models.fields.TextField', [], {}), + 'old_state': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '20', 'null': 'True', 'blank': 'True'}), + 'old_task': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '30', 'null': 'True', 'blank': 'True'}), + 'publisher': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '100', 'null': 'True', 'blank': 'True'}), + 'request_id': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'routing_key': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'service': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'state': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '20', 'null': 'True', 'blank': 'True'}), + 'task': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '30', 'null': 'True', 'blank': 'True'}), + 'tenant': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'when': ('django.db.models.fields.DecimalField', [], {'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}) + }, + u'stacktach.rawdataimagemeta': { + 'Meta': {'object_name': 'RawDataImageMeta'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'os_architecture': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'os_distro': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'os_version': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'raw': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['stacktach.RawData']"}), + 'rax_options': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}) + }, + u'stacktach.requesttracker': { + 'Meta': {'object_name': 'RequestTracker'}, + 'completed': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}), + 'duration': ('django.db.models.fields.DecimalField', [], {'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'last_timing': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['stacktach.Timing']", 'null': 'True'}), + 'lifecycle': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['stacktach.Lifecycle']"}), + 'request_id': ('django.db.models.fields.CharField', [], {'max_length': '50', 'db_index': 'True'}), + 'start': ('django.db.models.fields.DecimalField', [], {'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}) + }, + u'stacktach.timing': { + 'Meta': {'object_name': 'Timing'}, + 'diff': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}), + 'end_raw': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'null': 'True', 'to': u"orm['stacktach.RawData']"}), + 'end_when': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'lifecycle': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['stacktach.Lifecycle']"}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50', 'db_index': 'True'}), + 'start_raw': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'null': 'True', 'to': u"orm['stacktach.RawData']"}), + 'start_when': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6'}) + } + } + + complete_apps = ['stacktach'] \ No newline at end of file diff --git a/stacktach/models.py b/stacktach/models.py index f76fed9..6980ee4 100644 --- a/stacktach/models.py +++ b/stacktach/models.py @@ -1,18 +1,3 @@ -# Copyright 2012 - Dark Secret Software Inc. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - from django import forms from django.db import models @@ -24,6 +9,34 @@ class Deployment(models.Model): return self.name +class GenericRawData(models.Model): + deployment = models.ForeignKey(Deployment) + tenant = models.CharField(max_length=50, null=True, blank=True, + db_index=True) + json = models.TextField() + routing_key = models.CharField(max_length=50, null=True, + blank=True, db_index=True) + image_type = models.IntegerField(null=True, default=0, db_index=True) + when = models.DecimalField(max_digits=20, decimal_places=6, + db_index=True) + publisher = models.CharField(max_length=100, null=True, + blank=True, db_index=True) + event = models.CharField(max_length=50, null=True, + blank=True, db_index=True) + service = models.CharField(max_length=50, null=True, + blank=True, db_index=True) + host = models.CharField(max_length=100, null=True, + blank=True, db_index=True) + instance = models.CharField(max_length=50, null=True, + blank=True, db_index=True) + request_id = models.CharField(max_length=50, null=True, + blank=True, db_index=True) + + @staticmethod + def get_name(): + return GenericRawData.__name__ + + class RawData(models.Model): deployment = models.ForeignKey(Deployment) tenant = models.CharField(max_length=50, null=True, blank=True, @@ -58,6 +71,10 @@ class RawData(models.Model): def __repr__(self): return "%s %s %s" % (self.event, self.instance, self.state) + @staticmethod + def get_name(): + return RawData.__name__ + class RawDataImageMeta(models.Model): raw = models.ForeignKey(RawData, null=False) @@ -158,6 +175,7 @@ class InstanceExists(models.Model): (RECONCILED, 'Passed Verification After Reconciliation'), (FAILED, 'Failed Verification'), ] + instance = models.CharField(max_length=50, null=True, blank=True, db_index=True) launched_at = models.DecimalField(null=True, max_digits=20, @@ -238,5 +256,49 @@ class JsonReport(models.Model): json = models.TextField() +class GlanceRawData(models.Model): + ACTIVE = 'active' + DELETED = 'deleted' + KILLED = 'killed' + PENDING_DELETE = 'pending_delete' + QUEUED = 'queued' + SAVING = 'saving' + STATUS_CHOICES = [ + (ACTIVE, 'Active'), + (DELETED, 'Deleted'), + (KILLED, 'Killed'), + (PENDING_DELETE, 'Pending delete'), + (QUEUED, 'Queued'), + (SAVING, 'Saving'), + ] + + deployment = models.ForeignKey(Deployment) + owner = models.CharField(max_length=50, null=True, blank=True, + db_index=True) + json = models.TextField() + routing_key = models.CharField(max_length=50, null=True, blank=True, + db_index=True) + when = models.DecimalField(max_digits=20, decimal_places=6, db_index=True) + publisher = models.CharField(max_length=100, null=True, + blank=True, db_index=True) + event = models.CharField(max_length=50, null=True, blank=True, + db_index=True) + service = models.CharField(max_length=50, null=True, blank=True, + db_index=True) + host = models.CharField(max_length=100, null=True, blank=True, + db_index=True) + instance = models.CharField(max_length=50, null=True, blank=True, + db_index=True) + request_id = models.CharField(max_length=50, null=True, blank=True, + db_index=True) + uuid = models.CharField(max_length=50) + status = models.CharField(max_length=50, db_index=True, + choices=STATUS_CHOICES, default=QUEUED) + image_type = models.IntegerField(null=True, default=0, db_index=True) + + @staticmethod + def get_name(): + return GlanceRawData.__name__ + def get_model_fields(model): return model._meta.fields diff --git a/stacktach/notification.py b/stacktach/notification.py index edf367e..b69f76a 100644 --- a/stacktach/notification.py +++ b/stacktach/notification.py @@ -1,25 +1,37 @@ +# Copyright (c) 2013 - Rackspace Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. from stacktach import utils from stacktach import image_type +from stacktach import db class Notification(object): - def __init__(self, body): + def __init__(self, body, deployment, routing_key, json): self.body = body self.request_id = body.get('_context_request_id', "") + self.deployment = deployment + self.routing_key = routing_key + self.json = json self.payload = body.get('payload', {}) - self.state = self.payload.get('state', "") - self.old_state = self.payload.get('old_state', "") - self.old_task = self.payload.get('old_task_state', "") - self.task = self.payload.get('new_task_state', "") - self.image_type = image_type.get_numeric_code(self.payload) self.publisher = self.body['publisher_id'] self.event = self.body['event_type'] - image_meta = self.payload.get('image_meta', {}) - self.os_architecture = image_meta.get('org.openstack__1__architecture', - '') - self.os_distro = image_meta.get('org.openstack__1__os_distro', '') - self.os_version = image_meta.get('org.openstack__1__os_version', '') - self.rax_options = image_meta.get('com.rackspace__1__options', '') @property def when(self): @@ -29,30 +41,24 @@ class Notification(object): when = utils.str_time_to_unix(when) return when - def rawdata_kwargs(self, deployment, routing_key, json): - return { - 'deployment': deployment, - 'routing_key': routing_key, - 'event': self.event, - 'publisher': self.publisher, - 'json': json, - 'state': self.state, - 'old_state': self.old_state, - 'task': self.task, - 'old_task': self.old_task, - 'image_type': self.image_type, - 'when': self.when, - 'publisher': self.publisher, - 'service': self.service, - 'host': self.host, - 'instance': self.instance, - 'request_id': self.request_id, - 'tenant': self.tenant, - 'os_architecture': self.os_architecture, - 'os_distro': self.os_distro, - 'os_version': self.os_version, - 'rax_options': self.rax_options - } + @property + def service(self): + parts = self.publisher.split('.') + return parts[0] + + @property + def host(self): + host = None + parts = self.publisher.split('.') + if len(parts) > 1: + host = ".".join(parts[1:]) + return host + + @property + def tenant(self): + tenant = self.body.get('_context_project_id', None) + tenant = self.payload.get('tenant_id', tenant) + return tenant @property def instance(self): @@ -65,6 +71,71 @@ class Notification(object): instance = self.payload.get('instance', {}).get('uuid') return instance + def save(self): + return db.create_generic_rawdata(deployment=self.deployment, + routing_key=self.routing_key, + tenant=self.tenant, + json=self.json, + when=self.when, + publisher=self.publisher, + event=self.event, + service=self.service, + host=self.host, + instance=self.instance, + request_id=self.request_id) + + +class GlanceNotification(Notification): + def __init__(self, body, deployment, routing_key, json): + super(GlanceNotification, self).__init__(body, deployment, + routing_key, json) + self.properties = self.payload.get('properties', {}) + self.image_type = image_type.get_numeric_code(self.payload) + self.status = self.payload.get('status', None) + self.uuid = self.payload.get('id', None) + + + @property + def owner(self): + return self.payload.get('owner', None) + + @property + def instance(self): + return self.properties.get('instance_uuid', None) + + def save(self): + db.create_glance_rawdata(deployment=self.deployment, + routing_key=self.routing_key, + owner=self.owner, + json=self.json, + when=self.when, + publisher=self.publisher, + event=self.event, + service=self.service, + host=self.host, + instance=self.instance, + request_id=self.request_id, + image_type=self.image_type, + status=self.status, + uuid=self.uuid) + + +class NovaNotification(Notification): + def __init__(self, body, deployment, routing_key, json): + super(NovaNotification, self).__init__(body, deployment, routing_key, + json) + self.state = self.payload.get('state', "") + self.old_state = self.payload.get('old_state', "") + self.old_task = self.payload.get('old_task_state', "") + self.task = self.payload.get('new_task_state', "") + self.image_type = image_type.get_numeric_code(self.payload) + image_meta = self.payload.get('image_meta', {}) + self.os_architecture = image_meta.get('org.openstack__1__architecture', + '') + self.os_distro = image_meta.get('org.openstack__1__os_distro', '') + self.os_version = image_meta.get('org.openstack__1__os_version', '') + self.rax_options = image_meta.get('com.rackspace__1__options', '') + @property def host(self): host = None @@ -78,8 +149,31 @@ class Notification(object): parts = self.publisher.split('.') return parts[0] - @property - def tenant(self): - tenant = self.body.get('_context_project_id', None) - tenant = self.payload.get('tenant_id', tenant) - return tenant + def save(self): + return db.create_nova_rawdata(deployment=self.deployment, + routing_key=self.routing_key, + tenant=self.tenant, + json=self.json, + when=self.when, + publisher=self.publisher, + event=self.event, + service=self.service, + host=self.host, + instance=self.instance, + request_id=self.request_id, + state=self.state, + old_state=self.old_state, + task=self.task, + old_task=self.old_task, + os_architecture=self.os_architecture, + os_distro=self.os_distro, + os_version=self.os_version, + rax_options=self.rax_options) + + +def notification_factory(body, deployment, routing_key, json, exchange): + if exchange == 'nova': + return NovaNotification(body, deployment, routing_key, json) + if exchange == "glance": + return GlanceNotification(body, deployment, routing_key, json) + return Notification(body, deployment, routing_key, json) diff --git a/stacktach/tests.py b/stacktach/tests.py index 504c345..80ec3ad 100644 --- a/stacktach/tests.py +++ b/stacktach/tests.py @@ -19,39 +19,106 @@ # IN THE SOFTWARE. from datetime import datetime -import unittest +from django.test import TransactionTestCase import db from stacktach.datetime_to_decimal import dt_to_decimal from stacktach.models import RawDataImageMeta +from stacktach.models import GenericRawData +from stacktach.models import GlanceRawData from stacktach.models import RawData from stacktach.models import get_model_fields -class RawDataImageMetaDbTestCase(unittest.TestCase): +class RawDataImageMetaDbTestCase(TransactionTestCase): def test_create_raw_data_should_populate_rawdata_and_rawdata_imagemeta(self): deployment = db.get_or_create_deployment('deployment1')[0] kwargs = { 'deployment': deployment, 'when': dt_to_decimal(datetime.utcnow()), - 'tenant': '1', 'json': '{}', 'routing_key': 'monitor.info', - 'state': 'verifying', 'old_state': 'pending', - 'old_task': '', 'task': '', 'image_type': 1, - 'publisher': '', 'event': 'compute.instance.exists', - 'service': '', 'host': '', 'instance': '1234-5678-9012-3456', - 'request_id': '1234', 'os_architecture': 'x86', 'os_version': '1', - 'os_distro': 'windows', 'rax_options': '2'} + 'tenant': '1', + 'json': '{}', + 'routing_key': 'monitor.info', + 'state': 'verifying', + 'old_state': 'pending', + 'old_task': 'building', + 'task': 'saving', + 'image_type': 1, + 'publisher': 'publisher', + 'event': 'compute.instance.exists', + 'service': 'compute', + 'host': 'host', + 'instance': '1234-5678-9012-3456', + 'request_id': '1234', + 'os_architecture': 'x86', + 'os_version': '1', + 'os_distro': 'windows', + 'rax_options': '2'} - rawdata = db.create_rawdata(**kwargs) + rawdata = db.create_nova_rawdata(**kwargs) for field in get_model_fields(RawData): if field.name != 'id': self.assertEquals(getattr(rawdata, field.name), kwargs[field.name]) - raw_image_meta = RawDataImageMeta.objects.all()[0] - self.assertEquals(raw_image_meta.raw, rawdata) + raw_image_meta = RawDataImageMeta.objects.filter(raw_id=rawdata.id)[0] self.assertEquals(raw_image_meta.os_architecture, kwargs['os_architecture']) self.assertEquals(raw_image_meta.os_version, kwargs['os_version']) self.assertEquals(raw_image_meta.os_distro, kwargs['os_distro']) self.assertEquals(raw_image_meta.rax_options, kwargs['rax_options']) + + +class GlanceRawDataTestCase(TransactionTestCase): + def test_create_rawdata_should_populate_glance_rawdata(self): + deployment = db.get_or_create_deployment('deployment1')[0] + kwargs = { + 'deployment': deployment, + 'when': dt_to_decimal(datetime.utcnow()), + 'owner': '1234567', + 'json': '{}', + 'routing_key': 'glance_monitor.info', + 'image_type': 1, + 'publisher': 'publisher', + 'event': 'event', + 'service': 'service', + 'host': 'host', + 'instance': '1234-5678-9012-3456', + 'request_id': '1234', + 'uuid': '1234-5678-0912-3456', + 'status': 'active', + } + + db.create_glance_rawdata(**kwargs) + rawdata = GlanceRawData.objects.all().order_by('-id')[0] + + for field in get_model_fields(GlanceRawData): + if field.name != 'id': + self.assertEquals(getattr(rawdata, field.name), + kwargs[field.name]) + + +class GenericRawDataTestCase(TransactionTestCase): + def test_create_generic_rawdata_should_populate_generic_rawdata(self): + deployment = db.get_or_create_deployment('deployment1')[0] + kwargs = { + 'deployment': deployment, + 'when': dt_to_decimal(datetime.utcnow()), + 'tenant': '1234567', + 'json': '{}', + 'routing_key': 'monitor.info', + 'image_type': 1, + 'publisher': 'publisher', + 'event': 'event', + 'service': 'service', + 'host': 'host', + 'instance': '1234-5678-9012-3456', + 'request_id': '1234'} + + db.create_generic_rawdata(**kwargs) + rawdata = GenericRawData.objects.all()[0] + + for field in get_model_fields(GenericRawData): + if field.name != 'id': + self.assertEquals(getattr(rawdata, field.name), + kwargs[field.name]) diff --git a/stacktach/views.py b/stacktach/views.py index d639fe5..15da309 100644 --- a/stacktach/views.py +++ b/stacktach/views.py @@ -12,7 +12,7 @@ from stacktach import db as stackdb from stacktach import models from stacktach import stacklog from stacktach import utils -from stacktach.notification import Notification +from stacktach import notification STACKDB = stackdb @@ -25,13 +25,6 @@ def log_warn(msg): LOG.warn(msg) -# routing_key : handler - -NOTIFICATIONS = { - 'monitor.info': Notification, - 'monitor.error': Notification} - - def start_kpi_tracking(lifecycle, raw): """Start the clock for kpi timings when we see an instance.update coming in from an api node.""" @@ -279,10 +272,8 @@ def _process_exists(raw, body): values['tenant'] = payload['tenant_id'] image_meta = payload.get('image_meta', {}) values['rax_options'] = image_meta.get('com.rackspace__1__options', '') - os_arch = image_meta.get('org.openstack__1__architecture', '') - values['os_architecture'] = os_arch - os_version = image_meta.get('org.openstack__1__os_version', '') - values['os_version'] = os_version + values['os_architecture'] = image_meta.get('org.openstack__1__architecture', '') + values['os_version'] = image_meta.get('org.openstack__1__os_version', '') values['os_distro'] = image_meta.get('org.openstack__1__os_distro', '') deleted_at = payload.get('deleted_at') @@ -327,26 +318,29 @@ def aggregate_usage(raw, body): USAGE_PROCESS_MAPPING[raw.event](raw, body) -def process_raw_data(deployment, args, json_args): +def process_raw_data(deployment, args, json_args, exchange): """This is called directly by the worker to add the event to the db.""" db.reset_queries() routing_key, body = args - record = None - notification = NOTIFICATIONS[routing_key](body) - if notification: - values = notification.rawdata_kwargs(deployment, routing_key, json_args) - if not values: - return record - record = STACKDB.create_rawdata(**values) - return record + notif = notification.notification_factory(body, deployment, routing_key, + json_args, exchange) + return notif.save() -def post_process(raw, body): +def post_process_rawdata(raw, body): aggregate_lifecycle(raw) aggregate_usage(raw, body) +def post_process_glancerawdata(raw, body): + pass + + +def post_process_genericrawdata(raw, body): + pass + + def _post_process_raw_data(rows, highlight=None): for row in rows: if "error" in row.routing_key: diff --git a/tests/unit/test_models.py b/tests/unit/test_models.py new file mode 100644 index 0000000..9501e68 --- /dev/null +++ b/tests/unit/test_models.py @@ -0,0 +1,33 @@ +# Copyright (c) 2013 - Rackspace Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import unittest +from stacktach.models import RawData, GlanceRawData, GenericRawData + + +class ModelsTestCase(unittest.TestCase): + def test_get_name_for_rawdata(self): + self.assertEquals(RawData.get_name(), 'RawData') + + def test_get_name_for_glancerawdata(self): + self.assertEquals(GlanceRawData.get_name(), 'GlanceRawData') + + def test_get_name_for_genericrawdata(self): + self.assertEquals(GenericRawData.get_name(), 'GenericRawData') \ No newline at end of file diff --git a/tests/unit/test_notification.py b/tests/unit/test_notification.py index feeb696..b1dffe3 100644 --- a/tests/unit/test_notification.py +++ b/tests/unit/test_notification.py @@ -18,163 +18,167 @@ # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. -from decimal import Decimal import unittest + +import mox + +from stacktach import notification, utils + from stacktach.notification import Notification -from tests.unit.utils import REQUEST_ID_1, TENANT_ID_1, INSTANCE_ID_1 +from stacktach.notification import GlanceNotification +from stacktach import db +from tests.unit.utils import REQUEST_ID_1 +from tests.unit.utils import TIMESTAMP_1 +from tests.unit.utils import TENANT_ID_1 +from tests.unit.utils import INSTANCE_ID_1 + + +class NovaNotificationTestCase(unittest.TestCase): + + def setUp(self): + self.mox = mox.Mox() + + def tearDown(self): + self.mox.UnsetStubs() + + def test_factory_should_return_nova_notification_for_nova_exchange( + self): + body = {} + deployment = "1" + json = "{}" + routing_key = "monitor.info" + self.mox.StubOutWithMock(notification, 'NovaNotification') + notification.NovaNotification(body, deployment, routing_key, json) + + self.mox.ReplayAll() + notification.notification_factory(body, deployment, routing_key, json, + 'nova') + self.mox.VerifyAll() + + def test_factory_should_return_glance_notification_for_glance_exchange( + self): + body = {} + deployment = "1" + json = "{}" + routing_key = "monitor_glance.info" + + self.mox.StubOutWithMock(notification, 'GlanceNotification') + notification.GlanceNotification(body, deployment, routing_key, json) + + self.mox.ReplayAll() + notification.notification_factory(body, deployment, routing_key, json, + 'glance') + self.mox.VerifyAll() + + def test_factory_should_return_notification_for_unknown_exchange( + self): + body = {} + deployment = "1" + json = "{}" + routing_key = "unknown.info" + + self.mox.StubOutWithMock(notification, 'Notification') + notification.Notification(body, deployment, routing_key, json) + + self.mox.ReplayAll() + notification.notification_factory(body, deployment, routing_key, json, + 'unknown_exchange') + self.mox.VerifyAll() + + +class GlanceNotificationTestCase(unittest.TestCase): + def setUp(self): + self.mox = mox.Mox() + + def tearDown(self): + self.mox.UnsetStubs() + + def test_save_should_persist_glance_rawdata_to_database(self): + body = { + "event_type": "image.upload", + "timestamp": "2013-06-20 17:31:57.939614", + "publisher_id": "glance-api01-r2961.global.preprod-ord.ohthree.com", + "payload": { + "status": "saving", + "properties": { + "image_type": "snapshot", + "instance_uuid": INSTANCE_ID_1, + }, + "owner": TENANT_ID_1, + "id": "2df2ccf6-bc1b-4853-aab0-25fda346b3bb", + } + } + deployment = "1" + routing_key = "glance_monitor.info" + json = '{["routing_key", {%s}]}' % body + self.mox.StubOutWithMock(db, 'create_glance_rawdata') + db.create_glance_rawdata( + deployment="1", + owner=TENANT_ID_1, + json=json, + routing_key=routing_key, + when=utils.str_time_to_unix("2013-06-20 17:31:57.939614"), + publisher="glance-api01-r2961.global.preprod-ord.ohthree.com", + event="image.upload", + service="glance-api01-r2961", + host="global.preprod-ord.ohthree.com", + instance=INSTANCE_ID_1, + request_id=None, + image_type=0, + status="saving", + uuid="2df2ccf6-bc1b-4853-aab0-25fda346b3bb") + + self.mox.ReplayAll() + + notification = GlanceNotification(body, deployment, routing_key, + json) + notification.save() + self.mox.VerifyAll() class NotificationTestCase(unittest.TestCase): + def setUp(self): + self.mox = mox.Mox() - def test_rawdata_kwargs(self): - message = { - 'event_type': 'compute.instance.create.start', - 'publisher_id': 'compute.cpu1-n01.example.com', + def tearDown(self): + self.mox.UnsetStubs() + + def test_save_should_persist_generic_rawdata_to_database(self): + body = { + "event_type": "image.upload", '_context_request_id': REQUEST_ID_1, '_context_project_id': TENANT_ID_1, - 'timestamp': '2013-06-12 06:30:52.790476', - 'payload': { + "timestamp": TIMESTAMP_1, + "publisher_id": "glance-api01-r2961.global.preprod-ord.ohthree.com", + "payload": { 'instance_id': INSTANCE_ID_1, - 'state': 'active', - 'old_state': 'building', - 'old_task_state': 'build', - "new_task_state": 'rebuild_spawning', - 'image_meta': { - 'image_type': 'base', - 'org.openstack__1__architecture': 'x64', - 'org.openstack__1__os_distro': 'com.microsoft.server', - 'org.openstack__1__os_version': '2008.2', - 'com.rackspace__1__options': '36' - } + "status": "saving", + "container_format": "ovf", + "properties": { + "image_type": "snapshot", + }, + "tenant": "5877054", } } - kwargs = Notification(message).rawdata_kwargs('1', 'monitor.info', 'json') + deployment = "1" + routing_key = "generic_monitor.info" + json = '{["routing_key", {%s}]}' % body + self.mox.StubOutWithMock(db, 'create_generic_rawdata') + db.create_generic_rawdata( + deployment="1", + tenant=TENANT_ID_1, + json=json, + routing_key=routing_key, + when=utils.str_time_to_unix(TIMESTAMP_1), + publisher="glance-api01-r2961.global.preprod-ord.ohthree.com", + event="image.upload", + service="glance-api01-r2961", + host="global.preprod-ord.ohthree.com", + instance=INSTANCE_ID_1, + request_id=REQUEST_ID_1) - self.assertEquals(kwargs['host'], 'cpu1-n01.example.com') - self.assertEquals(kwargs['deployment'], '1') - self.assertEquals(kwargs['routing_key'], 'monitor.info') - self.assertEquals(kwargs['tenant'], TENANT_ID_1) - self.assertEquals(kwargs['json'], 'json') - self.assertEquals(kwargs['state'], 'active') - self.assertEquals(kwargs['old_state'], 'building') - self.assertEquals(kwargs['old_task'], 'build') - self.assertEquals(kwargs['task'], 'rebuild_spawning') - self.assertEquals(kwargs['image_type'], 1) - self.assertEquals(kwargs['when'], Decimal('1371018652.790476')) - self.assertEquals(kwargs['publisher'], 'compute.cpu1-n01.example.com') - self.assertEquals(kwargs['event'], 'compute.instance.create.start') - self.assertEquals(kwargs['request_id'], REQUEST_ID_1) + self.mox.ReplayAll() - def test_rawdata_kwargs_missing_image_meta(self): - message = { - 'event_type': 'compute.instance.create.start', - 'publisher_id': 'compute.cpu1-n01.example.com', - '_context_request_id': REQUEST_ID_1, - '_context_project_id': TENANT_ID_1, - 'timestamp': '2013-06-12 06:30:52.790476', - 'payload': { - 'instance_id': INSTANCE_ID_1, - 'state': 'active', - 'old_state': 'building', - 'old_task_state': 'build', - "new_task_state": 'rebuild_spawning', - 'image_meta': { - 'image_type': 'base', - } - } - } - kwargs = Notification(message).rawdata_kwargs('1', 'monitor.info', 'json') - - self.assertEquals(kwargs['host'], 'cpu1-n01.example.com') - self.assertEquals(kwargs['deployment'], '1') - self.assertEquals(kwargs['routing_key'], 'monitor.info') - self.assertEquals(kwargs['tenant'], TENANT_ID_1) - self.assertEquals(kwargs['json'], 'json') - self.assertEquals(kwargs['state'], 'active') - self.assertEquals(kwargs['old_state'], 'building') - self.assertEquals(kwargs['old_task'], 'build') - self.assertEquals(kwargs['task'], 'rebuild_spawning') - self.assertEquals(kwargs['image_type'], 1) - self.assertEquals(kwargs['when'], Decimal('1371018652.790476')) - self.assertEquals(kwargs['publisher'], 'compute.cpu1-n01.example.com') - self.assertEquals(kwargs['event'], 'compute.instance.create.start') - self.assertEquals(kwargs['request_id'], REQUEST_ID_1) - - def test_rawdata_kwargs_for_message_with_no_host(self): - message = { - 'event_type': 'compute.instance.create.start', - 'publisher_id': 'compute', - '_context_request_id': REQUEST_ID_1, - '_context_project_id': TENANT_ID_1, - 'timestamp': '2013-06-12 06:30:52.790476', - 'payload': { - 'instance_id': INSTANCE_ID_1, - 'state': 'active', - 'old_state': 'building', - 'old_task_state': 'build', - "new_task_state": 'rebuild_spawning', - 'image_meta': { - 'image_type': 'base', - 'org.openstack__1__architecture': 'x64', - 'org.openstack__1__os_distro': 'com.microsoft.server', - 'org.openstack__1__os_version': '2008.2', - 'com.rackspace__1__options': '36' - } - } - } - kwargs = Notification(message).rawdata_kwargs('1', 'monitor.info', 'json') - self.assertEquals(kwargs['host'], None) - - self.assertEquals(kwargs['deployment'], '1') - self.assertEquals(kwargs['routing_key'], 'monitor.info') - self.assertEquals(kwargs['tenant'], TENANT_ID_1) - self.assertEquals(kwargs['json'], 'json') - self.assertEquals(kwargs['state'], 'active') - self.assertEquals(kwargs['old_state'], 'building') - self.assertEquals(kwargs['old_task'], 'build') - self.assertEquals(kwargs['task'], 'rebuild_spawning') - self.assertEquals(kwargs['image_type'], 1) - self.assertEquals(kwargs['when'], Decimal('1371018652.790476')) - self.assertEquals(kwargs['publisher'], 'compute') - self.assertEquals(kwargs['event'], 'compute.instance.create.start') - self.assertEquals(kwargs['request_id'], REQUEST_ID_1) - - def test_rawdata_kwargs_for_message_with_exception(self): - message = { - 'event_type': 'compute.instance.create.start', - 'publisher_id': 'compute.cpu1-n01.example.com', - '_context_request_id': REQUEST_ID_1, - '_context_project_id': TENANT_ID_1, - 'timestamp': '2013-06-12 06:30:52.790476', - 'payload': { - 'exception': {'kwargs':{'uuid': INSTANCE_ID_1}}, - 'instance_id': INSTANCE_ID_1, - 'state': 'active', - 'old_state': 'building', - 'old_task_state': 'build', - "new_task_state": 'rebuild_spawning', - 'image_meta': { - 'image_type': 'base', - 'org.openstack__1__architecture': 'x64', - 'org.openstack__1__os_distro': 'com.microsoft.server', - 'org.openstack__1__os_version': '2008.2', - 'com.rackspace__1__options': '36' - } - } - } - kwargs = Notification(message).rawdata_kwargs('1', 'monitor.info', 'json') - - self.assertEquals(kwargs['host'], 'cpu1-n01.example.com') - self.assertEquals(kwargs['deployment'], '1') - self.assertEquals(kwargs['routing_key'], 'monitor.info') - self.assertEquals(kwargs['tenant'], TENANT_ID_1) - self.assertEquals(kwargs['json'], 'json') - self.assertEquals(kwargs['state'], 'active') - self.assertEquals(kwargs['old_state'], 'building') - self.assertEquals(kwargs['old_task'], 'build') - self.assertEquals(kwargs['task'], 'rebuild_spawning') - self.assertEquals(kwargs['image_type'], 1) - self.assertEquals(kwargs['when'], Decimal('1371018652.790476')) - self.assertEquals(kwargs['publisher'], 'compute.cpu1-n01.example.com') - self.assertEquals(kwargs['event'], 'compute.instance.create.start') - self.assertEquals(kwargs['request_id'], REQUEST_ID_1) + notification = Notification(body, deployment, routing_key, json) + notification.save() + self.mox.VerifyAll() diff --git a/tests/unit/test_stacktach.py b/tests/unit/test_stacktach.py index d3e0853..4138997 100644 --- a/tests/unit/test_stacktach.py +++ b/tests/unit/test_stacktach.py @@ -37,6 +37,7 @@ from utils import INSTANCE_TYPE_ID_1 from utils import DUMMY_TIME from utils import INSTANCE_TYPE_ID_2 from stacktach import stacklog +from stacktach import notification from stacktach import views @@ -59,54 +60,45 @@ class StacktachRawParsingTestCase(unittest.TestCase): dict = { 'timestamp': when, } - args = ('monitor.info', dict) + routing_key = 'monitor.info' + args = (routing_key, dict) json_args = json.dumps(args) - raw_values = { - 'deployment': deployment, - 'when': utils.decimal_utc(datetime.datetime.strptime(when, '%Y-%m-%d %H:%M:%S.%f')), - 'host': 'api', - 'routing_key': 'monitor.info', - 'json': json_args - } - - old_info_handler = views.NOTIFICATIONS['monitor.info'] + mock_record = self.mox.CreateMockAnything() mock_notification = self.mox.CreateMockAnything() - mock_notification.rawdata_kwargs(deployment, 'monitor.info', json_args).AndReturn(raw_values) - views.NOTIFICATIONS['monitor.info'] = lambda message_body: mock_notification - - views.STACKDB.create_rawdata(**raw_values) + mock_notification.save().AndReturn(mock_record) + self.mox.StubOutWithMock(notification, 'notification_factory') + exchange = 'nova' + notification.notification_factory(dict, deployment, routing_key, + json_args, exchange).AndReturn( + mock_notification) self.mox.ReplayAll() - views.process_raw_data(deployment, args, json_args) - self.mox.VerifyAll() - views.NOTIFICATIONS['monitor.info'] = old_info_handler + self.assertEquals( + views.process_raw_data(deployment, args, json_args, exchange), + mock_record) + self.mox.VerifyAll() def test_process_raw_data_old_timestamp(self): deployment = self.mox.CreateMockAnything() when = '2013-1-25T13:38:23.123' dict = { '_context_timestamp': when, - } + } + routing_key = 'monitor.info' args = ('monitor.info', dict) json_args = json.dumps(args[1]) - raw_values = { - 'deployment': deployment, - 'when': utils.decimal_utc(datetime.datetime.strptime(when, '%Y-%m-%dT%H:%M:%S.%f')), - 'host': 'api', - 'routing_key': 'monitor.info', - 'json': json_args - } - old_info_handler = views.NOTIFICATIONS['monitor.info'] - mock_notification = self.mox.CreateMockAnything() - mock_notification.rawdata_kwargs(deployment, 'monitor.info', json_args).AndReturn(raw_values) - views.NOTIFICATIONS['monitor.info'] = lambda message_body: mock_notification - views.STACKDB.create_rawdata(**raw_values) + mock_notification = self.mox.CreateMockAnything() + mock_notification.save() + self.mox.StubOutWithMock(notification, 'notification_factory') + exchange = 'nova' + notification.notification_factory(dict, deployment, routing_key, + json_args, exchange).AndReturn(mock_notification) self.mox.ReplayAll() - views.process_raw_data(deployment, args, json_args) + + views.process_raw_data(deployment, args, json_args, exchange) self.mox.VerifyAll() - views.NOTIFICATIONS['monitor.info'] = old_info_handler class StacktachLifecycleTestCase(unittest.TestCase): def setUp(self): diff --git a/tests/unit/test_worker.py b/tests/unit/test_worker.py index 442b967..f21c227 100644 --- a/tests/unit/test_worker.py +++ b/tests/unit/test_worker.py @@ -22,15 +22,14 @@ import json import unittest import kombu -import kombu.entity -import kombu.connection import mox -from stacktach import db, views +from stacktach import db +from stacktach import views import worker.worker as worker -class NovaConsumerTestCase(unittest.TestCase): +class ConsumerTestCase(unittest.TestCase): def setUp(self): self.mox = mox.Mox() @@ -47,9 +46,10 @@ class NovaConsumerTestCase(unittest.TestCase): consumer = self.mox.CreateMockAnything() created_consumers.append(consumer) return consumer - self.mox.StubOutWithMock(worker.NovaConsumer, '_create_exchange') - self.mox.StubOutWithMock(worker.NovaConsumer, '_create_queue') - consumer = worker.NovaConsumer('test', None, None, True, {}) + self.mox.StubOutWithMock(worker.Consumer, '_create_exchange') + self.mox.StubOutWithMock(worker.Consumer, '_create_queue') + consumer = worker.Consumer('test', None, None, True, {}, "nova", + ["monitor.info", "monitor.error"]) exchange = self.mox.CreateMockAnything() consumer._create_exchange('nova', 'topic').AndReturn(exchange) info_queue = self.mox.CreateMockAnything() @@ -71,7 +71,8 @@ class NovaConsumerTestCase(unittest.TestCase): def test_create_exchange(self): args = {'key': 'value'} - consumer = worker.NovaConsumer('test', None, None, True, args) + consumer = worker.Consumer('test', None, None, True, args, 'nova', + ["monitor.info", "monitor.error"]) self.mox.StubOutClassWithMocks(kombu.entity, 'Exchange') exchange = kombu.entity.Exchange('nova', type='topic', exclusive=False, @@ -87,7 +88,8 @@ class NovaConsumerTestCase(unittest.TestCase): queue = kombu.Queue('name', exchange, auto_delete=False, durable=True, exclusive=False, routing_key='routing.key', queue_arguments={}) - consumer = worker.NovaConsumer('test', None, None, True, {}) + consumer = worker.Consumer('test', None, None, True, {}, 'nova', + ["monitor.info", "monitor.error"]) self.mox.ReplayAll() actual_queue = consumer._create_queue('name', exchange, 'routing.key', exclusive=False, @@ -103,7 +105,8 @@ class NovaConsumerTestCase(unittest.TestCase): queue = kombu.Queue('name', exchange, auto_delete=False, durable=True, exclusive=False, routing_key='routing.key', queue_arguments=queue_args) - consumer = worker.NovaConsumer('test', None, None, True, queue_args) + consumer = worker.Consumer('test', None, None, True, queue_args, + 'nova', ["monitor.info", "monitor.error"]) self.mox.ReplayAll() actual_queue = consumer._create_queue('name', exchange, 'routing.key', exclusive=False, @@ -114,21 +117,29 @@ class NovaConsumerTestCase(unittest.TestCase): def test_process(self): deployment = self.mox.CreateMockAnything() raw = self.mox.CreateMockAnything() + raw.get_name().AndReturn('RawData') message = self.mox.CreateMockAnything() - consumer = worker.NovaConsumer('test', None, deployment, True, {}) + exchange = 'nova' + consumer = worker.Consumer('test', None, deployment, True, {}, + exchange, ["monitor.info", "monitor.error"]) routing_key = 'monitor.info' message.delivery_info = {'routing_key': routing_key} body_dict = {u'key': u'value'} message.body = json.dumps(body_dict) + + mock_post_process_method = self.mox.CreateMockAnything() + mock_post_process_method(raw, body_dict) + old_handler = worker.POST_PROCESS_METHODS + worker.POST_PROCESS_METHODS["RawData"] = mock_post_process_method + self.mox.StubOutWithMock(views, 'process_raw_data', use_mock_anything=True) args = (routing_key, body_dict) - views.process_raw_data(deployment, args, json.dumps(args))\ - .AndReturn(raw) + views.process_raw_data(deployment, args, json.dumps(args), exchange) \ + .AndReturn(raw) message.ack() - self.mox.StubOutWithMock(views, 'post_process') - views.post_process(raw, body_dict) + self.mox.StubOutWithMock(consumer, '_check_memory', use_mock_anything=True) consumer._check_memory() @@ -136,13 +147,16 @@ class NovaConsumerTestCase(unittest.TestCase): consumer._process(message) self.assertEqual(consumer.processed, 1) self.mox.VerifyAll() + worker.POST_PROCESS_METHODS["RawData"] = old_handler def test_process_no_raw_dont_ack(self): deployment = self.mox.CreateMockAnything() raw = self.mox.CreateMockAnything() message = self.mox.CreateMockAnything() - consumer = worker.NovaConsumer('test', None, deployment, True, {}) + exchange = 'nova' + consumer = worker.Consumer('test', None, deployment, True, {}, + exchange, ["monitor.info", "monitor.error"]) routing_key = 'monitor.info' message.delivery_info = {'routing_key': routing_key} body_dict = {u'key': u'value'} @@ -150,8 +164,8 @@ class NovaConsumerTestCase(unittest.TestCase): self.mox.StubOutWithMock(views, 'process_raw_data', use_mock_anything=True) args = (routing_key, body_dict) - views.process_raw_data(deployment, args, json.dumps(args))\ - .AndReturn(None) + views.process_raw_data(deployment, args, json.dumps(args), exchange) \ + .AndReturn(None) self.mox.StubOutWithMock(consumer, '_check_memory', use_mock_anything=True) consumer._check_memory() @@ -168,7 +182,9 @@ class NovaConsumerTestCase(unittest.TestCase): 'rabbit_port': 5672, 'rabbit_userid': 'rabbit', 'rabbit_password': 'rabbit', - 'rabbit_virtual_host': '/' + 'rabbit_virtual_host': '/', + "services": ["nova"], + "topics": {"nova": ["monitor.info", "monitor.error"]} } self.mox.StubOutWithMock(db, 'get_or_create_deployment') deployment = self.mox.CreateMockAnything() @@ -187,13 +203,15 @@ class NovaConsumerTestCase(unittest.TestCase): kombu.connection.BrokerConnection(**params).AndReturn(conn) conn.__enter__().AndReturn(conn) conn.__exit__(None, None, None).AndReturn(None) - self.mox.StubOutClassWithMocks(worker, 'NovaConsumer') - consumer = worker.NovaConsumer(config['name'], conn, deployment, - config['durable_queue'], {}) + self.mox.StubOutClassWithMocks(worker, 'Consumer') + exchange = 'nova' + consumer = worker.Consumer(config['name'], conn, deployment, + config['durable_queue'], {}, exchange, + ["monitor.info", "monitor.error"]) consumer.run() worker.continue_running().AndReturn(False) self.mox.ReplayAll() - worker.run(config) + worker.run(config, exchange) self.mox.VerifyAll() def test_run_queue_args(self): @@ -205,7 +223,9 @@ class NovaConsumerTestCase(unittest.TestCase): 'rabbit_userid': 'rabbit', 'rabbit_password': 'rabbit', 'rabbit_virtual_host': '/', - 'queue_arguments': {'x-ha-policy': 'all'} + 'queue_arguments': {'x-ha-policy': 'all'}, + "services": ["nova"], + "topics": {"nova": ["monitor.info", "monitor.error"]} } self.mox.StubOutWithMock(db, 'get_or_create_deployment') deployment = self.mox.CreateMockAnything() @@ -224,12 +244,14 @@ class NovaConsumerTestCase(unittest.TestCase): kombu.connection.BrokerConnection(**params).AndReturn(conn) conn.__enter__().AndReturn(conn) conn.__exit__(None, None, None).AndReturn(None) - self.mox.StubOutClassWithMocks(worker, 'NovaConsumer') - consumer = worker.NovaConsumer(config['name'], conn, deployment, - config['durable_queue'], - config['queue_arguments']) + self.mox.StubOutClassWithMocks(worker, 'Consumer') + exchange = 'nova' + consumer = worker.Consumer(config['name'], conn, deployment, + config['durable_queue'], + config['queue_arguments'], exchange, + ["monitor.info", "monitor.error"]) consumer.run() worker.continue_running().AndReturn(False) self.mox.ReplayAll() - worker.run(config) - self.mox.VerifyAll() \ No newline at end of file + worker.run(config, exchange) + self.mox.VerifyAll() diff --git a/tests/unit/utils.py b/tests/unit/utils.py index 961f3ce..437e169 100644 --- a/tests/unit/utils.py +++ b/tests/unit/utils.py @@ -52,6 +52,7 @@ OS_ARCH_2 = "x64" OS_VERSION_1 = "1" OS_VERSION_2 = "2" +TIMESTAMP_1 = "2013-06-20 17:31:57.939614" def decimal_utc(t = datetime.datetime.utcnow()): return dt.dt_to_decimal(t) diff --git a/worker/config.py b/worker/config.py new file mode 100644 index 0000000..28c8b5e --- /dev/null +++ b/worker/config.py @@ -0,0 +1,43 @@ +# Copyright (c) 2013 - Rackspace Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +import json +import os + +config_filename = os.environ.get('STACKTACH_DEPLOYMENTS_FILE', + 'stacktach_worker_config.json') +try: + from local_settings import * + config_filename = STACKTACH_DEPLOYMENTS_FILE +except ImportError: + pass + +config = None +with open(config_filename, "r") as f: + config = json.load(f) + + +def deployments(): + return config['deployments'] + + +def topics(): + return config['topics'] + + diff --git a/worker/start_workers.py b/worker/start_workers.py index 19a63d9..e7fc3d5 100644 --- a/worker/start_workers.py +++ b/worker/start_workers.py @@ -1,9 +1,9 @@ -import json import os import signal import sys from multiprocessing import Process +from worker import config POSSIBLE_TOPDIR = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), os.pardir, os.pardir)) @@ -12,14 +12,6 @@ if os.path.exists(os.path.join(POSSIBLE_TOPDIR, 'stacktach')): import worker.worker as worker -config_filename = os.environ.get('STACKTACH_DEPLOYMENTS_FILE', - 'stacktach_worker_config.json') -try: - from local_settings import * - config_filename = STACKTACH_DEPLOYMENTS_FILE -except ImportError: - pass - processes = [] @@ -35,18 +27,15 @@ def kill_time(signal, frame): if __name__ == '__main__': - config = None - with open(config_filename, "r") as f: - config = json.load(f) - deployments = config['deployments'] - - for deployment in deployments: + for deployment in config.deployments(): if deployment.get('enabled', True): - process = Process(target=worker.run, args=(deployment,)) - process.daemon = True - process.start() - processes.append(process) + for exchange in deployment.get('topics').keys(): + process = Process(target=worker.run, args=(deployment, + exchange,)) + process.daemon = True + process.start() + processes.append(process) signal.signal(signal.SIGINT, kill_time) signal.signal(signal.SIGTERM, kill_time) signal.pause() diff --git a/worker/worker.py b/worker/worker.py index a9e8080..c3b584e 100644 --- a/worker/worker.py +++ b/worker/worker.py @@ -17,12 +17,13 @@ # to set TENANT_ID and URL to point to your StackTach web server. import datetime -import kombu -import kombu.entity -import kombu.mixins import sys import time +import kombu +import kombu.mixins + + try: import ujson as json except ImportError: @@ -41,8 +42,9 @@ stacklog.set_default_logger_name('worker') LOG = stacklog.get_logger() -class NovaConsumer(kombu.mixins.ConsumerMixin): - def __init__(self, name, connection, deployment, durable, queue_arguments): +class Consumer(kombu.mixins.ConsumerMixin): + def __init__(self, name, connection, deployment, durable, queue_arguments, + exchange, topics): self.connection = connection self.deployment = deployment self.durable = durable @@ -52,6 +54,8 @@ class NovaConsumer(kombu.mixins.ConsumerMixin): self.pmi = None self.processed = 0 self.total_processed = 0 + self.topics = topics + self.exchange = exchange def _create_exchange(self, name, type, exclusive=False, auto_delete=False): return kombu.entity.Exchange(name, type=type, exclusive=exclusive, @@ -66,14 +70,12 @@ class NovaConsumer(kombu.mixins.ConsumerMixin): routing_key=routing_key) def get_consumers(self, Consumer, channel): - nova_exchange = self._create_exchange("nova", "topic") + exchange = self._create_exchange(self.exchange, "topic") - nova_queues = [ - self._create_queue('monitor.info', nova_exchange, 'monitor.info'), - self._create_queue('monitor.error', nova_exchange, 'monitor.error') - ] + queues = [self._create_queue(topic, exchange, topic) + for topic in self.topics] - return [Consumer(queues=nova_queues, callbacks=[self.on_nova])] + return [Consumer(queues=queues, callbacks=[self.on_nova])] def _process(self, message): routing_key = message.delivery_info['routing_key'] @@ -81,14 +83,13 @@ class NovaConsumer(kombu.mixins.ConsumerMixin): body = str(message.body) args = (routing_key, json.loads(body)) asJson = json.dumps(args) - # save raw and ack the message - raw = views.process_raw_data(self.deployment, args, asJson) + raw = views.process_raw_data(self.deployment, args, asJson, self.exchange) if raw: self.processed += 1 message.ack() - views.post_process(raw, args[1]) + POST_PROCESS_METHODS[raw.get_name()](raw, args[1]) self._check_memory() @@ -140,7 +141,7 @@ def exit_or_sleep(exit=False): time.sleep(5) -def run(deployment_config): +def run(deployment_config, exchange): name = deployment_config['name'] host = deployment_config.get('rabbit_host', 'localhost') port = deployment_config.get('rabbit_port', 5672) @@ -150,11 +151,13 @@ def run(deployment_config): durable = deployment_config.get('durable_queue', True) queue_arguments = deployment_config.get('queue_arguments', {}) exit_on_exception = deployment_config.get('exit_on_exception', False) + topics = deployment_config.get('topics', {}) deployment, new = db.get_or_create_deployment(name) - print "Starting worker for '%s'" % name - LOG.info("%s: %s %s %s %s" % (name, host, port, user_id, virtual_host)) + print "Starting worker for '%s %s'" % (name, exchange) + LOG.info("%s: %s %s %s %s %s" % (name, exchange, host, port, user_id, + virtual_host)) params = dict(hostname=host, port=port, @@ -166,21 +169,30 @@ def run(deployment_config): # continue_running() is used for testing while continue_running(): try: - LOG.debug("Processing on '%s'" % name) + LOG.debug("Processing on '%s %s'" % (name, exchange)) with kombu.connection.BrokerConnection(**params) as conn: try: - consumer = NovaConsumer(name, conn, deployment, durable, - queue_arguments) + consumer = Consumer(name, conn, deployment, durable, + queue_arguments, exchange, + topics[exchange]) consumer.run() except Exception as e: LOG.error("!!!!Exception!!!!") - LOG.exception("name=%s, exception=%s. Reconnecting in 5s" % - (name, e)) + LOG.exception("name=%s, exchange=%s, exception=%s. " + "Reconnecting in 5s" % + (name, exchange, e)) exit_or_sleep(exit_on_exception) - LOG.debug("Completed processing on '%s'" % name) + LOG.debug("Completed processing on '%s %s'" % (name, exchange)) except: LOG.error("!!!!Exception!!!!") e = sys.exc_info()[0] - msg = "Uncaught exception: deployment=%s, exception=%s. Retrying in 5s" - LOG.exception(msg % (name, e)) + msg = "Uncaught exception: deployment=%s, exchange=%s, " \ + "exception=%s. Retrying in 5s" + LOG.exception(msg % (name, exchange, e)) exit_or_sleep(exit_on_exception) + +POST_PROCESS_METHODS = { + 'RawData': views.post_process_rawdata, + 'GlanceRawData': views.post_process_glancerawdata, + 'GenericRawData': views.post_process_genericrawdata +} \ No newline at end of file From 3b1e086a26eae748c217867cd18dcd3d1b97498d Mon Sep 17 00:00:00 2001 From: Anuj Mathur Date: Tue, 2 Jul 2013 14:53:43 +0530 Subject: [PATCH 02/11] - Moved config.py to worker - Removed exchange from config and started reading them from topics - Fixed lengths of GlanceRawData fields to match Glance fields - Made uuid and status accept null values as the image.send notification does not contain those values --- stacktach/config.py | 43 +++++++++++++++++++ ...eexists_and_instanceusages_from_rawdata.py | 3 +- ...reate_glancerawdata_and_genericrawdata.py} | 23 +++++++--- stacktach/models.py | 26 ++++++++--- worker/start_workers.py | 2 +- worker/worker.py | 4 +- 6 files changed, 86 insertions(+), 15 deletions(-) create mode 100644 stacktach/config.py rename stacktach/migrations/{0004_create_glancerawdata_and_genericrawdata.py => 0005_create_glancerawdata_and_genericrawdata.py} (92%) diff --git a/stacktach/config.py b/stacktach/config.py new file mode 100644 index 0000000..28c8b5e --- /dev/null +++ b/stacktach/config.py @@ -0,0 +1,43 @@ +# Copyright (c) 2013 - Rackspace Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +import json +import os + +config_filename = os.environ.get('STACKTACH_DEPLOYMENTS_FILE', + 'stacktach_worker_config.json') +try: + from local_settings import * + config_filename = STACKTACH_DEPLOYMENTS_FILE +except ImportError: + pass + +config = None +with open(config_filename, "r") as f: + config = json.load(f) + + +def deployments(): + return config['deployments'] + + +def topics(): + return config['topics'] + + diff --git a/stacktach/migrations/0003_populate_usage_related_fields_in_rawdataimagemeta_instanceexists_and_instanceusages_from_rawdata.py b/stacktach/migrations/0003_populate_usage_related_fields_in_rawdataimagemeta_instanceexists_and_instanceusages_from_rawdata.py index 86ae331..afa045e 100644 --- a/stacktach/migrations/0003_populate_usage_related_fields_in_rawdataimagemeta_instanceexists_and_instanceusages_from_rawdata.py +++ b/stacktach/migrations/0003_populate_usage_related_fields_in_rawdataimagemeta_instanceexists_and_instanceusages_from_rawdata.py @@ -42,7 +42,8 @@ class Migration(DataMigration): json_dict = json.loads(json_message) routing_key = json_dict[0] body = json_dict[1] - return notification_factory(body, None, routing_key, json_message, None) + return notification_factory(body, None, routing_key, json_message, + 'nova') def forwards(self, orm): # Note: Don't use "from appname.models import ModelName". diff --git a/stacktach/migrations/0004_create_glancerawdata_and_genericrawdata.py b/stacktach/migrations/0005_create_glancerawdata_and_genericrawdata.py similarity index 92% rename from stacktach/migrations/0004_create_glancerawdata_and_genericrawdata.py rename to stacktach/migrations/0005_create_glancerawdata_and_genericrawdata.py index 277c5ca..172c5bc 100644 --- a/stacktach/migrations/0004_create_glancerawdata_and_genericrawdata.py +++ b/stacktach/migrations/0005_create_glancerawdata_and_genericrawdata.py @@ -12,7 +12,7 @@ class Migration(SchemaMigration): db.create_table(u'stacktach_glancerawdata', ( (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), ('deployment', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['stacktach.Deployment'])), - ('owner', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=50, null=True, blank=True)), + ('owner', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=255, null=True, blank=True)), ('json', self.gf('django.db.models.fields.TextField')()), ('routing_key', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=50, null=True, blank=True)), ('when', self.gf('django.db.models.fields.DecimalField')(max_digits=20, decimal_places=6, db_index=True)), @@ -22,8 +22,8 @@ class Migration(SchemaMigration): ('host', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=100, null=True, blank=True)), ('instance', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=50, null=True, blank=True)), ('request_id', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=50, null=True, blank=True)), - ('uuid', self.gf('django.db.models.fields.CharField')(max_length=50)), - ('status', self.gf('django.db.models.fields.CharField')(default='queued', max_length=50, db_index=True)), + ('uuid', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=36, null=True, blank=True)), + ('status', self.gf('django.db.models.fields.CharField')(max_length=30, null=True, db_index=True)), ('image_type', self.gf('django.db.models.fields.IntegerField')(default=0, null=True, db_index=True)), )) db.send_create_signal(u'stacktach', ['GlanceRawData']) @@ -86,13 +86,13 @@ class Migration(SchemaMigration): 'image_type': ('django.db.models.fields.IntegerField', [], {'default': '0', 'null': 'True', 'db_index': 'True'}), 'instance': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), 'json': ('django.db.models.fields.TextField', [], {}), - 'owner': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'owner': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '255', 'null': 'True', 'blank': 'True'}), 'publisher': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '100', 'null': 'True', 'blank': 'True'}), 'request_id': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), 'routing_key': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), 'service': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), - 'status': ('django.db.models.fields.CharField', [], {'default': "'queued'", 'max_length': '50', 'db_index': 'True'}), - 'uuid': ('django.db.models.fields.CharField', [], {'max_length': '50'}), + 'status': ('django.db.models.fields.CharField', [], {'max_length': '30', 'null': 'True', 'db_index': 'True'}), + 'uuid': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '36', 'null': 'True', 'blank': 'True'}), 'when': ('django.db.models.fields.DecimalField', [], {'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}) }, u'stacktach.instancedeletes': { @@ -125,6 +125,17 @@ class Migration(SchemaMigration): 'tenant': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), 'usage': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'null': 'True', 'to': u"orm['stacktach.InstanceUsage']"}) }, + u'stacktach.instancereconcile': { + 'Meta': {'object_name': 'InstanceReconcile'}, + 'deleted_at': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'instance': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'instance_type_id': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'launched_at': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}), + 'row_created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'row_updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}), + 'source': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '150', 'null': 'True', 'blank': 'True'}) + }, u'stacktach.instanceusage': { 'Meta': {'object_name': 'InstanceUsage'}, u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), diff --git a/stacktach/models.py b/stacktach/models.py index 6980ee4..f2d3df4 100644 --- a/stacktach/models.py +++ b/stacktach/models.py @@ -1,3 +1,18 @@ +# Copyright 2012 - Dark Secret Software Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + from django import forms from django.db import models @@ -273,8 +288,8 @@ class GlanceRawData(models.Model): ] deployment = models.ForeignKey(Deployment) - owner = models.CharField(max_length=50, null=True, blank=True, - db_index=True) + owner = models.CharField(max_length=255, null=True, blank=True, + db_index=True) json = models.TextField() routing_key = models.CharField(max_length=50, null=True, blank=True, db_index=True) @@ -291,9 +306,10 @@ class GlanceRawData(models.Model): db_index=True) request_id = models.CharField(max_length=50, null=True, blank=True, db_index=True) - uuid = models.CharField(max_length=50) - status = models.CharField(max_length=50, db_index=True, - choices=STATUS_CHOICES, default=QUEUED) + uuid = models.CharField(max_length=36, null=True, blank=True, + db_index=True) + status = models.CharField(max_length=30, db_index=True, + choices=STATUS_CHOICES, null=True) image_type = models.IntegerField(null=True, default=0, db_index=True) @staticmethod diff --git a/worker/start_workers.py b/worker/start_workers.py index e7fc3d5..0558b57 100644 --- a/worker/start_workers.py +++ b/worker/start_workers.py @@ -3,7 +3,6 @@ import signal import sys from multiprocessing import Process -from worker import config POSSIBLE_TOPDIR = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), os.pardir, os.pardir)) @@ -11,6 +10,7 @@ if os.path.exists(os.path.join(POSSIBLE_TOPDIR, 'stacktach')): sys.path.insert(0, POSSIBLE_TOPDIR) import worker.worker as worker +from worker import config processes = [] diff --git a/worker/worker.py b/worker/worker.py index c3b584e..11efb98 100644 --- a/worker/worker.py +++ b/worker/worker.py @@ -114,9 +114,9 @@ class Consumer(kombu.mixins.ConsumerMixin): per_message = 0 if self.total_processed: per_message = idiff / self.total_processed - LOG.debug("%20s %6dk/%6dk ram, " + LOG.debug("%20s %20s %6dk/%6dk ram, " "%3d/%4d msgs @ %6dk/msg" % - (self.name, diff, idiff, self.processed, + (self.name, self.exchange, diff, idiff, self.processed, self.total_processed, per_message)) self.last_vsz = self.pmi.vsz self.processed = 0 From ec5bbf31a23447ef538bfc4c5016b895fb46d351 Mon Sep 17 00:00:00 2001 From: Anuj Mathur Date: Mon, 1 Jul 2013 15:16:10 +0530 Subject: [PATCH 03/11] Added image exists, usage and delete models for glance --- stacktach/db.py | 31 ++++++- stacktach/models.py | 70 +++++++++++---- stacktach/tests.py | 48 ++++++++-- stacktach/views.py | 77 ++++++++++++++-- tests/unit/test_stacktach.py | 169 ++++++++++++++++++++++++++++++++++- tests/unit/test_worker.py | 30 +------ tests/unit/utils.py | 3 + worker/worker.py | 12 +-- 8 files changed, 378 insertions(+), 62 deletions(-) diff --git a/stacktach/db.py b/stacktach/db.py index 0043ec7..d28d723 100644 --- a/stacktach/db.py +++ b/stacktach/db.py @@ -103,4 +103,33 @@ def create_generic_rawdata(**kwargs): rawdata = models.GenericRawData(**kwargs) rawdata.save() - return rawdata \ No newline at end of file + return rawdata + + +def create_image_usage(**kwargs): + usage = models.ImageUsage(**kwargs) + usage.save() + + return usage + + +def create_image_delete(**kwargs): + delete = models.ImageDeletes(**kwargs) + delete.save() + + return delete + + +def create_image_exists(**kwargs): + exists = models.ImageExists(**kwargs) + exists.save() + + return exists + + +def get_image_delete(**kwargs): + return _safe_get(models.ImageDeletes, **kwargs) + + +def get_image_usage(**kwargs): + return _safe_get(models.ImageUsage, **kwargs) \ No newline at end of file diff --git a/stacktach/models.py b/stacktach/models.py index f2d3df4..4876313 100644 --- a/stacktach/models.py +++ b/stacktach/models.py @@ -1,18 +1,3 @@ -# Copyright 2012 - Dark Secret Software Inc. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - from django import forms from django.db import models @@ -316,5 +301,60 @@ class GlanceRawData(models.Model): def get_name(): return GlanceRawData.__name__ + +class ImageUsage(models.Model): + uuid = models.CharField(max_length=50, db_index=True) + created_at = models.DecimalField(max_digits=20, + decimal_places=6, db_index=True) + owner = models.CharField(max_length=50, db_index=True) + size = models.BigIntegerField(max_length=20) + last_raw = models.ForeignKey(GlanceRawData) + + +class ImageDeletes(models.Model): + uuid = models.CharField(max_length=50, db_index=True) + created_at = models.DecimalField(max_digits=20, + decimal_places=6, db_index=True) + deleted_at = models.DecimalField(max_digits=20, + decimal_places=6, db_index=True) + owner = models.CharField(max_length=50, db_index=True) + size = models.BigIntegerField(max_length=20) + raw = models.ForeignKey(GlanceRawData) + + +class ImageExists(models.Model): + PENDING = 'pending' + VERIFYING = 'verifying' + VERIFIED = 'verified' + FAILED = 'failed' + STATUS_CHOICES = [ + (PENDING, 'Pending Verification'), + (VERIFYING, 'Currently Being Verified'), + (VERIFIED, 'Passed Verification'), + (FAILED, 'Failed Verification'), + ] + + uuid = models.CharField(max_length=50, db_index=True) + created_at = models.DecimalField(max_digits=20, + decimal_places=6, db_index=True) + deleted_at = models.DecimalField(max_digits=20, + decimal_places=6, db_index=True) + audit_period_beginning = models.DecimalField(max_digits=20, + decimal_places=6, + db_index=True) + audit_period_ending = models.DecimalField(max_digits=20, + decimal_places=6, db_index=True) + status = models.CharField(max_length=50, db_index=True, + choices=STATUS_CHOICES, + default=PENDING) + fail_reason = models.CharField(max_length=300, db_index=True, null=True) + raw = models.ForeignKey(GlanceRawData, related_name='+') + usage = models.ForeignKey(ImageUsage, related_name='+') + delete = models.ForeignKey(ImageDeletes, related_name='+') + send_status = models.IntegerField(default=0, db_index=True) + owner = models.CharField(max_length=255, db_index=True) + size = models.BigIntegerField(max_length=20) + + def get_model_fields(model): return model._meta.fields diff --git a/stacktach/tests.py b/stacktach/tests.py index 80ec3ad..802a5fb 100644 --- a/stacktach/tests.py +++ b/stacktach/tests.py @@ -22,7 +22,7 @@ from datetime import datetime from django.test import TransactionTestCase import db from stacktach.datetime_to_decimal import dt_to_decimal -from stacktach.models import RawDataImageMeta +from stacktach.models import RawDataImageMeta, ImageUsage, ImageDeletes from stacktach.models import GenericRawData from stacktach.models import GlanceRawData from stacktach.models import RawData @@ -69,8 +69,8 @@ class RawDataImageMetaDbTestCase(TransactionTestCase): self.assertEquals(raw_image_meta.rax_options, kwargs['rax_options']) -class GlanceRawDataTestCase(TransactionTestCase): - def test_create_rawdata_should_populate_glance_rawdata(self): +class GlanceTestCase(TransactionTestCase): + def _create_glance_rawdata(self): deployment = db.get_or_create_deployment('deployment1')[0] kwargs = { 'deployment': deployment, @@ -88,15 +88,53 @@ class GlanceRawDataTestCase(TransactionTestCase): 'uuid': '1234-5678-0912-3456', 'status': 'active', } - db.create_glance_rawdata(**kwargs) - rawdata = GlanceRawData.objects.all().order_by('-id')[0] + rawdata = GlanceRawData.objects.all()[0] + return kwargs, rawdata + + def test_create_rawdata_should_populate_glance_rawdata(self): + kwargs, rawdata = self._create_glance_rawdata() for field in get_model_fields(GlanceRawData): if field.name != 'id': self.assertEquals(getattr(rawdata, field.name), kwargs[field.name]) + def test_create_glance_usage_should_populate_image_usage(self): + _, rawdata = self._create_glance_rawdata() + kwargs = { + 'uuid': '1', + 'created_at': dt_to_decimal(datetime.utcnow()), + 'owner': '1234567', + 'size': 12345, + 'last_raw': rawdata + } + db.create_image_usage(**kwargs) + usage = ImageUsage.objects.all()[0] + + for field in get_model_fields(ImageUsage): + if field.name != 'id': + self.assertEquals(getattr(usage, field.name), + kwargs[field.name]) + + def test_create_image_delete_should_populate_image_delete(self): + _, rawdata = self._create_glance_rawdata() + kwargs = { + 'uuid': '1', + 'created_at': dt_to_decimal(datetime.utcnow()), + 'owner': 'owner', + 'size': 12345, + 'raw': rawdata, + 'deleted_at': dt_to_decimal(datetime.utcnow()) + } + db.create_image_delete(**kwargs) + image_delete = ImageDeletes.objects.all()[0] + + for field in get_model_fields(ImageDeletes): + if field.name != 'id': + self.assertEquals(getattr(image_delete, field.name), + kwargs[field.name]) + class GenericRawDataTestCase(TransactionTestCase): def test_create_generic_rawdata_should_populate_generic_rawdata(self): diff --git a/stacktach/views.py b/stacktach/views.py index 15da309..386eee8 100644 --- a/stacktach/views.py +++ b/stacktach/views.py @@ -295,6 +295,55 @@ def _process_exists(raw, body): stacklog.warn("Ignoring exists without launched_at. RawData(%s)" % raw.id) +def _process_glance_usage(raw, notification): + values = { + 'uuid': notification.uuid, + 'created_at': notification.created_at, + 'owner': notification.owner, + 'size': notification.size, + 'last_raw': raw + } + STACKDB.create_image_usage(**values) + + +def _process_glance_delete(raw, notification): + values = { + 'uuid': notification.uuid, + 'created_at': notification.created_at, + 'owner': notification.owner, + 'size': notification.size, + 'raw': raw, + 'deleted_at': notification.deleted_at + } + STACKDB.create_image_delete(**values) + + +def _process_glance_exists(raw, notification): + if notification.created_at: + values = { + 'uuid': notification.uuid, + 'audit_period_beginning': notification.audit_period_beginning, + 'audit_period_ending': notification.audit_period_ending, + 'owner': notification.owner, + 'size': notification.size, + 'raw': raw, + } + created_at_range = (notification.created_at, notification.created_at+1) + usage = STACKDB.get_image_usage(uuid=notification.uuid, + created_at__range=created_at_range) + values['usage'] = usage + values['created_at'] = notification.created_at + if notification.deleted_at: + delete = STACKDB.get_image_delete(uuid=notification.uuid, + created_at__range=created_at_range) + values['delete'] = delete + values['deleted_at'] = notification.deleted_at + + STACKDB.create_image_exists(**values) + else: + stacklog.warn("Ignoring exists without created_at. GlanceRawData(%s)" + % raw.id) + USAGE_PROCESS_MAPPING = { INSTANCE_EVENT['create_start']: _process_usage_for_new_launch, INSTANCE_EVENT['rebuild_start']: _process_usage_for_new_launch, @@ -306,8 +355,14 @@ USAGE_PROCESS_MAPPING = { INSTANCE_EVENT['resize_finish_end']: _process_usage_for_updates, INSTANCE_EVENT['resize_revert_end']: _process_usage_for_updates, INSTANCE_EVENT['delete_end']: _process_delete, - INSTANCE_EVENT['exists']: _process_exists, -} + INSTANCE_EVENT['exists']: _process_exists +} + +GLANCE_USAGE_PROCESS_MAPPING = { + 'image.activate': _process_glance_usage, + 'image.delete': _process_glance_delete, + 'image.exists': _process_glance_exists +} def aggregate_usage(raw, body): @@ -318,6 +373,11 @@ def aggregate_usage(raw, body): USAGE_PROCESS_MAPPING[raw.event](raw, body) +def aggregate_glance_usage(raw, body): + if raw.event in GLANCE_USAGE_PROCESS_MAPPING.keys(): + GLANCE_USAGE_PROCESS_MAPPING[raw.event](raw, body) + + def process_raw_data(deployment, args, json_args, exchange): """This is called directly by the worker to add the event to the db.""" db.reset_queries() @@ -325,19 +385,20 @@ def process_raw_data(deployment, args, json_args, exchange): routing_key, body = args notif = notification.notification_factory(body, deployment, routing_key, json_args, exchange) - return notif.save() + raw = notif.save() + return raw, notif -def post_process_rawdata(raw, body): +def post_process_rawdata(raw, notification): aggregate_lifecycle(raw) - aggregate_usage(raw, body) + aggregate_usage(raw, notification) -def post_process_glancerawdata(raw, body): - pass +def post_process_glancerawdata(raw, notification): + aggregate_glance_usage(raw, notification) -def post_process_genericrawdata(raw, body): +def post_process_genericrawdata(raw, body, notification): pass diff --git a/tests/unit/test_stacktach.py b/tests/unit/test_stacktach.py index 4138997..055d3d3 100644 --- a/tests/unit/test_stacktach.py +++ b/tests/unit/test_stacktach.py @@ -26,6 +26,7 @@ import mox import utils from utils import INSTANCE_ID_1 +from utils import DECIMAL_DUMMY_TIME from utils import OS_VERSION_1 from utils import OS_ARCH_1 from utils import OS_DISTRO_1 @@ -36,6 +37,7 @@ from utils import TENANT_ID_1 from utils import INSTANCE_TYPE_ID_1 from utils import DUMMY_TIME from utils import INSTANCE_TYPE_ID_2 +from utils import IMAGE_UUID_1 from stacktach import stacklog from stacktach import notification from stacktach import views @@ -75,7 +77,7 @@ class StacktachRawParsingTestCase(unittest.TestCase): self.assertEquals( views.process_raw_data(deployment, args, json_args, exchange), - mock_record) + (mock_record, mock_notification)) self.mox.VerifyAll() def test_process_raw_data_old_timestamp(self): @@ -710,3 +712,168 @@ class StacktachUsageParsingTestCase(unittest.TestCase): views._process_exists(raw, notif[1]) self.mox.VerifyAll() + +class StacktachImageUsageParsingTestCase(unittest.TestCase): + def setUp(self): + self.mox = mox.Mox() + views.STACKDB = self.mox.CreateMockAnything() + + def tearDown(self): + self.mox.UnsetStubs() + + def test_process_image_usage_for_new_launch(self): + raw = self.mox.CreateMockAnything() + values = { + 'created_at': str(DUMMY_TIME), + 'owner': TENANT_ID_1, + 'uuid': IMAGE_UUID_1, + 'size': 1234, + 'last_raw': raw + } + notification = self.mox.CreateMockAnything() + notification.created_at = values['created_at'] + notification.owner = values['owner'] + notification.uuid = values['uuid'] + notification.size = values['size'] + notification.raw = values['last_raw'] + views.STACKDB.create_image_usage(**values) + self.mox.ReplayAll() + views._process_glance_usage(raw, notification) + self.mox.VerifyAll() + + def test_process_image_deletes(self): + raw = self.mox.CreateMockAnything() + values = { + 'uuid': IMAGE_UUID_1, + 'created_at': str(DUMMY_TIME), + 'deleted_at': str(DUMMY_TIME), + 'owner': TENANT_ID_1, + 'size': 1234, + 'raw': raw + } + + notification = self.mox.CreateMockAnything() + notification.created_at = values['created_at'] + notification.deleted_at = values['deleted_at'] + notification.owner = values['owner'] + notification.uuid = values['uuid'] + notification.size = values['size'] + notification.raw = values['raw'] + views.STACKDB.create_image_delete(**values) + self.mox.ReplayAll() + views._process_glance_delete(raw, notification) + self.mox.VerifyAll() + + def test_process_image_exists(self): + raw = self.mox.CreateMockAnything() + values = { + 'uuid': IMAGE_UUID_1, + 'created_at': DECIMAL_DUMMY_TIME, + 'deleted_at': DECIMAL_DUMMY_TIME, + 'audit_period_beginning': DECIMAL_DUMMY_TIME, + 'audit_period_ending': DECIMAL_DUMMY_TIME, + 'owner': TENANT_ID_1, + 'size': 1234, + 'raw': raw, + 'usage': None, + 'delete': None + } + + notification = self.mox.CreateMockAnything() + notification.created_at = values['created_at'] + notification.deleted_at = values['deleted_at'] + notification.owner = values['owner'] + notification.uuid = values['uuid'] + notification.size = values['size'] + notification.raw = values['raw'] + notification.audit_period_beginning = values['audit_period_beginning'] + notification.audit_period_ending = values['audit_period_ending'] + created_at_range = (DECIMAL_DUMMY_TIME, DECIMAL_DUMMY_TIME+1) + views.STACKDB.get_image_usage(created_at__range=created_at_range, + uuid=notification.uuid).AndReturn(None) + views.STACKDB.get_image_delete(created_at__range=created_at_range, + uuid=notification.uuid).AndReturn(None) + views.STACKDB.create_image_exists(**values) + + self.mox.ReplayAll() + + views._process_glance_exists(raw, notification) + self.mox.VerifyAll() + + def test_process_image_exists_with_usage_not_none(self): + raw = self.mox.CreateMockAnything() + usage = self.mox.CreateMockAnything() + values = { + 'uuid': IMAGE_UUID_1, + 'created_at': DECIMAL_DUMMY_TIME, + 'deleted_at': DECIMAL_DUMMY_TIME, + 'audit_period_beginning': DECIMAL_DUMMY_TIME, + 'audit_period_ending': DECIMAL_DUMMY_TIME, + 'owner': TENANT_ID_1, + 'size': 1234, + 'raw': raw, + 'usage': usage, + 'delete': None + } + + notification = self.mox.CreateMockAnything() + notification.created_at = values['created_at'] + notification.deleted_at = values['deleted_at'] + notification.owner = values['owner'] + notification.uuid = values['uuid'] + notification.size = values['size'] + notification.raw = values['raw'] + notification.audit_period_beginning = values['audit_period_beginning'] + notification.audit_period_ending = values['audit_period_ending'] + created_at_range = (DECIMAL_DUMMY_TIME, DECIMAL_DUMMY_TIME+1) + views.STACKDB.get_image_usage(created_at__range=created_at_range, + uuid=notification.uuid).AndReturn(usage) + views.STACKDB.get_image_delete(created_at__range=created_at_range, + uuid=notification.uuid).AndReturn(None) + views.STACKDB.create_image_exists(**values) + + self.mox.ReplayAll() + + views._process_glance_exists(raw, notification) + self.mox.VerifyAll() + + + def test_process_image_exists_with_delete_not_none(self): + raw = self.mox.CreateMockAnything() + delete = self.mox.CreateMockAnything() + values = { + 'uuid': IMAGE_UUID_1, + 'created_at': DECIMAL_DUMMY_TIME, + 'deleted_at': DECIMAL_DUMMY_TIME, + 'audit_period_beginning': DECIMAL_DUMMY_TIME, + 'audit_period_ending': DECIMAL_DUMMY_TIME, + 'owner': TENANT_ID_1, + 'size': 1234, + 'raw': raw, + 'usage': None, + 'delete': delete + } + + notification = self.mox.CreateMockAnything() + notification.created_at = values['created_at'] + notification.deleted_at = values['deleted_at'] + notification.owner = values['owner'] + notification.uuid = values['uuid'] + notification.size = values['size'] + notification.raw = values['raw'] + notification.audit_period_beginning = values['audit_period_beginning'] + notification.audit_period_ending = values['audit_period_ending'] + created_at_range = (DECIMAL_DUMMY_TIME, DECIMAL_DUMMY_TIME+1) + views.STACKDB.get_image_usage(created_at__range=created_at_range, + uuid=notification.uuid).AndReturn(None) + views.STACKDB.get_image_delete(created_at__range=created_at_range, + uuid=notification.uuid).AndReturn(delete) + views.STACKDB.create_image_exists(**values) + + self.mox.ReplayAll() + + views._process_glance_exists(raw, notification) + self.mox.VerifyAll() + + # def test_process_image_exists_should_not_populate_delete_and_deleted_at_when_deleted_at_is_absent(self): + diff --git a/tests/unit/test_worker.py b/tests/unit/test_worker.py index f21c227..a7db64b 100644 --- a/tests/unit/test_worker.py +++ b/tests/unit/test_worker.py @@ -128,8 +128,9 @@ class ConsumerTestCase(unittest.TestCase): body_dict = {u'key': u'value'} message.body = json.dumps(body_dict) + mock_notification = self.mox.CreateMockAnything() mock_post_process_method = self.mox.CreateMockAnything() - mock_post_process_method(raw, body_dict) + mock_post_process_method(raw, mock_notification) old_handler = worker.POST_PROCESS_METHODS worker.POST_PROCESS_METHODS["RawData"] = mock_post_process_method @@ -137,7 +138,7 @@ class ConsumerTestCase(unittest.TestCase): use_mock_anything=True) args = (routing_key, body_dict) views.process_raw_data(deployment, args, json.dumps(args), exchange) \ - .AndReturn(raw) + .AndReturn((raw, mock_notification)) message.ack() self.mox.StubOutWithMock(consumer, '_check_memory', @@ -149,31 +150,6 @@ class ConsumerTestCase(unittest.TestCase): self.mox.VerifyAll() worker.POST_PROCESS_METHODS["RawData"] = old_handler - def test_process_no_raw_dont_ack(self): - deployment = self.mox.CreateMockAnything() - raw = self.mox.CreateMockAnything() - message = self.mox.CreateMockAnything() - - exchange = 'nova' - consumer = worker.Consumer('test', None, deployment, True, {}, - exchange, ["monitor.info", "monitor.error"]) - routing_key = 'monitor.info' - message.delivery_info = {'routing_key': routing_key} - body_dict = {u'key': u'value'} - message.body = json.dumps(body_dict) - self.mox.StubOutWithMock(views, 'process_raw_data', - use_mock_anything=True) - args = (routing_key, body_dict) - views.process_raw_data(deployment, args, json.dumps(args), exchange) \ - .AndReturn(None) - self.mox.StubOutWithMock(consumer, '_check_memory', - use_mock_anything=True) - consumer._check_memory() - self.mox.ReplayAll() - consumer._process(message) - self.assertEqual(consumer.processed, 0) - self.mox.VerifyAll() - def test_run(self): config = { 'name': 'east_coast.prod.global', diff --git a/tests/unit/utils.py b/tests/unit/utils.py index 437e169..222b427 100644 --- a/tests/unit/utils.py +++ b/tests/unit/utils.py @@ -25,6 +25,8 @@ TENANT_ID_2 = 'testtenantid2' from stacktach import datetime_to_decimal as dt +IMAGE_UUID_1 = "1" + INSTANCE_ID_1 = "08f685d9-6352-4dbc-8271-96cc54bf14cd" INSTANCE_ID_2 = "515adf96-41d3-b86d-5467-e584edc61dab" @@ -32,6 +34,7 @@ INSTANCE_TYPE_ID_1 = "12345" INSTANCE_TYPE_ID_2 = '54321' DUMMY_TIME = datetime.datetime.utcnow() +DECIMAL_DUMMY_TIME = dt.dt_to_decimal(DUMMY_TIME) MESSAGE_ID_1 = "7f28f81b-29a2-43f2-9ba1-ccb3e53ab6c8" MESSAGE_ID_2 = "4d596126-0f04-4329-865f-7b9a7bd69bcf" diff --git a/worker/worker.py b/worker/worker.py index 11efb98..e542b9c 100644 --- a/worker/worker.py +++ b/worker/worker.py @@ -34,7 +34,9 @@ except ImportError: from pympler.process import ProcessMemoryInfo +from django import db as django_db from stacktach import db +from stacktach import notification from stacktach import stacklog from stacktach import views @@ -84,12 +86,12 @@ class Consumer(kombu.mixins.ConsumerMixin): args = (routing_key, json.loads(body)) asJson = json.dumps(args) # save raw and ack the message - raw = views.process_raw_data(self.deployment, args, asJson, self.exchange) + raw, notif = views.process_raw_data( + self.deployment, args, asJson, self.exchange) - if raw: - self.processed += 1 - message.ack() - POST_PROCESS_METHODS[raw.get_name()](raw, args[1]) + self.processed += 1 + message.ack() + POST_PROCESS_METHODS[raw.get_name()](raw, notif) self._check_memory() From 32a73158ef63091706ead128558223e372dfad49 Mon Sep 17 00:00:00 2001 From: Anuj Mathur Date: Wed, 3 Jul 2013 15:57:07 +0530 Subject: [PATCH 04/11] Extracted save method for image_exists to glance_notification --- ...reate_glance_usage_verification_tables.py} | 87 +++ stacktach/models.py | 15 +- stacktach/notification.py | 95 ++- stacktach/views.py | 121 ++-- tests/unit/test_notification.py | 240 +++++++- tests/unit/test_stacktach.py | 580 +++++++++--------- worker/worker.py | 2 - 7 files changed, 745 insertions(+), 395 deletions(-) rename stacktach/migrations/{0005_create_glancerawdata_and_genericrawdata.py => 0005_create_glance_usage_verification_tables.py} (73%) diff --git a/stacktach/migrations/0005_create_glancerawdata_and_genericrawdata.py b/stacktach/migrations/0005_create_glance_usage_verification_tables.py similarity index 73% rename from stacktach/migrations/0005_create_glancerawdata_and_genericrawdata.py rename to stacktach/migrations/0005_create_glance_usage_verification_tables.py index 172c5bc..34aafeb 100644 --- a/stacktach/migrations/0005_create_glancerawdata_and_genericrawdata.py +++ b/stacktach/migrations/0005_create_glance_usage_verification_tables.py @@ -8,6 +8,18 @@ from django.db import models class Migration(SchemaMigration): def forwards(self, orm): + # Adding model 'ImageDeletes' + db.create_table(u'stacktach_imagedeletes', ( + (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('uuid', self.gf('django.db.models.fields.CharField')(max_length=50, db_index=True)), + ('created_at', self.gf('django.db.models.fields.DecimalField')(max_digits=20, decimal_places=6, db_index=True)), + ('deleted_at', self.gf('django.db.models.fields.DecimalField')(null=True, max_digits=20, decimal_places=6, db_index=True)), + ('owner', self.gf('django.db.models.fields.CharField')(max_length=50, db_index=True)), + ('size', self.gf('django.db.models.fields.BigIntegerField')(max_length=20)), + ('raw', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['stacktach.GlanceRawData'])), + )) + db.send_create_signal(u'stacktach', ['ImageDeletes']) + # Adding model 'GlanceRawData' db.create_table(u'stacktach_glancerawdata', ( (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), @@ -28,6 +40,17 @@ class Migration(SchemaMigration): )) db.send_create_signal(u'stacktach', ['GlanceRawData']) + # Adding model 'ImageUsage' + db.create_table(u'stacktach_imageusage', ( + (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('uuid', self.gf('django.db.models.fields.CharField')(max_length=50, db_index=True)), + ('created_at', self.gf('django.db.models.fields.DecimalField')(max_digits=20, decimal_places=6, db_index=True)), + ('owner', self.gf('django.db.models.fields.CharField')(max_length=50, db_index=True)), + ('size', self.gf('django.db.models.fields.BigIntegerField')(max_length=20)), + ('last_raw', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['stacktach.GlanceRawData'])), + )) + db.send_create_signal(u'stacktach', ['ImageUsage']) + # Adding model 'GenericRawData' db.create_table(u'stacktach_genericrawdata', ( (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), @@ -46,14 +69,42 @@ class Migration(SchemaMigration): )) db.send_create_signal(u'stacktach', ['GenericRawData']) + # Adding model 'ImageExists' + db.create_table(u'stacktach_imageexists', ( + (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('uuid', self.gf('django.db.models.fields.CharField')(max_length=50, db_index=True)), + ('created_at', self.gf('django.db.models.fields.DecimalField')(null=True, max_digits=20, decimal_places=6, db_index=True)), + ('deleted_at', self.gf('django.db.models.fields.DecimalField')(null=True, max_digits=20, decimal_places=6, db_index=True)), + ('audit_period_beginning', self.gf('django.db.models.fields.DecimalField')(max_digits=20, decimal_places=6, db_index=True)), + ('audit_period_ending', self.gf('django.db.models.fields.DecimalField')(max_digits=20, decimal_places=6, db_index=True)), + ('status', self.gf('django.db.models.fields.CharField')(default='pending', max_length=50, db_index=True)), + ('fail_reason', self.gf('django.db.models.fields.CharField')(max_length=300, null=True)), + ('raw', self.gf('django.db.models.fields.related.ForeignKey')(related_name='+', to=orm['stacktach.GlanceRawData'])), + ('usage', self.gf('django.db.models.fields.related.ForeignKey')(related_name='+', null=True, to=orm['stacktach.ImageUsage'])), + ('delete', self.gf('django.db.models.fields.related.ForeignKey')(related_name='+', null=True, to=orm['stacktach.ImageDeletes'])), + ('send_status', self.gf('django.db.models.fields.IntegerField')(default=0, db_index=True)), + ('owner', self.gf('django.db.models.fields.CharField')(max_length=255, db_index=True)), + ('size', self.gf('django.db.models.fields.BigIntegerField')(max_length=20)), + )) + db.send_create_signal(u'stacktach', ['ImageExists']) + def backwards(self, orm): + # Deleting model 'ImageDeletes' + db.delete_table(u'stacktach_imagedeletes') + # Deleting model 'GlanceRawData' db.delete_table(u'stacktach_glancerawdata') + # Deleting model 'ImageUsage' + db.delete_table(u'stacktach_imageusage') + # Deleting model 'GenericRawData' db.delete_table(u'stacktach_genericrawdata') + # Deleting model 'ImageExists' + db.delete_table(u'stacktach_imageexists') + models = { u'stacktach.deployment': { @@ -95,6 +146,42 @@ class Migration(SchemaMigration): 'uuid': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '36', 'null': 'True', 'blank': 'True'}), 'when': ('django.db.models.fields.DecimalField', [], {'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}) }, + u'stacktach.imagedeletes': { + 'Meta': {'object_name': 'ImageDeletes'}, + 'created_at': ('django.db.models.fields.DecimalField', [], {'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}), + 'deleted_at': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'owner': ('django.db.models.fields.CharField', [], {'max_length': '50', 'db_index': 'True'}), + 'raw': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['stacktach.GlanceRawData']"}), + 'size': ('django.db.models.fields.BigIntegerField', [], {'max_length': '20'}), + 'uuid': ('django.db.models.fields.CharField', [], {'max_length': '50', 'db_index': 'True'}) + }, + u'stacktach.imageexists': { + 'Meta': {'object_name': 'ImageExists'}, + 'audit_period_beginning': ('django.db.models.fields.DecimalField', [], {'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}), + 'audit_period_ending': ('django.db.models.fields.DecimalField', [], {'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}), + 'created_at': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}), + 'delete': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'null': 'True', 'to': u"orm['stacktach.ImageDeletes']"}), + 'deleted_at': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}), + 'fail_reason': ('django.db.models.fields.CharField', [], {'max_length': '300', 'null': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'owner': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}), + 'raw': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'to': u"orm['stacktach.GlanceRawData']"}), + 'send_status': ('django.db.models.fields.IntegerField', [], {'default': '0', 'db_index': 'True'}), + 'size': ('django.db.models.fields.BigIntegerField', [], {'max_length': '20'}), + 'status': ('django.db.models.fields.CharField', [], {'default': "'pending'", 'max_length': '50', 'db_index': 'True'}), + 'usage': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'null': 'True', 'to': u"orm['stacktach.ImageUsage']"}), + 'uuid': ('django.db.models.fields.CharField', [], {'max_length': '50', 'db_index': 'True'}) + }, + u'stacktach.imageusage': { + 'Meta': {'object_name': 'ImageUsage'}, + 'created_at': ('django.db.models.fields.DecimalField', [], {'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'last_raw': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['stacktach.GlanceRawData']"}), + 'owner': ('django.db.models.fields.CharField', [], {'max_length': '50', 'db_index': 'True'}), + 'size': ('django.db.models.fields.BigIntegerField', [], {'max_length': '20'}), + 'uuid': ('django.db.models.fields.CharField', [], {'max_length': '50', 'db_index': 'True'}) + }, u'stacktach.instancedeletes': { 'Meta': {'object_name': 'InstanceDeletes'}, 'deleted_at': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}), diff --git a/stacktach/models.py b/stacktach/models.py index 4876313..5119db3 100644 --- a/stacktach/models.py +++ b/stacktach/models.py @@ -316,7 +316,8 @@ class ImageDeletes(models.Model): created_at = models.DecimalField(max_digits=20, decimal_places=6, db_index=True) deleted_at = models.DecimalField(max_digits=20, - decimal_places=6, db_index=True) + decimal_places=6, db_index=True, + null=True) owner = models.CharField(max_length=50, db_index=True) size = models.BigIntegerField(max_length=20) raw = models.ForeignKey(GlanceRawData) @@ -336,9 +337,11 @@ class ImageExists(models.Model): uuid = models.CharField(max_length=50, db_index=True) created_at = models.DecimalField(max_digits=20, - decimal_places=6, db_index=True) + decimal_places=6, db_index=True, + null=True) deleted_at = models.DecimalField(max_digits=20, - decimal_places=6, db_index=True) + decimal_places=6, db_index=True, + null=True) audit_period_beginning = models.DecimalField(max_digits=20, decimal_places=6, db_index=True) @@ -347,10 +350,10 @@ class ImageExists(models.Model): status = models.CharField(max_length=50, db_index=True, choices=STATUS_CHOICES, default=PENDING) - fail_reason = models.CharField(max_length=300, db_index=True, null=True) + fail_reason = models.CharField(max_length=300, null=True) raw = models.ForeignKey(GlanceRawData, related_name='+') - usage = models.ForeignKey(ImageUsage, related_name='+') - delete = models.ForeignKey(ImageDeletes, related_name='+') + usage = models.ForeignKey(ImageUsage, related_name='+', null=True) + delete = models.ForeignKey(ImageDeletes, related_name='+', null=True) send_status = models.IntegerField(default=0, db_index=True) owner = models.CharField(max_length=255, db_index=True) size = models.BigIntegerField(max_length=20) diff --git a/stacktach/notification.py b/stacktach/notification.py index b69f76a..656fb5f 100644 --- a/stacktach/notification.py +++ b/stacktach/notification.py @@ -18,6 +18,7 @@ # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. from stacktach import utils +from stacktach import stacklog from stacktach import image_type from stacktach import db @@ -93,7 +94,17 @@ class GlanceNotification(Notification): self.image_type = image_type.get_numeric_code(self.payload) self.status = self.payload.get('status', None) self.uuid = self.payload.get('id', None) - + self.size = self.payload.get('size', None) + created_at = self.payload.get('created_at', None) + self.created_at = created_at and utils.str_time_to_unix(created_at) + audit_period_beginning = self.payload.get( + 'audit_period_beginning', None) + self.audit_period_beginning = audit_period_beginning and\ + utils.str_time_to_unix(audit_period_beginning) + audit_period_ending = self.payload.get( + 'audit_period_ending', None) + self.audit_period_ending = audit_period_ending and \ + utils.str_time_to_unix(audit_period_ending) @property def owner(self): @@ -102,39 +113,81 @@ class GlanceNotification(Notification): @property def instance(self): return self.properties.get('instance_uuid', None) + @property + def deleted_at(self): + deleted_at = self.body.get('deleted_at', None) + deleted_at = deleted_at or self.payload.get('deleted_at', None) + return deleted_at and utils.str_time_to_unix(deleted_at) def save(self): - db.create_glance_rawdata(deployment=self.deployment, - routing_key=self.routing_key, - owner=self.owner, - json=self.json, - when=self.when, - publisher=self.publisher, - event=self.event, - service=self.service, - host=self.host, - instance=self.instance, - request_id=self.request_id, - image_type=self.image_type, - status=self.status, - uuid=self.uuid) + return db.create_glance_rawdata(deployment=self.deployment, + routing_key=self.routing_key, + owner=self.owner, + json=self.json, + when=self.when, + publisher=self.publisher, + event=self.event, + service=self.service, + host=self.host, + instance=self.instance, + request_id=self.request_id, + image_type=self.image_type, + status=self.status, + uuid=self.uuid) + + def save_exists(self, raw): + if self.created_at: + values = { + 'uuid': self.uuid, + 'audit_period_beginning': self.audit_period_beginning, + 'audit_period_ending': self.audit_period_ending, + 'owner': self.owner, + 'size': self.size, + 'raw': raw + } + created_at_range = (self.created_at, self.created_at+1) + usage = db.get_image_usage( + uuid=self.uuid, created_at__range=created_at_range) + values['usage'] = usage + values['created_at'] = self.created_at + if self.deleted_at: + delete = db.get_image_delete( + uuid=self.uuid, created_at__range=created_at_range) + values['delete'] = delete + values['deleted_at'] = self.deleted_at + + db.create_image_exists(**values) + else: + stacklog.warn("Ignoring exists without created_at. GlanceRawData(%s)" + % raw.id) class NovaNotification(Notification): def __init__(self, body, deployment, routing_key, json): super(NovaNotification, self).__init__(body, deployment, routing_key, json) - self.state = self.payload.get('state', "") - self.old_state = self.payload.get('old_state', "") - self.old_task = self.payload.get('old_task_state', "") - self.task = self.payload.get('new_task_state', "") + self.state = self.payload.get('state', '') + self.old_state = self.payload.get('old_state', '') + self.old_task = self.payload.get('old_task_state', '') + self.task = self.payload.get('new_task_state', '') self.image_type = image_type.get_numeric_code(self.payload) image_meta = self.payload.get('image_meta', {}) - self.os_architecture = image_meta.get('org.openstack__1__architecture', - '') + self.os_architecture = \ + image_meta.get('org.openstack__1__architecture', '') self.os_distro = image_meta.get('org.openstack__1__os_distro', '') self.os_version = image_meta.get('org.openstack__1__os_version', '') self.rax_options = image_meta.get('com.rackspace__1__options', '') + self.instance_type_id = self.payload.get('instance_type_id', None) + self.new_instance_type_id = \ + self.payload.get('new_instance_type_id', None) + self.launched_at = self.payload.get('launched_at', None) + self.deleted_at = self.payload.get('deleted_at', None) + self.audit_period_beginning = self.payload.get( + 'audit_period_beginning', None) + self.audit_period_ending = self.payload.get( + 'audit_period_ending', None) + self.message = self.payload.get('message', None) + self.message_id = self.body.get('message_id', None) @property def host(self): diff --git a/stacktach/views.py b/stacktach/views.py index 386eee8..8e419bc 100644 --- a/stacktach/views.py +++ b/stacktach/views.py @@ -163,17 +163,16 @@ INSTANCE_EVENT = { } -def _process_usage_for_new_launch(raw, body): - payload = body['payload'] +def _process_usage_for_new_launch(raw, notification): values = {} - values['instance'] = payload['instance_id'] - values['request_id'] = body['_context_request_id'] + values['instance'] = notification.instance + values['request_id'] = notification.request_id (usage, new) = STACKDB.get_or_create_instance_usage(**values) if raw.event in [INSTANCE_EVENT['create_start'], INSTANCE_EVENT['rebuild_start']]: - usage.instance_type_id = payload['instance_type_id'] + usage.instance_type_id = notification.instance_type_id if raw.event in [INSTANCE_EVENT['rebuild_start'], INSTANCE_EVENT['resize_prep_start'], @@ -183,27 +182,23 @@ def _process_usage_for_new_launch(raw, body): # we will have a launch record corresponding to the exists. # We don't want to override a launched_at if it is already set # though, because we may have already received the end event - usage.launched_at = utils.str_time_to_unix(payload['launched_at']) + usage.launched_at = utils.str_time_to_unix(notification.launched_at) - usage.tenant = payload['tenant_id'] - image_meta = payload.get('image_meta', {}) - usage.rax_options = image_meta.get('com.rackspace__1__options', '') - usage.os_architecture = image_meta.get('org.openstack__1__architecture', - '') - usage.os_version = image_meta.get('org.openstack__1__os_version', '') - usage.os_distro = image_meta.get('org.openstack__1__os_distro', '') + usage.tenant = notification.tenant + usage.rax_options = notification.rax_options + usage.os_architecture = notification.os_architecture + usage.os_version = notification.os_version + usage.os_distro = notification.os_distro STACKDB.save(usage) -def _process_usage_for_updates(raw, body): - payload = body['payload'] - +def _process_usage_for_updates(raw, notification): if raw.event == INSTANCE_EVENT['create_end']: - if 'message' in payload and payload['message'] != 'Success': + if notification.message and notification.message != 'Success': return - instance_id = payload['instance_id'] - request_id = body['_context_request_id'] + instance_id = notification.instance + request_id = notification.request_id (usage, new) = STACKDB.get_or_create_instance_usage(instance=instance_id, request_id=request_id) @@ -211,28 +206,25 @@ def _process_usage_for_updates(raw, body): INSTANCE_EVENT['rebuild_end'], INSTANCE_EVENT['resize_finish_end'], INSTANCE_EVENT['resize_revert_end']]: - usage.launched_at = utils.str_time_to_unix(payload['launched_at']) + usage.launched_at = utils.str_time_to_unix(notification.launched_at) if raw.event == INSTANCE_EVENT['resize_revert_end']: - usage.instance_type_id = payload['instance_type_id'] + usage.instance_type_id = notification.instance_type_id elif raw.event == INSTANCE_EVENT['resize_prep_end']: - usage.instance_type_id = payload['new_instance_type_id'] + usage.instance_type_id = notification.new_instance_type_id - usage.tenant = payload['tenant_id'] - image_meta = payload.get('image_meta', {}) - usage.rax_options = image_meta.get('com.rackspace__1__options', '') - usage.os_architecture = image_meta.get('org.openstack__1__architecture', - '') - usage.os_version = image_meta.get('org.openstack__1__os_version', '') - usage.os_distro = image_meta.get('org.openstack__1__os_distro', '') + usage.tenant = notification.tenant + usage.rax_options = notification.rax_options + usage.os_architecture = notification.os_architecture + usage.os_version = notification.os_version + usage.os_distro = notification.os_distro STACKDB.save(usage) -def _process_delete(raw, body): - payload = body['payload'] - instance_id = payload['instance_id'] - deleted_at = utils.str_time_to_unix(payload['deleted_at']) +def _process_delete(raw, notification): + instance_id = notification.instance + deleted_at = utils.str_time_to_unix(notification.deleted_at) values = { 'instance': instance_id, 'deleted_at': deleted_at, @@ -240,7 +232,7 @@ def _process_delete(raw, body): (delete, new) = STACKDB.get_or_create_instance_delete(**values) delete.raw = raw - launched_at = payload.get('launched_at') + launched_at = notification.launched_at if launched_at and launched_at != '': launched_at = utils.str_time_to_unix(launched_at) delete.launched_at = launched_at @@ -248,35 +240,33 @@ def _process_delete(raw, body): STACKDB.save(delete) -def _process_exists(raw, body): - payload = body['payload'] - instance_id = payload['instance_id'] - launched_at_str = payload.get('launched_at') +def _process_exists(raw, notification): + instance_id = notification.instance + launched_at_str = notification.launched_at if launched_at_str is not None and launched_at_str != '': - launched_at = utils.str_time_to_unix(payload['launched_at']) + launched_at = utils.str_time_to_unix(notification.launched_at) launched_range = (launched_at, launched_at+1) usage = STACKDB.get_instance_usage(instance=instance_id, launched_at__range=launched_range) values = {} - values['message_id'] = body['message_id'] + values['message_id'] = notification.message_id values['instance'] = instance_id values['launched_at'] = launched_at - beginning = utils.str_time_to_unix(payload['audit_period_beginning']) + beginning = utils.str_time_to_unix(notification.audit_period_beginning) values['audit_period_beginning'] = beginning - ending = utils.str_time_to_unix(payload['audit_period_ending']) + ending = utils.str_time_to_unix(notification.audit_period_ending) values['audit_period_ending'] = ending - values['instance_type_id'] = payload['instance_type_id'] + values['instance_type_id'] = notification.instance_type_id if usage: values['usage'] = usage values['raw'] = raw - values['tenant'] = payload['tenant_id'] - image_meta = payload.get('image_meta', {}) - values['rax_options'] = image_meta.get('com.rackspace__1__options', '') - values['os_architecture'] = image_meta.get('org.openstack__1__architecture', '') - values['os_version'] = image_meta.get('org.openstack__1__os_version', '') - values['os_distro'] = image_meta.get('org.openstack__1__os_distro', '') + values['tenant'] = notification.tenant + values['rax_options'] = notification.rax_options + values['os_architecture'] = notification.os_architecture + values['os_version'] = notification.os_version + values['os_distro'] = notification.os_distro - deleted_at = payload.get('deleted_at') + deleted_at = notification.deleted_at if deleted_at and deleted_at != '': # We only want to pre-populate the 'delete' if we know this is in # fact an exist event for a deleted instance. Otherwise, there @@ -319,30 +309,7 @@ def _process_glance_delete(raw, notification): def _process_glance_exists(raw, notification): - if notification.created_at: - values = { - 'uuid': notification.uuid, - 'audit_period_beginning': notification.audit_period_beginning, - 'audit_period_ending': notification.audit_period_ending, - 'owner': notification.owner, - 'size': notification.size, - 'raw': raw, - } - created_at_range = (notification.created_at, notification.created_at+1) - usage = STACKDB.get_image_usage(uuid=notification.uuid, - created_at__range=created_at_range) - values['usage'] = usage - values['created_at'] = notification.created_at - if notification.deleted_at: - delete = STACKDB.get_image_delete(uuid=notification.uuid, - created_at__range=created_at_range) - values['delete'] = delete - values['deleted_at'] = notification.deleted_at - - STACKDB.create_image_exists(**values) - else: - stacklog.warn("Ignoring exists without created_at. GlanceRawData(%s)" - % raw.id) + notification.save_exists(raw) USAGE_PROCESS_MAPPING = { INSTANCE_EVENT['create_start']: _process_usage_for_new_launch, @@ -359,18 +326,18 @@ USAGE_PROCESS_MAPPING = { } GLANCE_USAGE_PROCESS_MAPPING = { - 'image.activate': _process_glance_usage, + 'image.upload': _process_glance_usage, 'image.delete': _process_glance_delete, 'image.exists': _process_glance_exists } -def aggregate_usage(raw, body): +def aggregate_usage(raw, notification): if not raw.instance: return if raw.event in USAGE_PROCESS_MAPPING: - USAGE_PROCESS_MAPPING[raw.event](raw, body) + USAGE_PROCESS_MAPPING[raw.event](raw, notification) def aggregate_glance_usage(raw, body): diff --git a/tests/unit/test_notification.py b/tests/unit/test_notification.py index b1dffe3..0347cbf 100644 --- a/tests/unit/test_notification.py +++ b/tests/unit/test_notification.py @@ -22,12 +22,16 @@ import unittest import mox -from stacktach import notification, utils +from stacktach import notification +from stacktach import utils from stacktach.notification import Notification +from stacktach.notification import NovaNotification from stacktach.notification import GlanceNotification from stacktach import db from tests.unit.utils import REQUEST_ID_1 +from tests.unit.utils import DECIMAL_DUMMY_TIME +from tests.unit.utils import DUMMY_TIME from tests.unit.utils import TIMESTAMP_1 from tests.unit.utils import TENANT_ID_1 from tests.unit.utils import INSTANCE_ID_1 @@ -85,6 +89,65 @@ class NovaNotificationTestCase(unittest.TestCase): 'unknown_exchange') self.mox.VerifyAll() + def test_save_should_persist_nova_rawdata_to_database(self): + body = { + "event_type": "compute.instance.exists", + '_context_request_id': REQUEST_ID_1, + '_context_project_id': TENANT_ID_1, + "timestamp": TIMESTAMP_1, + "publisher_id": "compute.global.preprod-ord.ohthree.com", + "payload": { + 'instance_id': INSTANCE_ID_1, + "status": "saving", + "container_format": "ovf", + "properties": { + "image_type": "snapshot", + }, + "tenant": "5877054", + "old_state": 'old_state', + "old_task_state": 'old_task', + "image_meta": { + "org.openstack__1__architecture": 'os_arch', + "org.openstack__1__os_distro": 'os_distro', + "org.openstack__1__os_version": 'os_version', + "com.rackspace__1__options": 'rax_opt', + }, + "state": 'state', + "new_task_state": 'task' + } + } + deployment = "1" + routing_key = "monitor.info" + json = '{["routing_key", {%s}]}' % body + raw = self.mox.CreateMockAnything() + self.mox.StubOutWithMock(db, 'create_nova_rawdata') + db.create_nova_rawdata( + deployment="1", + tenant=TENANT_ID_1, + json=json, + routing_key=routing_key, + when=utils.str_time_to_unix(TIMESTAMP_1), + publisher="compute.global.preprod-ord.ohthree.com", + event="compute.instance.exists", + service="compute", + host="global.preprod-ord.ohthree.com", + instance=INSTANCE_ID_1, + request_id=REQUEST_ID_1, + old_state='old_state', + old_task='old_task', + os_architecture='os_arch', + os_distro='os_distro', + os_version='os_version', + rax_options='rax_opt', + state='state', + task='task').AndReturn(raw) + + self.mox.ReplayAll() + + notification = NovaNotification(body, deployment, routing_key, json) + self.assertEquals(notification.save(), raw) + self.mox.VerifyAll() + class GlanceNotificationTestCase(unittest.TestCase): def setUp(self): @@ -111,6 +174,7 @@ class GlanceNotificationTestCase(unittest.TestCase): deployment = "1" routing_key = "glance_monitor.info" json = '{["routing_key", {%s}]}' % body + raw = self.mox.CreateMockAnything() self.mox.StubOutWithMock(db, 'create_glance_rawdata') db.create_glance_rawdata( deployment="1", @@ -126,13 +190,178 @@ class GlanceNotificationTestCase(unittest.TestCase): request_id=None, image_type=0, status="saving", - uuid="2df2ccf6-bc1b-4853-aab0-25fda346b3bb") + uuid="2df2ccf6-bc1b-4853-aab0-25fda346b3bb").AndReturn(raw) self.mox.ReplayAll() notification = GlanceNotification(body, deployment, routing_key, json) - notification.save() + self.assertEquals(notification.save(), raw) + self.mox.VerifyAll() + + def test_process_image_exists(self): + usage = self.mox.CreateMockAnything() + raw = self.mox.CreateMockAnything() + audit_period_beginning = "2013-05-20 17:31:57.939614" + audit_period_ending = "2013-06-20 17:31:57.939614" + size = 123 + uuid = "2df2ccf6-bc1b-4853-aab0-25fda346b3bb" + body = { + "event_type": "image.upload", + "timestamp": "2013-06-20 18:31:57.939614", + "publisher_id": "glance-api01-r2961.global.preprod-ord.ohthree.com", + "payload": { + "created_at": str(DUMMY_TIME), + "status": "saving", + "audit_period_beginning": audit_period_beginning, + "audit_period_ending": audit_period_ending, + "properties": { + "image_type": "snapshot", + "instance_uuid": INSTANCE_ID_1, + }, + "size": size, + "owner": TENANT_ID_1, + "id": uuid + } + } + deployment = "1" + routing_key = "glance_monitor.info" + json = '{["routing_key", {%s}]}' % body + + self.mox.StubOutWithMock(db, 'create_image_exists') + self.mox.StubOutWithMock(db, 'get_image_usage') + + created_at_range = (DECIMAL_DUMMY_TIME, DECIMAL_DUMMY_TIME+1) + db.get_image_usage(created_at__range=created_at_range, + uuid=uuid).AndReturn(None) + db.create_image_exists( + created_at=utils.str_time_to_unix(str(DUMMY_TIME)), + owner=TENANT_ID_1, + raw=raw, + audit_period_beginning=utils.str_time_to_unix(audit_period_beginning), + audit_period_ending=utils.str_time_to_unix(audit_period_ending), + size=size, + uuid=uuid, + usage=None).AndReturn(raw) + + self.mox.ReplayAll() + + notification = GlanceNotification(body, deployment, routing_key, + json) + notification.save_exists(raw) + self.mox.VerifyAll() + + def test_process_image_exists_with_delete_not_none(self): + raw = self.mox.CreateMockAnything() + usage = self.mox.CreateMockAnything() + delete = self.mox.CreateMockAnything() + audit_period_beginning = "2013-05-20 17:31:57.939614" + audit_period_ending = "2013-06-20 17:31:57.939614" + size = 123 + uuid = "2df2ccf6-bc1b-4853-aab0-25fda346b3bb" + deleted_at = "2013-06-20 14:31:57.939614" + body = { + "event_type": "image.upload", + "timestamp": "2013-06-20 18:31:57.939614", + "publisher_id": "glance-api01-r2961.global.preprod-ord.ohthree.com", + "payload": { + "created_at": str(DUMMY_TIME), + "status": "saving", + "audit_period_beginning": audit_period_beginning, + "audit_period_ending": audit_period_ending, + "properties": { + "image_type": "snapshot", + "instance_uuid": INSTANCE_ID_1, + }, + "deleted_at": deleted_at, + "size": size, + "owner": TENANT_ID_1, + "id": "2df2ccf6-bc1b-4853-aab0-25fda346b3bb", + } + } + deployment = "1" + routing_key = "glance_monitor.info" + json = '{["routing_key", {%s}]}' % body + + self.mox.StubOutWithMock(db, 'create_image_exists') + self.mox.StubOutWithMock(db, 'get_image_usage') + self.mox.StubOutWithMock(db, 'get_image_delete') + + created_at_range = (DECIMAL_DUMMY_TIME, DECIMAL_DUMMY_TIME+1) + db.get_image_usage(created_at__range=created_at_range, + uuid=uuid).AndReturn(None) + db.get_image_delete(created_at__range=created_at_range, + uuid=uuid).AndReturn(delete) + db.create_image_exists( + created_at=utils.str_time_to_unix(str(DUMMY_TIME)), + owner=TENANT_ID_1, + raw=raw, + audit_period_beginning=utils.str_time_to_unix(audit_period_beginning), + audit_period_ending=utils.str_time_to_unix(audit_period_ending), + size=size, + uuid=uuid, + usage=None, + delete=delete, + deleted_at=utils.str_time_to_unix(str(deleted_at))).AndReturn(raw) + + self.mox.ReplayAll() + + notification = GlanceNotification(body, deployment, routing_key, + json) + notification.save_exists(raw) + self.mox.VerifyAll() + + def test_process_image_exists_with_usage_not_none(self): + raw = self.mox.CreateMockAnything() + usage = self.mox.CreateMockAnything() + audit_period_beginning = "2013-05-20 17:31:57.939614" + audit_period_ending = "2013-06-20 17:31:57.939614" + size = 123 + uuid = "2df2ccf6-bc1b-4853-aab0-25fda346b3bb" + body = { + "event_type": "image.upload", + "timestamp": "2013-06-20 18:31:57.939614", + "publisher_id": "glance-api01-r2961.global.preprod-ord.ohthree.com", + "payload": { + "created_at": str(DUMMY_TIME), + "status": "saving", + "audit_period_beginning": audit_period_beginning, + "audit_period_ending": audit_period_ending, + "properties": { + "image_type": "snapshot", + "instance_uuid": INSTANCE_ID_1, + }, + "size": size, + "owner": TENANT_ID_1, + "id": "2df2ccf6-bc1b-4853-aab0-25fda346b3bb", + } + } + deployment = "1" + routing_key = "glance_monitor.info" + json = '{["routing_key", {%s}]}' % body + + self.mox.StubOutWithMock(db, 'create_image_exists') + self.mox.StubOutWithMock(db, 'get_image_usage') + self.mox.StubOutWithMock(db, 'get_image_delete') + + created_at_range = (DECIMAL_DUMMY_TIME, DECIMAL_DUMMY_TIME+1) + db.get_image_usage(created_at__range=created_at_range, + uuid=uuid).AndReturn(usage) + db.create_image_exists( + created_at=utils.str_time_to_unix(str(DUMMY_TIME)), + owner=TENANT_ID_1, + raw=raw, + audit_period_beginning=utils.str_time_to_unix(audit_period_beginning), + audit_period_ending=utils.str_time_to_unix(audit_period_ending), + size=size, + uuid=uuid, + usage=usage).AndReturn(raw) + + self.mox.ReplayAll() + + notification = GlanceNotification(body, deployment, routing_key, + json) + notification.save_exists(raw) self.mox.VerifyAll() @@ -163,6 +392,7 @@ class NotificationTestCase(unittest.TestCase): deployment = "1" routing_key = "generic_monitor.info" json = '{["routing_key", {%s}]}' % body + raw = self.mox.CreateMockAnything() self.mox.StubOutWithMock(db, 'create_generic_rawdata') db.create_generic_rawdata( deployment="1", @@ -175,10 +405,10 @@ class NotificationTestCase(unittest.TestCase): service="glance-api01-r2961", host="global.preprod-ord.ohthree.com", instance=INSTANCE_ID_1, - request_id=REQUEST_ID_1) + request_id=REQUEST_ID_1).AndReturn(raw) self.mox.ReplayAll() notification = Notification(body, deployment, routing_key, json) - notification.save() + self.assertEquals(notification.save(), raw) self.mox.VerifyAll() diff --git a/tests/unit/test_stacktach.py b/tests/unit/test_stacktach.py index 055d3d3..b493b65 100644 --- a/tests/unit/test_stacktach.py +++ b/tests/unit/test_stacktach.py @@ -26,7 +26,6 @@ import mox import utils from utils import INSTANCE_ID_1 -from utils import DECIMAL_DUMMY_TIME from utils import OS_VERSION_1 from utils import OS_ARCH_1 from utils import OS_DISTRO_1 @@ -305,15 +304,30 @@ class StacktachUsageParsingTestCase(unittest.TestCase): stacklog.get_logger(name=name).AndReturn(self.log) def test_process_usage_for_new_launch_create_start(self): - kwargs = {'launched': str(DUMMY_TIME), 'tenant_id': TENANT_ID_1, 'rax_options': RAX_OPTIONS_1, - 'os_architecture': OS_ARCH_1, 'os_version': OS_VERSION_1, 'os_distro': OS_DISTRO_1 } - notification = utils.create_nova_notif(request_id=REQUEST_ID_1, **kwargs) - event = 'compute.instance.create.start' - raw, usage = self._setup_process_usage_mocks(event, notification) + notification = self.mox.CreateMockAnything() + notification.launched_at = str(DUMMY_TIME) + notification.tenant = TENANT_ID_1 + notification.rax_options = RAX_OPTIONS_1 + notification.os_architecture = OS_ARCH_1 + notification.os_version = OS_VERSION_1 + notification.os_distro = OS_DISTRO_1 + notification.instance = INSTANCE_ID_1 + notification.request_id = REQUEST_ID_1 + notification.instance_type_id = INSTANCE_TYPE_ID_1 - views._process_usage_for_new_launch(raw, notification[1]) + raw = self.mox.CreateMockAnything() + raw.event = 'compute.instance.create.start' - self.assertEquals(usage.instance_type_id, '1') + usage = self.mox.CreateMockAnything() + views.STACKDB.get_or_create_instance_usage(instance=INSTANCE_ID_1, + request_id=REQUEST_ID_1) \ + .AndReturn((usage, True)) + views.STACKDB.save(usage) + self.mox.ReplayAll() + + views._process_usage_for_new_launch(raw, notification) + + self.assertEquals(usage.instance_type_id, INSTANCE_TYPE_ID_1) self.assertEquals(usage.tenant, TENANT_ID_1) self.assertEquals(usage.os_architecture, OS_ARCH_1) self.assertEquals(usage.os_version, OS_VERSION_1) @@ -323,15 +337,29 @@ class StacktachUsageParsingTestCase(unittest.TestCase): self.mox.VerifyAll() def test_process_usage_for_new_launch_rebuild_start(self): - kwargs = {'launched': str(DUMMY_TIME), 'tenant_id': TENANT_ID_1, 'rax_options': RAX_OPTIONS_1, - 'os_architecture': OS_ARCH_1, 'os_version': OS_VERSION_1, 'os_distro': OS_DISTRO_1 } - notification = utils.create_nova_notif(request_id=REQUEST_ID_1, **kwargs) - event = 'compute.instance.rebuild.start' - raw, usage = self._setup_process_usage_mocks(event, notification) + notification = self.mox.CreateMockAnything() + notification.launched_at = str(DUMMY_TIME) + notification.tenant = TENANT_ID_1 + notification.rax_options = RAX_OPTIONS_1 + notification.os_architecture = OS_ARCH_1 + notification.os_version = OS_VERSION_1 + notification.os_distro = OS_DISTRO_1 + notification.instance = INSTANCE_ID_1 + notification.request_id = REQUEST_ID_1 + notification.instance_type_id = INSTANCE_TYPE_ID_1 - views._process_usage_for_new_launch(raw, notification[1]) + raw = self.mox.CreateMockAnything() + raw.event = 'compute.instance.rebuild.start' + usage = self.mox.CreateMockAnything() + views.STACKDB.get_or_create_instance_usage(instance=INSTANCE_ID_1, + request_id=REQUEST_ID_1) \ + .AndReturn((usage, True)) + views.STACKDB.save(usage) + self.mox.ReplayAll() - self.assertEquals(usage.instance_type_id, '1') + views._process_usage_for_new_launch(raw, notification) + + self.assertEquals(usage.instance_type_id, INSTANCE_TYPE_ID_1) self.assertEquals(usage.tenant, TENANT_ID_1) self.assertEquals(usage.os_architecture, OS_ARCH_1) self.assertEquals(usage.os_version, OS_VERSION_1) @@ -340,14 +368,29 @@ class StacktachUsageParsingTestCase(unittest.TestCase): self.mox.VerifyAll() def test_process_usage_for_new_launch_rebuild_start_when_no_launched_at_in_db(self): - kwargs = {'launched': str(DUMMY_TIME), 'tenant_id': TENANT_ID_1, 'rax_options': RAX_OPTIONS_1, - 'os_architecture': OS_ARCH_1, 'os_version': OS_VERSION_1, 'os_distro': OS_DISTRO_1 } - notification = utils.create_nova_notif(request_id=REQUEST_ID_1, **kwargs) - event = 'compute.instance.rebuild.start' - raw, usage = self._setup_process_usage_mocks(event, notification) - usage.launched_at = None + notification = self.mox.CreateMockAnything() + notification.launched_at = str(DUMMY_TIME) + notification.tenant = TENANT_ID_1 + notification.rax_options = RAX_OPTIONS_1 + notification.os_architecture = OS_ARCH_1 + notification.os_version = OS_VERSION_1 + notification.os_distro = OS_DISTRO_1 + notification.instance = INSTANCE_ID_1 + notification.request_id = REQUEST_ID_1 + notification.instance_type_id = INSTANCE_TYPE_ID_1 - views._process_usage_for_new_launch(raw, notification[1]) + raw = self.mox.CreateMockAnything() + raw.event = 'compute.instance.rebuild.start' + + usage = self.mox.CreateMockAnything() + usage.launched_at = None + views.STACKDB.get_or_create_instance_usage(instance=INSTANCE_ID_1, + request_id=REQUEST_ID_1) \ + .AndReturn((usage, True)) + views.STACKDB.save(usage) + self.mox.ReplayAll() + + views._process_usage_for_new_launch(raw, notification) self.assertEqual(usage.launched_at, utils.decimal_utc(DUMMY_TIME)) self.assertEquals(usage.tenant, TENANT_ID_1) @@ -359,14 +402,31 @@ class StacktachUsageParsingTestCase(unittest.TestCase): self.mox.VerifyAll() def test_process_usage_for_new_launch_resize_prep_start_when_no_launched_at_in_db(self): - kwargs = {'launched': str(DUMMY_TIME), 'tenant_id': TENANT_ID_1, 'rax_options': RAX_OPTIONS_1, - 'os_architecture': OS_ARCH_1, 'os_version': OS_VERSION_1, 'os_distro': OS_DISTRO_1 } - notification = utils.create_nova_notif(request_id=REQUEST_ID_1, **kwargs) - event = 'compute.instance.resize.prep.start' - raw, usage = self._setup_process_usage_mocks(event, notification) + notification = self.mox.CreateMockAnything() + notification.launched_at = str(DUMMY_TIME) + notification.tenant = TENANT_ID_1 + notification.rax_options = RAX_OPTIONS_1 + notification.os_architecture = OS_ARCH_1 + notification.os_version = OS_VERSION_1 + notification.os_distro = OS_DISTRO_1 + notification.instance = INSTANCE_ID_1 + notification.request_id = REQUEST_ID_1 + notification.instance_type_id = INSTANCE_TYPE_ID_1 + + raw = self.mox.CreateMockAnything() + raw.event = 'compute.instance.resize.prep.start' + + usage = self.mox.CreateMockAnything() + usage.launched_at = None + views.STACKDB.get_or_create_instance_usage(instance=INSTANCE_ID_1, + request_id=REQUEST_ID_1) \ + .AndReturn((usage, True)) + views.STACKDB.save(usage) + self.mox.ReplayAll() + usage.launched_at = None - views._process_usage_for_new_launch(raw, notification[1]) + views._process_usage_for_new_launch(raw, notification) self.assertEqual(usage.launched_at, utils.decimal_utc(DUMMY_TIME)) self.assertEquals(usage.tenant, TENANT_ID_1) @@ -378,14 +438,29 @@ class StacktachUsageParsingTestCase(unittest.TestCase): self.mox.VerifyAll() def test_process_usage_for_new_launch_resize_revert_start_when_no_launched_at_in_db(self): - kwargs = {'launched': str(DUMMY_TIME), 'tenant_id': TENANT_ID_1,'rax_options': RAX_OPTIONS_1, - 'os_architecture': OS_ARCH_1, 'os_version': OS_VERSION_1, 'os_distro': OS_DISTRO_1 } - notification = utils.create_nova_notif(request_id=REQUEST_ID_1, **kwargs) - event = 'compute.instance.resize.revert.start' - raw, usage = self._setup_process_usage_mocks(event, notification) - usage.launched_at = None + notification = self.mox.CreateMockAnything() + notification.launched_at = str(DUMMY_TIME) + notification.tenant = TENANT_ID_1 + notification.rax_options = RAX_OPTIONS_1 + notification.os_architecture = OS_ARCH_1 + notification.os_version = OS_VERSION_1 + notification.os_distro = OS_DISTRO_1 + notification.instance = INSTANCE_ID_1 + notification.request_id = REQUEST_ID_1 + notification.instance_type_id = INSTANCE_TYPE_ID_1 - views._process_usage_for_new_launch(raw, notification[1]) + raw = self.mox.CreateMockAnything() + raw.event = 'compute.instance.resize.revert.start' + + usage = self.mox.CreateMockAnything() + usage.launched_at = None + views.STACKDB.get_or_create_instance_usage(instance=INSTANCE_ID_1, + request_id=REQUEST_ID_1) \ + .AndReturn((usage, True)) + views.STACKDB.save(usage) + self.mox.ReplayAll() + + views._process_usage_for_new_launch(raw, notification) self.assertEquals(usage.tenant, TENANT_ID_1) self.assertEqual(usage.launched_at, utils.decimal_utc(DUMMY_TIME)) @@ -397,17 +472,30 @@ class StacktachUsageParsingTestCase(unittest.TestCase): self.mox.VerifyAll() def test_process_usage_for_new_launch_resize_prep_start_when_launched_at_in_db(self): - kwargs = {'launched': str(DUMMY_TIME), 'tenant_id': TENANT_ID_1, - 'rax_options': RAX_OPTIONS_1, 'os_architecture': OS_ARCH_1, - 'os_version': OS_VERSION_1, 'os_distro': OS_DISTRO_1 } - notification = utils.create_nova_notif(request_id=REQUEST_ID_1, - **kwargs) - event = 'compute.instance.resize.prep.start' - raw, usage = self._setup_process_usage_mocks(event, notification) - orig_launched_at = utils.decimal_utc(DUMMY_TIME - datetime.timedelta(days=1)) - usage.launched_at = orig_launched_at + notification = self.mox.CreateMockAnything() + notification.launched_at = str(DUMMY_TIME) + notification.tenant = TENANT_ID_1 + notification.rax_options = RAX_OPTIONS_1 + notification.os_architecture = OS_ARCH_1 + notification.os_version = OS_VERSION_1 + notification.os_distro = OS_DISTRO_1 + notification.instance = INSTANCE_ID_1 + notification.request_id = REQUEST_ID_1 + notification.instance_type_id = INSTANCE_TYPE_ID_1 - views._process_usage_for_new_launch(raw, notification[1]) + raw = self.mox.CreateMockAnything() + raw.event = 'compute.instance.resize.prep.start' + + orig_launched_at = utils.decimal_utc(DUMMY_TIME - datetime.timedelta(days=1)) + usage = self.mox.CreateMockAnything() + usage.launched_at = orig_launched_at + views.STACKDB.get_or_create_instance_usage(instance=INSTANCE_ID_1, + request_id=REQUEST_ID_1) \ + .AndReturn((usage, True)) + views.STACKDB.save(usage) + self.mox.ReplayAll() + + views._process_usage_for_new_launch(raw, notification) self.assertEqual(usage.launched_at, orig_launched_at) self.assertEqual(usage.tenant, TENANT_ID_1) @@ -419,16 +507,30 @@ class StacktachUsageParsingTestCase(unittest.TestCase): self.mox.VerifyAll() def test_process_usage_for_updates_create_end(self): - kwargs = {'launched': str(DUMMY_TIME), - 'tenant_id': TENANT_ID_1, 'rax_options': RAX_OPTIONS_1, - 'os_architecture': OS_ARCH_1, 'os_version': OS_VERSION_1, - 'os_distro': OS_DISTRO_1 } - notification = utils.create_nova_notif(request_id=REQUEST_ID_1, - **kwargs) - event = 'compute.instance.create.end' - raw, usage = self._setup_process_usage_mocks(event, notification) + notification = self.mox.CreateMockAnything() + notification.launched_at = str(DUMMY_TIME) + notification.tenant = TENANT_ID_1 + notification.rax_options = RAX_OPTIONS_1 + notification.os_architecture = OS_ARCH_1 + notification.os_version = OS_VERSION_1 + notification.os_distro = OS_DISTRO_1 + notification.instance = INSTANCE_ID_1 + notification.request_id = REQUEST_ID_1 + notification.instance_type_id = INSTANCE_TYPE_ID_1 + notification.message = None - views._process_usage_for_updates(raw, notification[1]) + raw = self.mox.CreateMockAnything() + raw.event = 'compute.instance.create.end' + + usage = self.mox.CreateMockAnything() + usage.launched_at = None + views.STACKDB.get_or_create_instance_usage(instance=INSTANCE_ID_1, + request_id=REQUEST_ID_1) \ + .AndReturn((usage, True)) + views.STACKDB.save(usage) + self.mox.ReplayAll() + + views._process_usage_for_updates(raw, notification) self.assertEqual(usage.launched_at, utils.decimal_utc(DUMMY_TIME)) self.assertEqual(usage.tenant, TENANT_ID_1) @@ -440,17 +542,30 @@ class StacktachUsageParsingTestCase(unittest.TestCase): self.mox.VerifyAll() def test_process_usage_for_updates_create_end_success_message(self): - kwargs = {'launched': str(DUMMY_TIME), - 'tenant_id': TENANT_ID_1, 'rax_options': RAX_OPTIONS_1, - 'os_architecture': OS_ARCH_1, 'os_version': OS_VERSION_1, - 'os_distro': OS_DISTRO_1 } - notification = utils.create_nova_notif(request_id=REQUEST_ID_1, - **kwargs) - notification[1]['payload']['message'] = "Success" - event = 'compute.instance.create.end' - raw, usage = self._setup_process_usage_mocks(event, notification) + notification = self.mox.CreateMockAnything() + notification.launched_at = str(DUMMY_TIME) + notification.tenant = TENANT_ID_1 + notification.rax_options = RAX_OPTIONS_1 + notification.os_architecture = OS_ARCH_1 + notification.os_version = OS_VERSION_1 + notification.os_distro = OS_DISTRO_1 + notification.instance = INSTANCE_ID_1 + notification.request_id = REQUEST_ID_1 + notification.instance_type_id = INSTANCE_TYPE_ID_1 + notification.message = 'Success' - views._process_usage_for_updates(raw, notification[1]) + raw = self.mox.CreateMockAnything() + raw.event = 'compute.instance.create.end' + + usage = self.mox.CreateMockAnything() + usage.launched_at = None + views.STACKDB.get_or_create_instance_usage(instance=INSTANCE_ID_1, + request_id=REQUEST_ID_1) \ + .AndReturn((usage, True)) + views.STACKDB.save(usage) + self.mox.ReplayAll() + + views._process_usage_for_updates(raw, notification) self.assertEqual(usage.launched_at, utils.decimal_utc(DUMMY_TIME)) self.assertEqual(usage.tenant, TENANT_ID_1) @@ -462,37 +577,42 @@ class StacktachUsageParsingTestCase(unittest.TestCase): self.mox.VerifyAll() def test_process_usage_for_updates_create_end_error_message(self): - kwargs = {'launched': str(DUMMY_TIME), - 'tenant_id': TENANT_ID_1, 'rax_options': RAX_OPTIONS_1, - 'os_architecture': OS_ARCH_1, 'os_version': OS_VERSION_1, - 'os_distro': OS_DISTRO_1 } - notification = utils.create_nova_notif(request_id=REQUEST_ID_1, - **kwargs) - notification[1]['payload']['message'] = "Error" - event = 'compute.instance.create.end' - when_time = DUMMY_TIME - when_decimal = utils.decimal_utc(when_time) - json_str = json.dumps(notification) - raw = utils.create_raw(self.mox, when_decimal, event=event, - json_str=json_str) + notification = self.mox.CreateMockAnything() + notification.message = 'Error' + + raw = self.mox.CreateMockAnything() + raw.event = 'compute.instance.create.end' self.mox.ReplayAll() - views._process_usage_for_updates(raw, notification[1]) + views._process_usage_for_updates(raw, notification) self.mox.VerifyAll() def test_process_usage_for_updates_revert_end(self): - kwargs = {'launched': str(DUMMY_TIME), - 'type_id': INSTANCE_TYPE_ID_1, - 'tenant_id': TENANT_ID_1, 'rax_options': RAX_OPTIONS_1, - 'os_architecture': OS_ARCH_1, 'os_version': OS_VERSION_1, - 'os_distro': OS_DISTRO_1 } - notification = utils.create_nova_notif(request_id=REQUEST_ID_1, - **kwargs) - event = 'compute.instance.resize.revert.end' - raw, usage = self._setup_process_usage_mocks(event, notification) + notification = self.mox.CreateMockAnything() + notification.launched_at = str(DUMMY_TIME) + notification.tenant = TENANT_ID_1 + notification.rax_options = RAX_OPTIONS_1 + notification.os_architecture = OS_ARCH_1 + notification.os_version = OS_VERSION_1 + notification.os_distro = OS_DISTRO_1 + notification.instance = INSTANCE_ID_1 + notification.request_id = REQUEST_ID_1 + notification.instance_type_id = INSTANCE_TYPE_ID_1 + notification.message = None - views._process_usage_for_updates(raw, notification[1]) + raw = self.mox.CreateMockAnything() + raw.event = 'compute.instance.resize.revert.end' + + usage = self.mox.CreateMockAnything() + usage.launched_at = None + views.STACKDB.get_or_create_instance_usage(instance=INSTANCE_ID_1, + request_id=REQUEST_ID_1) \ + .AndReturn((usage, True)) + views.STACKDB.save(usage) + self.mox.ReplayAll() + + views._process_usage_for_updates(raw, notification) self.assertEqual(usage.instance_type_id, INSTANCE_TYPE_ID_1) self.assertEqual(usage.launched_at, utils.decimal_utc(DUMMY_TIME)) @@ -505,17 +625,30 @@ class StacktachUsageParsingTestCase(unittest.TestCase): self.mox.VerifyAll() def test_process_usage_for_updates_prep_end(self): - kwargs = {'launched': str(DUMMY_TIME), - 'new_type_id': INSTANCE_TYPE_ID_2, - 'tenant_id': TENANT_ID_1, 'rax_options': RAX_OPTIONS_1, - 'os_architecture': OS_ARCH_1, 'os_version': OS_VERSION_1, - 'os_distro': OS_DISTRO_1 } - notification = utils.create_nova_notif(request_id=REQUEST_ID_1, - **kwargs) - event = 'compute.instance.resize.prep.end' - raw, usage = self._setup_process_usage_mocks(event, notification) + notification = self.mox.CreateMockAnything() + notification.launched_at = str(DUMMY_TIME) + notification.tenant = TENANT_ID_1 + notification.rax_options = RAX_OPTIONS_1 + notification.os_architecture = OS_ARCH_1 + notification.os_version = OS_VERSION_1 + notification.os_distro = OS_DISTRO_1 + notification.instance = INSTANCE_ID_1 + notification.request_id = REQUEST_ID_1 + notification.new_instance_type_id = INSTANCE_TYPE_ID_2 + notification.message = None - views._process_usage_for_updates(raw, notification[1]) + raw = self.mox.CreateMockAnything() + raw.event = 'compute.instance.resize.prep.end' + + usage = self.mox.CreateMockAnything() + usage.launched_at = None + views.STACKDB.get_or_create_instance_usage(instance=INSTANCE_ID_1, + request_id=REQUEST_ID_1) \ + .AndReturn((usage, True)) + views.STACKDB.save(usage) + self.mox.ReplayAll() + + views._process_usage_for_updates(raw, notification) self.assertEqual(usage.instance_type_id, INSTANCE_TYPE_ID_2) self.assertEquals(usage.tenant, TENANT_ID_1) @@ -526,43 +659,29 @@ class StacktachUsageParsingTestCase(unittest.TestCase): self.mox.VerifyAll() - def _setup_process_usage_mocks(self, event, notification): - when_time = DUMMY_TIME - when_decimal = utils.decimal_utc(when_time) - json_str = json.dumps(notification) - raw = utils.create_raw(self.mox, when_decimal, event=event, - json_str=json_str) - usage = self.mox.CreateMockAnything() - views.STACKDB.get_or_create_instance_usage(instance=INSTANCE_ID_1, - request_id=REQUEST_ID_1) \ - .AndReturn((usage, True)) - views.STACKDB.save(usage) - self.mox.ReplayAll() - return raw, usage - def test_process_delete(self): delete_time = datetime.datetime.utcnow() launch_time = delete_time-datetime.timedelta(days=1) launch_decimal = utils.decimal_utc(launch_time) delete_decimal = utils.decimal_utc(delete_time) - notif = utils.create_nova_notif(request_id=REQUEST_ID_1, - launched=str(launch_time), - deleted=str(delete_time)) - json_str = json.dumps(notif) - event = 'compute.instance.delete.end' - raw = utils.create_raw(self.mox, delete_decimal, event=event, - json_str=json_str) + notification = self.mox.CreateMockAnything() + notification.instance = INSTANCE_ID_1 + notification.deleted_at = str(delete_time) + notification.launched_at = str(launch_time) + + raw = self.mox.CreateMockAnything() delete = self.mox.CreateMockAnything() delete.instance = INSTANCE_ID_1 delete.launched_at = launch_decimal delete.deleted_at = delete_decimal - views.STACKDB.get_or_create_instance_delete(instance=INSTANCE_ID_1, - deleted_at=delete_decimal)\ - .AndReturn((delete, True)) + views.STACKDB.get_or_create_instance_delete( + instance=INSTANCE_ID_1, deleted_at=delete_decimal)\ + .AndReturn((delete, True)) views.STACKDB.save(delete) self.mox.ReplayAll() - views._process_delete(raw, notif[1]) + views._process_delete(raw, notification) + self.assertEqual(delete.instance, INSTANCE_ID_1) self.assertEqual(delete.launched_at, launch_decimal) self.assertEqual(delete.deleted_at, delete_decimal) @@ -570,47 +689,50 @@ class StacktachUsageParsingTestCase(unittest.TestCase): def test_process_delete_no_launch(self): delete_time = datetime.datetime.utcnow() + launch_time = delete_time-datetime.timedelta(days=1) delete_decimal = utils.decimal_utc(delete_time) - notif = utils.create_nova_notif(request_id=REQUEST_ID_1, - deleted=str(delete_time)) - json_str = json.dumps(notif) - event = 'compute.instance.delete.end' - raw = utils.create_raw(self.mox, delete_decimal, event=event, - json_str=json_str) + notification = self.mox.CreateMockAnything() + notification.instance = INSTANCE_ID_1 + notification.deleted_at = str(delete_time) + notification.launched_at = str(launch_time) + + raw = self.mox.CreateMockAnything() delete = self.mox.CreateMockAnything() delete.instance = INSTANCE_ID_1 delete.deleted_at = delete_decimal - views.STACKDB.get_or_create_instance_delete(instance=INSTANCE_ID_1, - deleted_at=delete_decimal)\ - .AndReturn((delete, True)) + views.STACKDB.get_or_create_instance_delete( + instance=INSTANCE_ID_1, deleted_at=delete_decimal)\ + .AndReturn((delete, True)) views.STACKDB.save(delete) self.mox.ReplayAll() - views._process_delete(raw, notif[1]) + views._process_delete(raw, notification) + self.assertEqual(delete.instance, INSTANCE_ID_1) self.assertEqual(delete.deleted_at, delete_decimal) self.mox.VerifyAll() def test_process_exists(self): + notification = self.mox.CreateMockAnything() current_time = datetime.datetime.utcnow() launch_time = current_time - datetime.timedelta(hours=23) launch_decimal = utils.decimal_utc(launch_time) - current_decimal = utils.decimal_utc(current_time) audit_beginning = current_time - datetime.timedelta(hours=20) audit_beginning_decimal = utils.decimal_utc(audit_beginning) audit_ending_decimal = utils.decimal_utc(current_time) - notif = utils.create_nova_notif(launched=str(launch_time), - audit_period_beginning=str(audit_beginning), - audit_period_ending=str(current_time), - tenant_id=TENANT_ID_1, - os_architecture=OS_ARCH_1, - os_version=OS_VERSION_1, - os_distro=OS_DISTRO_1, - rax_options=RAX_OPTIONS_1) - json_str = json.dumps(notif) - event = 'compute.instance.exists' - raw = utils.create_raw(self.mox, current_decimal, event=event, - json_str=json_str) + notification.launched_at = str(launch_time) + notification.audit_period_beginning = str(audit_beginning) + notification.audit_period_ending = str(current_time) + notification.tenant = TENANT_ID_1 + notification.os_architecture = OS_ARCH_1 + notification.os_version = OS_VERSION_1 + notification.os_distro = OS_DISTRO_1 + notification.rax_options = RAX_OPTIONS_1 + notification.instance = INSTANCE_ID_1 + notification.deleted_at = '' + notification.instance_type_id = INSTANCE_TYPE_ID_1 + notification.message_id = MESSAGE_ID_1 + raw = self.mox.CreateMockAnything() usage = self.mox.CreateMockAnything() launched_range = (launch_decimal, launch_decimal+1) views.STACKDB.get_instance_usage(instance=INSTANCE_ID_1, @@ -622,7 +744,7 @@ class StacktachUsageParsingTestCase(unittest.TestCase): 'launched_at': launch_decimal, 'audit_period_beginning': audit_beginning_decimal, 'audit_period_ending': audit_ending_decimal, - 'instance_type_id': '1', + 'instance_type_id': INSTANCE_TYPE_ID_1, 'usage': usage, 'raw': raw, 'tenant': TENANT_ID_1, @@ -635,50 +757,45 @@ class StacktachUsageParsingTestCase(unittest.TestCase): views.STACKDB.create_instance_exists(**exists_values).AndReturn(exists) views.STACKDB.save(exists) self.mox.ReplayAll() - views._process_exists(raw, notif[1]) + views._process_exists(raw, notification) self.mox.VerifyAll() def test_process_exists_no_launched_at(self): - current_time = datetime.datetime.utcnow() - current_decimal = utils.decimal_utc(current_time) - audit_beginning = current_time - datetime.timedelta(hours=20) - notif = utils.create_nova_notif(audit_period_beginning=str(audit_beginning), - audit_period_ending=str(current_time), - tenant_id=TENANT_ID_1) - json_str = json.dumps(notif) - event = 'compute.instance.exists' - raw = utils.create_raw(self.mox, current_decimal, event=event, - json_str=json_str) - raw.id = 1 + notification = self.mox.CreateMockAnything() + notification.instance = INSTANCE_ID_1 + notification.launched_at = None + raw = self.mox.CreateMockAnything() + raw.id = '1' self.setup_mock_log() self.log.warn('Ignoring exists without launched_at. RawData(1)') self.mox.ReplayAll() - views._process_exists(raw, notif[1]) + views._process_exists(raw, notification) self.mox.VerifyAll() def test_process_exists_with_deleted_at(self): + notification = self.mox.CreateMockAnything() current_time = datetime.datetime.utcnow() launch_time = current_time - datetime.timedelta(hours=23) launch_decimal = utils.decimal_utc(launch_time) - deleted_time = current_time - datetime.timedelta(hours=12) - deleted_decimal = utils.decimal_utc(deleted_time) - current_decimal = utils.decimal_utc(current_time) + delete_time = datetime.datetime.utcnow() + deleted_decimal = utils.decimal_utc(delete_time) audit_beginning = current_time - datetime.timedelta(hours=20) audit_beginning_decimal = utils.decimal_utc(audit_beginning) audit_ending_decimal = utils.decimal_utc(current_time) - notif = utils.create_nova_notif(launched=str(launch_time), - deleted=str(deleted_time), - audit_period_beginning=str(audit_beginning), - audit_period_ending=str(current_time), - tenant_id=TENANT_ID_1, - os_architecture=OS_ARCH_1, - os_version=OS_VERSION_1, - os_distro=OS_DISTRO_1, - rax_options=RAX_OPTIONS_1) - json_str = json.dumps(notif) - event = 'compute.instance.exists' - raw = utils.create_raw(self.mox, current_decimal, event=event, - json_str=json_str) + + notification.launched_at = str(launch_time) + notification.audit_period_beginning = str(audit_beginning) + notification.audit_period_ending = str(current_time) + notification.tenant = TENANT_ID_1 + notification.os_architecture = OS_ARCH_1 + notification.os_version = OS_VERSION_1 + notification.os_distro = OS_DISTRO_1 + notification.rax_options = RAX_OPTIONS_1 + notification.instance = INSTANCE_ID_1 + notification.instance_type_id = INSTANCE_TYPE_ID_1 + notification.message_id = MESSAGE_ID_1 + notification.deleted_at = str(delete_time) + raw = self.mox.CreateMockAnything() usage = self.mox.CreateMockAnything() launched_range = (launch_decimal, launch_decimal+1) views.STACKDB.get_instance_usage(instance=INSTANCE_ID_1, @@ -695,7 +812,7 @@ class StacktachUsageParsingTestCase(unittest.TestCase): 'deleted_at': deleted_decimal, 'audit_period_beginning': audit_beginning_decimal, 'audit_period_ending': audit_ending_decimal, - 'instance_type_id': '1', + 'instance_type_id': INSTANCE_TYPE_ID_1, 'usage': usage, 'delete': delete, 'raw': raw, @@ -709,7 +826,7 @@ class StacktachUsageParsingTestCase(unittest.TestCase): views.STACKDB.create_instance_exists(**exists_values).AndReturn(exists) views.STACKDB.save(exists) self.mox.ReplayAll() - views._process_exists(raw, notif[1]) + views._process_exists(raw, notification) self.mox.VerifyAll() @@ -766,114 +883,9 @@ class StacktachImageUsageParsingTestCase(unittest.TestCase): def test_process_image_exists(self): raw = self.mox.CreateMockAnything() - values = { - 'uuid': IMAGE_UUID_1, - 'created_at': DECIMAL_DUMMY_TIME, - 'deleted_at': DECIMAL_DUMMY_TIME, - 'audit_period_beginning': DECIMAL_DUMMY_TIME, - 'audit_period_ending': DECIMAL_DUMMY_TIME, - 'owner': TENANT_ID_1, - 'size': 1234, - 'raw': raw, - 'usage': None, - 'delete': None - } - notification = self.mox.CreateMockAnything() - notification.created_at = values['created_at'] - notification.deleted_at = values['deleted_at'] - notification.owner = values['owner'] - notification.uuid = values['uuid'] - notification.size = values['size'] - notification.raw = values['raw'] - notification.audit_period_beginning = values['audit_period_beginning'] - notification.audit_period_ending = values['audit_period_ending'] - created_at_range = (DECIMAL_DUMMY_TIME, DECIMAL_DUMMY_TIME+1) - views.STACKDB.get_image_usage(created_at__range=created_at_range, - uuid=notification.uuid).AndReturn(None) - views.STACKDB.get_image_delete(created_at__range=created_at_range, - uuid=notification.uuid).AndReturn(None) - views.STACKDB.create_image_exists(**values) - + notification.save_exists(raw) self.mox.ReplayAll() views._process_glance_exists(raw, notification) self.mox.VerifyAll() - - def test_process_image_exists_with_usage_not_none(self): - raw = self.mox.CreateMockAnything() - usage = self.mox.CreateMockAnything() - values = { - 'uuid': IMAGE_UUID_1, - 'created_at': DECIMAL_DUMMY_TIME, - 'deleted_at': DECIMAL_DUMMY_TIME, - 'audit_period_beginning': DECIMAL_DUMMY_TIME, - 'audit_period_ending': DECIMAL_DUMMY_TIME, - 'owner': TENANT_ID_1, - 'size': 1234, - 'raw': raw, - 'usage': usage, - 'delete': None - } - - notification = self.mox.CreateMockAnything() - notification.created_at = values['created_at'] - notification.deleted_at = values['deleted_at'] - notification.owner = values['owner'] - notification.uuid = values['uuid'] - notification.size = values['size'] - notification.raw = values['raw'] - notification.audit_period_beginning = values['audit_period_beginning'] - notification.audit_period_ending = values['audit_period_ending'] - created_at_range = (DECIMAL_DUMMY_TIME, DECIMAL_DUMMY_TIME+1) - views.STACKDB.get_image_usage(created_at__range=created_at_range, - uuid=notification.uuid).AndReturn(usage) - views.STACKDB.get_image_delete(created_at__range=created_at_range, - uuid=notification.uuid).AndReturn(None) - views.STACKDB.create_image_exists(**values) - - self.mox.ReplayAll() - - views._process_glance_exists(raw, notification) - self.mox.VerifyAll() - - - def test_process_image_exists_with_delete_not_none(self): - raw = self.mox.CreateMockAnything() - delete = self.mox.CreateMockAnything() - values = { - 'uuid': IMAGE_UUID_1, - 'created_at': DECIMAL_DUMMY_TIME, - 'deleted_at': DECIMAL_DUMMY_TIME, - 'audit_period_beginning': DECIMAL_DUMMY_TIME, - 'audit_period_ending': DECIMAL_DUMMY_TIME, - 'owner': TENANT_ID_1, - 'size': 1234, - 'raw': raw, - 'usage': None, - 'delete': delete - } - - notification = self.mox.CreateMockAnything() - notification.created_at = values['created_at'] - notification.deleted_at = values['deleted_at'] - notification.owner = values['owner'] - notification.uuid = values['uuid'] - notification.size = values['size'] - notification.raw = values['raw'] - notification.audit_period_beginning = values['audit_period_beginning'] - notification.audit_period_ending = values['audit_period_ending'] - created_at_range = (DECIMAL_DUMMY_TIME, DECIMAL_DUMMY_TIME+1) - views.STACKDB.get_image_usage(created_at__range=created_at_range, - uuid=notification.uuid).AndReturn(None) - views.STACKDB.get_image_delete(created_at__range=created_at_range, - uuid=notification.uuid).AndReturn(delete) - views.STACKDB.create_image_exists(**values) - - self.mox.ReplayAll() - - views._process_glance_exists(raw, notification) - self.mox.VerifyAll() - - # def test_process_image_exists_should_not_populate_delete_and_deleted_at_when_deleted_at_is_absent(self): - diff --git a/worker/worker.py b/worker/worker.py index e542b9c..615b693 100644 --- a/worker/worker.py +++ b/worker/worker.py @@ -34,9 +34,7 @@ except ImportError: from pympler.process import ProcessMemoryInfo -from django import db as django_db from stacktach import db -from stacktach import notification from stacktach import stacklog from stacktach import views From 5589c27a67c052716f726c3ecded084e1f47d97f Mon Sep 17 00:00:00 2001 From: Anuj Mathur Date: Fri, 5 Jul 2013 12:41:23 +0530 Subject: [PATCH 05/11] Created exchange logger which appends the name of the exchange to the logs --- stacktach/stacklog.py | 63 +++++++++++++++++----- tests/unit/test_stacklog.py | 102 ++++++++++++++++++++++++++++++++++++ 2 files changed, 152 insertions(+), 13 deletions(-) create mode 100644 tests/unit/test_stacklog.py diff --git a/stacktach/stacklog.py b/stacktach/stacklog.py index bf142ae..28344ba 100644 --- a/stacktach/stacklog.py +++ b/stacktach/stacklog.py @@ -36,31 +36,33 @@ def set_default_logger_name(name): default_logger_name = name -def _make_logger(name): - log = logging.getLogger(__name__) - log.setLevel(logging.DEBUG) - handler = logging.handlers.TimedRotatingFileHandler(default_logger_location % name, - when='midnight', interval=1, backupCount=3) - formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') - handler.setFormatter(formatter) - log.addHandler(handler) - log.handlers[0].doRollover() +def _logger_factory(exchange, name): + if exchange: + return ExchangeLogger(exchange, name) + else: + logger = logging.getLogger(__name__) + _configure(logger, name) + return logger + + +def _make_logger(name, exchange=None): + log = _logger_factory(exchange, name) return log -def init_logger(name=None): +def init_logger(name=None, exchange=None): global LOGGERS if name is None: name = default_logger_name if name not in LOGGERS: - LOGGERS[name] = _make_logger(name) + LOGGERS[name] = _make_logger(name, exchange) -def get_logger(name=None): +def get_logger(name=None, exchange=None): global LOGGERS if name is None: name = default_logger_name - init_logger(name=name) + init_logger(name=name, exchange=exchange) return LOGGERS[name] @@ -80,3 +82,38 @@ def info(msg, name=None): if name is None: name = default_logger_name get_logger(name=name).info(msg) + + +def _configure(logger, name): + logger.setLevel(logging.DEBUG) + handler = logging.handlers.TimedRotatingFileHandler( + default_logger_location % name, + when='midnight', interval=1, backupCount=3) + formatter = logging.Formatter( + '%(asctime)s - %(name)s - %(levelname)s - %(message)s') + handler.setFormatter(formatter) + logger.addHandler(handler) + logger.handlers[0].doRollover() + + +class ExchangeLogger(): + def __init__(self, exchange, name='stacktach-default'): + self.logger = logging.getLogger(name) + _configure(self.logger, name) + self.exchange = exchange + + def info(self, msg, *args, **kwargs): + msg = self.exchange + ': ' + msg + self.logger.info(msg, *args, **kwargs) + + def warn(self, msg, *args, **kwargs): + msg = self.exchange + ': ' + msg + self.logger.warn(msg, *args, **kwargs) + + def error(self, msg, *args, **kwargs): + msg = self.exchange + ': ' + msg + self.logger.error(msg, *args, **kwargs) + + def exception(self, msg, *args, **kwargs): + msg = self.exchange + ': ' + msg + self.logger.error(msg, *args, **kwargs) \ No newline at end of file diff --git a/tests/unit/test_stacklog.py b/tests/unit/test_stacklog.py new file mode 100644 index 0000000..d121953 --- /dev/null +++ b/tests/unit/test_stacklog.py @@ -0,0 +1,102 @@ +import glob +import logging +import os +from unittest import TestCase +import mox +from stacktach import stacklog +import __builtin__ +from stacktach.stacklog import ExchangeLogger + + +class StacklogTestCase(TestCase): + def setUp(self): + self.mox = mox.Mox() + + def tearDown(self): + self.mox.UnsetStubs() + + def test_get_logger_should_get_exchange_logger_if_exchange_provided(self): + filename = 'filename' + logger = stacklog.get_logger(filename, 'nova') + self.assertIsInstance(logger, ExchangeLogger) + for file in glob.glob('{0}.log*'.format(filename)): + os.remove(file) + + def test_get_logger_should_get_default_logger_if_exchange_not_provided(self): + filename = 'default_logger' + logger = stacklog.get_logger(filename) + self.assertIsInstance(logger, logging.Logger) + for file in glob.glob('{0}.log*'.format(filename)): + os.remove(file) + + +class ExchangeLoggerTestCase(TestCase): + def setUp(self): + self.mox = mox.Mox() + + def tearDown(self): + self.mox.UnsetStubs() + + def _setup_logger_mocks(self, name='name'): + mock_logger = self.mox.CreateMockAnything() + self.mox.StubOutWithMock(logging, 'getLogger') + logging.getLogger(name).AndReturn(mock_logger) + mock_logger.setLevel(logging.DEBUG) + self.mox.StubOutClassWithMocks(logging.handlers, + 'TimedRotatingFileHandler') + filename = "{0}.log".format(name) + handler = logging.handlers.TimedRotatingFileHandler( + filename, backupCount=3, interval=1, when='midnight') + self.mox.StubOutClassWithMocks(logging, 'Formatter') + mock_formatter = logging.Formatter( + "%(asctime)s - %(name)s - %(levelname)s - %(message)s") + handler.setFormatter(mock_formatter) + mock_logger.addHandler(handler) + mock_logger.handlers = [handler] + handler.doRollover() + return mock_logger + + def test_exchange_logger_should_append_exchange_name_to_info(self): + mock_logger = self._setup_logger_mocks() + mock_logger.info('exchange: Log %s', 'args', xyz='xyz') + self.mox.ReplayAll() + + log = ExchangeLogger('exchange', 'name') + log.info("Log %s", 'args', xyz='xyz') + self.mox.VerifyAll() + + def test_exchange_logger_should_append_exchange_name_to_warn(self): + mock_logger = self._setup_logger_mocks() + mock_logger.warn('exchange: Log %s', 'args', xyz='xyz') + self.mox.ReplayAll() + + logger = ExchangeLogger('exchange', 'name') + logger.warn("Log %s", 'args', xyz='xyz') + self.mox.VerifyAll() + + def test_exchange_logger_should_append_exchange_name_to_error(self): + mock_logger = self._setup_logger_mocks() + mock_logger.error('exchange: Log %s', 'args', xyz='xyz') + self.mox.ReplayAll() + + logger = ExchangeLogger('exchange', 'name') + logger.error("Log %s", 'args', xyz='xyz') + self.mox.VerifyAll() + + def test_exchange_logger_should_append_exchange_name_to_exception(self): + mock_logger = self._setup_logger_mocks() + mock_logger.error('exchange: Log %s', 'args', xyz='xyz') + self.mox.ReplayAll() + + logger = ExchangeLogger('exchange', 'name') + logger.exception("Log %s", 'args', xyz='xyz') + self.mox.VerifyAll() + + def test_exchange_logger_should_use_default_name_if_not_provided(self): + self._setup_logger_mocks('stacktach-default') + self.mox.ReplayAll() + + ExchangeLogger('exchange') + self.mox.VerifyAll() + + From f227803dd5eb5e2f99a03db7acb1f9cbe69e93bd Mon Sep 17 00:00:00 2001 From: Anuj Mathur Date: Tue, 9 Jul 2013 18:02:25 +0530 Subject: [PATCH 06/11] Extracted save methods for image_usage and image_delete to glance_notification --- stacktach/notification.py | 20 +++++++++ stacktach/views.py | 20 +-------- tests/unit/test_notification.py | 78 ++++++++++++++++++++++++++++++--- tests/unit/test_stacktach.py | 39 +++-------------- 4 files changed, 101 insertions(+), 56 deletions(-) diff --git a/stacktach/notification.py b/stacktach/notification.py index 656fb5f..48a4abe 100644 --- a/stacktach/notification.py +++ b/stacktach/notification.py @@ -161,6 +161,26 @@ class GlanceNotification(Notification): stacklog.warn("Ignoring exists without created_at. GlanceRawData(%s)" % raw.id) + def save_usage(self, raw): + values = { + 'uuid': self.uuid, + 'created_at': self.created_at, + 'owner': self.owner, + 'size': self.size, + 'last_raw': raw + } + db.create_image_usage(**values) + + def save_delete(self, raw): + values = { + 'uuid': self.uuid, + 'created_at': self.created_at, + 'owner': self.owner, + 'size': self.size, + 'raw': raw, + 'deleted_at': self.deleted_at + } + db.create_image_delete(**values) class NovaNotification(Notification): def __init__(self, body, deployment, routing_key, json): diff --git a/stacktach/views.py b/stacktach/views.py index 8e419bc..a26db0d 100644 --- a/stacktach/views.py +++ b/stacktach/views.py @@ -286,26 +286,10 @@ def _process_exists(raw, notification): def _process_glance_usage(raw, notification): - values = { - 'uuid': notification.uuid, - 'created_at': notification.created_at, - 'owner': notification.owner, - 'size': notification.size, - 'last_raw': raw - } - STACKDB.create_image_usage(**values) - + notification.save_usage(raw) def _process_glance_delete(raw, notification): - values = { - 'uuid': notification.uuid, - 'created_at': notification.created_at, - 'owner': notification.owner, - 'size': notification.size, - 'raw': raw, - 'deleted_at': notification.deleted_at - } - STACKDB.create_image_delete(**values) + notification.save_delete(raw) def _process_glance_exists(raw, notification): diff --git a/tests/unit/test_notification.py b/tests/unit/test_notification.py index 0347cbf..4acd2e6 100644 --- a/tests/unit/test_notification.py +++ b/tests/unit/test_notification.py @@ -187,7 +187,7 @@ class GlanceNotificationTestCase(unittest.TestCase): service="glance-api01-r2961", host="global.preprod-ord.ohthree.com", instance=INSTANCE_ID_1, - request_id=None, + request_id='', image_type=0, status="saving", uuid="2df2ccf6-bc1b-4853-aab0-25fda346b3bb").AndReturn(raw) @@ -199,8 +199,7 @@ class GlanceNotificationTestCase(unittest.TestCase): self.assertEquals(notification.save(), raw) self.mox.VerifyAll() - def test_process_image_exists(self): - usage = self.mox.CreateMockAnything() + def test_save_image_exists(self): raw = self.mox.CreateMockAnything() audit_period_beginning = "2013-05-20 17:31:57.939614" audit_period_ending = "2013-06-20 17:31:57.939614" @@ -251,9 +250,8 @@ class GlanceNotificationTestCase(unittest.TestCase): notification.save_exists(raw) self.mox.VerifyAll() - def test_process_image_exists_with_delete_not_none(self): + def test_save_image_exists_with_delete_not_none(self): raw = self.mox.CreateMockAnything() - usage = self.mox.CreateMockAnything() delete = self.mox.CreateMockAnything() audit_period_beginning = "2013-05-20 17:31:57.939614" audit_period_ending = "2013-06-20 17:31:57.939614" @@ -311,7 +309,7 @@ class GlanceNotificationTestCase(unittest.TestCase): notification.save_exists(raw) self.mox.VerifyAll() - def test_process_image_exists_with_usage_not_none(self): + def test_save_image_exists_with_usage_not_none(self): raw = self.mox.CreateMockAnything() usage = self.mox.CreateMockAnything() audit_period_beginning = "2013-05-20 17:31:57.939614" @@ -364,6 +362,74 @@ class GlanceNotificationTestCase(unittest.TestCase): notification.save_exists(raw) self.mox.VerifyAll() + def test_save_usage_should_persist_image_usage(self): + raw = self.mox.CreateMockAnything() + size = 123 + uuid = "2df2ccf6-bc1b-4853-aab0-25fda346b3bb" + body = { + "event_type": "image.upload", + "timestamp": "2013-06-20 18:31:57.939614", + "publisher_id": "glance-api01-r2961.global.preprod-ord.ohthree.com", + "payload": { + "created_at": str(DUMMY_TIME), + "size": size, + "owner": TENANT_ID_1, + "id": "2df2ccf6-bc1b-4853-aab0-25fda346b3bb", + } + } + deployment = "1" + routing_key = "glance_monitor.info" + json = '{["routing_key", {%s}]}' % body + + self.mox.StubOutWithMock(db, 'create_image_usage') + db.create_image_usage( + created_at=utils.str_time_to_unix(str(DUMMY_TIME)), + owner=TENANT_ID_1, + last_raw=raw, + size=size, + uuid=uuid).AndReturn(raw) + self.mox.ReplayAll() + + notification = GlanceNotification(body, deployment, routing_key, json) + notification.save_usage(raw) + self.mox.VerifyAll() + + def test_save_delete_should_persist_image_delete(self): + raw = self.mox.CreateMockAnything() + size = 123 + uuid = "2df2ccf6-bc1b-4853-aab0-25fda346b3bb" + deleted_at = "2013-06-20 14:31:57.939614" + body = { + "event_type": "image.delete", + "timestamp": "2013-06-20 18:31:57.939614", + "publisher_id": "glance-api01-r2961.global.preprod-ord.ohthree.com", + "payload": { + "created_at": str(DUMMY_TIME), + "size": size, + "owner": TENANT_ID_1, + "id": "2df2ccf6-bc1b-4853-aab0-25fda346b3bb", + "deleted_at": deleted_at + } + } + deployment = "1" + routing_key = "glance_monitor.info" + json = '{["routing_key", {%s}]}' % body + + self.mox.StubOutWithMock(db, 'create_image_delete') + db.create_image_delete( + created_at=utils.str_time_to_unix(str(DUMMY_TIME)), + owner=TENANT_ID_1, + raw=raw, + size=size, + uuid=uuid, + deleted_at=utils.str_time_to_unix(deleted_at)).AndReturn(raw) + self.mox.ReplayAll() + + notification = GlanceNotification(body, deployment, routing_key, json) + notification.save_delete(raw) + self.mox.VerifyAll() + + class NotificationTestCase(unittest.TestCase): def setUp(self): diff --git a/tests/unit/test_stacktach.py b/tests/unit/test_stacktach.py index b493b65..a57c08e 100644 --- a/tests/unit/test_stacktach.py +++ b/tests/unit/test_stacktach.py @@ -838,50 +838,25 @@ class StacktachImageUsageParsingTestCase(unittest.TestCase): def tearDown(self): self.mox.UnsetStubs() - def test_process_image_usage_for_new_launch(self): + def test_save_image_usage(self): raw = self.mox.CreateMockAnything() - values = { - 'created_at': str(DUMMY_TIME), - 'owner': TENANT_ID_1, - 'uuid': IMAGE_UUID_1, - 'size': 1234, - 'last_raw': raw - } notification = self.mox.CreateMockAnything() - notification.created_at = values['created_at'] - notification.owner = values['owner'] - notification.uuid = values['uuid'] - notification.size = values['size'] - notification.raw = values['last_raw'] - views.STACKDB.create_image_usage(**values) + notification.save_usage(raw) self.mox.ReplayAll() + views._process_glance_usage(raw, notification) self.mox.VerifyAll() - def test_process_image_deletes(self): + def test_save_image_delete(self): raw = self.mox.CreateMockAnything() - values = { - 'uuid': IMAGE_UUID_1, - 'created_at': str(DUMMY_TIME), - 'deleted_at': str(DUMMY_TIME), - 'owner': TENANT_ID_1, - 'size': 1234, - 'raw': raw - } - notification = self.mox.CreateMockAnything() - notification.created_at = values['created_at'] - notification.deleted_at = values['deleted_at'] - notification.owner = values['owner'] - notification.uuid = values['uuid'] - notification.size = values['size'] - notification.raw = values['raw'] - views.STACKDB.create_image_delete(**values) + notification.save_delete(raw) self.mox.ReplayAll() + views._process_glance_delete(raw, notification) self.mox.VerifyAll() - def test_process_image_exists(self): + def test_save_image_exists(self): raw = self.mox.CreateMockAnything() notification = self.mox.CreateMockAnything() notification.save_exists(raw) From 7e44644790aaaf411bf65ec40344c3dc7b8f054c Mon Sep 17 00:00:00 2001 From: Anuj Mathur Date: Wed, 10 Jul 2013 12:03:16 +0530 Subject: [PATCH 07/11] - Removed duplicate stacktach/config.py - Image usages are now created from image.activate instead of image.update - Removed unneccessary fields from image_deletes - Made image_usage.last_raw and image_deletes.raw nullable because the seed script will not populate these fields --- stacktach/config.py | 43 -------------- ...create_glance_usage_verification_tables.py | 56 +++++++++---------- stacktach/models.py | 8 +-- stacktach/notification.py | 3 - stacktach/tests.py | 3 - stacktach/views.py | 2 +- tests/unit/test_notification.py | 8 --- 7 files changed, 28 insertions(+), 95 deletions(-) delete mode 100644 stacktach/config.py diff --git a/stacktach/config.py b/stacktach/config.py deleted file mode 100644 index 28c8b5e..0000000 --- a/stacktach/config.py +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright (c) 2013 - Rackspace Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to -# deal in the Software without restriction, including without limitation the -# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or -# sell copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -import json -import os - -config_filename = os.environ.get('STACKTACH_DEPLOYMENTS_FILE', - 'stacktach_worker_config.json') -try: - from local_settings import * - config_filename = STACKTACH_DEPLOYMENTS_FILE -except ImportError: - pass - -config = None -with open(config_filename, "r") as f: - config = json.load(f) - - -def deployments(): - return config['deployments'] - - -def topics(): - return config['topics'] - - diff --git a/stacktach/migrations/0005_create_glance_usage_verification_tables.py b/stacktach/migrations/0005_create_glance_usage_verification_tables.py index 34aafeb..b6d89af 100644 --- a/stacktach/migrations/0005_create_glance_usage_verification_tables.py +++ b/stacktach/migrations/0005_create_glance_usage_verification_tables.py @@ -12,11 +12,8 @@ class Migration(SchemaMigration): db.create_table(u'stacktach_imagedeletes', ( (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), ('uuid', self.gf('django.db.models.fields.CharField')(max_length=50, db_index=True)), - ('created_at', self.gf('django.db.models.fields.DecimalField')(max_digits=20, decimal_places=6, db_index=True)), ('deleted_at', self.gf('django.db.models.fields.DecimalField')(null=True, max_digits=20, decimal_places=6, db_index=True)), - ('owner', self.gf('django.db.models.fields.CharField')(max_length=50, db_index=True)), - ('size', self.gf('django.db.models.fields.BigIntegerField')(max_length=20)), - ('raw', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['stacktach.GlanceRawData'])), + ('raw', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['stacktach.GlanceRawData'], null=True)), )) db.send_create_signal(u'stacktach', ['ImageDeletes']) @@ -47,28 +44,10 @@ class Migration(SchemaMigration): ('created_at', self.gf('django.db.models.fields.DecimalField')(max_digits=20, decimal_places=6, db_index=True)), ('owner', self.gf('django.db.models.fields.CharField')(max_length=50, db_index=True)), ('size', self.gf('django.db.models.fields.BigIntegerField')(max_length=20)), - ('last_raw', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['stacktach.GlanceRawData'])), + ('last_raw', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['stacktach.GlanceRawData'], null=True)), )) db.send_create_signal(u'stacktach', ['ImageUsage']) - # Adding model 'GenericRawData' - db.create_table(u'stacktach_genericrawdata', ( - (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), - ('deployment', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['stacktach.Deployment'])), - ('tenant', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=50, null=True, blank=True)), - ('json', self.gf('django.db.models.fields.TextField')()), - ('routing_key', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=50, null=True, blank=True)), - ('image_type', self.gf('django.db.models.fields.IntegerField')(default=0, null=True, db_index=True)), - ('when', self.gf('django.db.models.fields.DecimalField')(max_digits=20, decimal_places=6, db_index=True)), - ('publisher', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=100, null=True, blank=True)), - ('event', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=50, null=True, blank=True)), - ('service', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=50, null=True, blank=True)), - ('host', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=100, null=True, blank=True)), - ('instance', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=50, null=True, blank=True)), - ('request_id', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=50, null=True, blank=True)), - )) - db.send_create_signal(u'stacktach', ['GenericRawData']) - # Adding model 'ImageExists' db.create_table(u'stacktach_imageexists', ( (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), @@ -88,6 +67,24 @@ class Migration(SchemaMigration): )) db.send_create_signal(u'stacktach', ['ImageExists']) + # Adding model 'GenericRawData' + db.create_table(u'stacktach_genericrawdata', ( + (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('deployment', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['stacktach.Deployment'])), + ('tenant', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=50, null=True, blank=True)), + ('json', self.gf('django.db.models.fields.TextField')()), + ('routing_key', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=50, null=True, blank=True)), + ('image_type', self.gf('django.db.models.fields.IntegerField')(default=0, null=True, db_index=True)), + ('when', self.gf('django.db.models.fields.DecimalField')(max_digits=20, decimal_places=6, db_index=True)), + ('publisher', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=100, null=True, blank=True)), + ('event', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=50, null=True, blank=True)), + ('service', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=50, null=True, blank=True)), + ('host', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=100, null=True, blank=True)), + ('instance', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=50, null=True, blank=True)), + ('request_id', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=50, null=True, blank=True)), + )) + db.send_create_signal(u'stacktach', ['GenericRawData']) + def backwards(self, orm): # Deleting model 'ImageDeletes' @@ -99,12 +96,12 @@ class Migration(SchemaMigration): # Deleting model 'ImageUsage' db.delete_table(u'stacktach_imageusage') - # Deleting model 'GenericRawData' - db.delete_table(u'stacktach_genericrawdata') - # Deleting model 'ImageExists' db.delete_table(u'stacktach_imageexists') + # Deleting model 'GenericRawData' + db.delete_table(u'stacktach_genericrawdata') + models = { u'stacktach.deployment': { @@ -148,12 +145,9 @@ class Migration(SchemaMigration): }, u'stacktach.imagedeletes': { 'Meta': {'object_name': 'ImageDeletes'}, - 'created_at': ('django.db.models.fields.DecimalField', [], {'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}), 'deleted_at': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}), u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'owner': ('django.db.models.fields.CharField', [], {'max_length': '50', 'db_index': 'True'}), - 'raw': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['stacktach.GlanceRawData']"}), - 'size': ('django.db.models.fields.BigIntegerField', [], {'max_length': '20'}), + 'raw': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['stacktach.GlanceRawData']", 'null': 'True'}), 'uuid': ('django.db.models.fields.CharField', [], {'max_length': '50', 'db_index': 'True'}) }, u'stacktach.imageexists': { @@ -177,7 +171,7 @@ class Migration(SchemaMigration): 'Meta': {'object_name': 'ImageUsage'}, 'created_at': ('django.db.models.fields.DecimalField', [], {'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}), u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'last_raw': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['stacktach.GlanceRawData']"}), + 'last_raw': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['stacktach.GlanceRawData']", 'null': 'True'}), 'owner': ('django.db.models.fields.CharField', [], {'max_length': '50', 'db_index': 'True'}), 'size': ('django.db.models.fields.BigIntegerField', [], {'max_length': '20'}), 'uuid': ('django.db.models.fields.CharField', [], {'max_length': '50', 'db_index': 'True'}) diff --git a/stacktach/models.py b/stacktach/models.py index 5119db3..0fa2734 100644 --- a/stacktach/models.py +++ b/stacktach/models.py @@ -308,19 +308,15 @@ class ImageUsage(models.Model): decimal_places=6, db_index=True) owner = models.CharField(max_length=50, db_index=True) size = models.BigIntegerField(max_length=20) - last_raw = models.ForeignKey(GlanceRawData) + last_raw = models.ForeignKey(GlanceRawData, null=True) class ImageDeletes(models.Model): uuid = models.CharField(max_length=50, db_index=True) - created_at = models.DecimalField(max_digits=20, - decimal_places=6, db_index=True) deleted_at = models.DecimalField(max_digits=20, decimal_places=6, db_index=True, null=True) - owner = models.CharField(max_length=50, db_index=True) - size = models.BigIntegerField(max_length=20) - raw = models.ForeignKey(GlanceRawData) + raw = models.ForeignKey(GlanceRawData, null=True) class ImageExists(models.Model): diff --git a/stacktach/notification.py b/stacktach/notification.py index 48a4abe..e30a032 100644 --- a/stacktach/notification.py +++ b/stacktach/notification.py @@ -174,9 +174,6 @@ class GlanceNotification(Notification): def save_delete(self, raw): values = { 'uuid': self.uuid, - 'created_at': self.created_at, - 'owner': self.owner, - 'size': self.size, 'raw': raw, 'deleted_at': self.deleted_at } diff --git a/stacktach/tests.py b/stacktach/tests.py index 802a5fb..a4539a2 100644 --- a/stacktach/tests.py +++ b/stacktach/tests.py @@ -121,9 +121,6 @@ class GlanceTestCase(TransactionTestCase): _, rawdata = self._create_glance_rawdata() kwargs = { 'uuid': '1', - 'created_at': dt_to_decimal(datetime.utcnow()), - 'owner': 'owner', - 'size': 12345, 'raw': rawdata, 'deleted_at': dt_to_decimal(datetime.utcnow()) } diff --git a/stacktach/views.py b/stacktach/views.py index a26db0d..00f78b0 100644 --- a/stacktach/views.py +++ b/stacktach/views.py @@ -310,7 +310,7 @@ USAGE_PROCESS_MAPPING = { } GLANCE_USAGE_PROCESS_MAPPING = { - 'image.upload': _process_glance_usage, + 'image.activate': _process_glance_usage, 'image.delete': _process_glance_delete, 'image.exists': _process_glance_exists } diff --git a/tests/unit/test_notification.py b/tests/unit/test_notification.py index 4acd2e6..79dc0e9 100644 --- a/tests/unit/test_notification.py +++ b/tests/unit/test_notification.py @@ -396,17 +396,12 @@ class GlanceNotificationTestCase(unittest.TestCase): def test_save_delete_should_persist_image_delete(self): raw = self.mox.CreateMockAnything() - size = 123 uuid = "2df2ccf6-bc1b-4853-aab0-25fda346b3bb" deleted_at = "2013-06-20 14:31:57.939614" body = { "event_type": "image.delete", - "timestamp": "2013-06-20 18:31:57.939614", "publisher_id": "glance-api01-r2961.global.preprod-ord.ohthree.com", "payload": { - "created_at": str(DUMMY_TIME), - "size": size, - "owner": TENANT_ID_1, "id": "2df2ccf6-bc1b-4853-aab0-25fda346b3bb", "deleted_at": deleted_at } @@ -417,10 +412,7 @@ class GlanceNotificationTestCase(unittest.TestCase): self.mox.StubOutWithMock(db, 'create_image_delete') db.create_image_delete( - created_at=utils.str_time_to_unix(str(DUMMY_TIME)), - owner=TENANT_ID_1, raw=raw, - size=size, uuid=uuid, deleted_at=utils.str_time_to_unix(deleted_at)).AndReturn(raw) self.mox.ReplayAll() From 755e7061073423d646653f6f5a5ce615f8724ae0 Mon Sep 17 00:00:00 2001 From: Anuj Mathur Date: Fri, 12 Jul 2013 15:40:08 +0530 Subject: [PATCH 08/11] Added seed script to populate glance usage and deletes using the glance database --- util/glance_usage_seed.py | 145 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 145 insertions(+) create mode 100644 util/glance_usage_seed.py diff --git a/util/glance_usage_seed.py b/util/glance_usage_seed.py new file mode 100644 index 0000000..ff87032 --- /dev/null +++ b/util/glance_usage_seed.py @@ -0,0 +1,145 @@ +# Copyright (c) 2013 - Rackspace Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" + Usage: python glance_usage_seed.py [period_length] [sql_connection] + python glance_usage_seed.py hour mysql://user:password@nova-db.example + .com/nova?charset=utf8 + + The idea behind glance_usage seeding is to take the current state of all + active, deleted and pending_delete images from glance and insert that + data into Stacktach's image_usage and image_deletes tables. +""" + +import __builtin__ +setattr(__builtin__, '_', lambda x: x) +import datetime +import os +import sys +from oslo.config import cfg + +CONF = cfg.CONF +if __name__ == '__main__': + if len(sys.argv) != 3: + print "Proper Usage: glance_usage_seed.py [period_length] [" \ + "sql_connection]" + sys.exit(1) + CONF.sql_connection = sys.argv[2] + +import glance.context +import glance.db.sqlalchemy.api as db_api +from sqlalchemy import or_ +from sqlalchemy import and_ +import glance.db.sqlalchemy.api as db_api +from glance.db.sqlalchemy import models as glancemodels + +POSSIBLE_TOPDIR = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), + os.pardir, os.pardir)) +if os.path.exists(os.path.join(POSSIBLE_TOPDIR, 'stacktach')): + sys.path.insert(0, POSSIBLE_TOPDIR) + +from stacktach import datetime_to_decimal as dt +from stacktach import models + + +# start yanked from reports/nova_usage_audit.py +def get_period_start(time, period_length): + if period_length == 'day': + last_period = time - datetime.timedelta(days=1) + start = datetime.datetime(year=last_period.year, + month=last_period.month, + day=last_period.day) + return start + elif period_length == 'hour': + last_period = time - datetime.timedelta(hours=1) + start = datetime.datetime(year=last_period.year, + month=last_period.month, + day=last_period.day, + hour=last_period.hour) + return start +# end yanked from reports/nova_usage_audit.py + + +def _usage_for_image(image): + return { + 'uuid': image.id, + 'owner': image.owner, + 'created_at': dt.dt_to_decimal(image.created_at), + 'owner': image.owner, + 'size': image.size, + 'last_raw_id': None + } + + +def _delete_for_image(image): + return { + 'uuid': image.id, + 'deleted_at': dt.dt_to_decimal(image.deleted_at), + 'raw_id': None + } + + +def _get_usages(start, session): + usage_filter = (glancemodels.Image.status == 'active', + glancemodels.Image.deleted_at > start) + query = session.query(glancemodels.Image) + images = query.filter(or_(*usage_filter)).all() + return [_usage_for_image(image) for image in images] + + +def _get_deletes(start, session): + delete_filter = (glancemodels.Image.status == 'deleted', + glancemodels.Image.deleted_at > start) + query = session.query(glancemodels.Image) + images = query.filter(and_(*delete_filter)).all() + return [_delete_for_image(image) for image in images] + + +def seed(period_length): + start = get_period_start(datetime.datetime.utcnow(), period_length) + db_api.configure_db() + session = db_api.get_session() + + print "Populating active image usages" + usages = _get_usages(start, session) + + if usages: + print "Saving active image images" + active_images = map(lambda x: models.ImageUsage(**x), usages) + models.ImageUsage.objects.bulk_create(active_images, batch_size=100) + + print "Populating image deletes" + deletes = _get_deletes(start, session) + + if deletes: + print "Saving image deletes" + deleted_images = map(lambda x: models.ImageDeletes(**x), deletes) + models.ImageDeletes.objects.bulk_create(deleted_images, batch_size=100) + + print "Seeding completed" + return len(usages), len(deletes) + +if __name__ == '__main__': + msg = ("Seeded system with: \n" + "%s Active images \n" + "%s Deleted images \n") + period = sys.argv[1] + print msg % seed(period) + From 25baf823288430037b15e9bba2000827efc17936 Mon Sep 17 00:00:00 2001 From: Anuj Mathur Date: Mon, 15 Jul 2013 16:21:40 +0530 Subject: [PATCH 09/11] Removed image_type from GenericRawData Added message_id to GenericRawdata Regenerated migration --- ...reate_glance_usage_verification_tables.py} | 51 ++++++++++--------- stacktach/models.py | 18 ++++++- stacktach/notification.py | 28 +++++----- stacktach/tests.py | 4 +- stacktach/views.py | 2 +- tests/unit/test_notification.py | 11 ++-- 6 files changed, 69 insertions(+), 45 deletions(-) rename stacktach/migrations/{0005_create_glance_usage_verification_tables.py => 0006_create_glance_usage_verification_tables.py} (96%) diff --git a/stacktach/migrations/0005_create_glance_usage_verification_tables.py b/stacktach/migrations/0006_create_glance_usage_verification_tables.py similarity index 96% rename from stacktach/migrations/0005_create_glance_usage_verification_tables.py rename to stacktach/migrations/0006_create_glance_usage_verification_tables.py index b6d89af..1369fe1 100644 --- a/stacktach/migrations/0005_create_glance_usage_verification_tables.py +++ b/stacktach/migrations/0006_create_glance_usage_verification_tables.py @@ -48,6 +48,24 @@ class Migration(SchemaMigration): )) db.send_create_signal(u'stacktach', ['ImageUsage']) + # Adding model 'GenericRawData' + db.create_table(u'stacktach_genericrawdata', ( + (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('deployment', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['stacktach.Deployment'])), + ('tenant', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=50, null=True, blank=True)), + ('json', self.gf('django.db.models.fields.TextField')()), + ('routing_key', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=50, null=True, blank=True)), + ('when', self.gf('django.db.models.fields.DecimalField')(max_digits=20, decimal_places=6, db_index=True)), + ('publisher', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=100, null=True, blank=True)), + ('event', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=50, null=True, blank=True)), + ('service', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=50, null=True, blank=True)), + ('host', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=100, null=True, blank=True)), + ('instance', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=50, null=True, blank=True)), + ('request_id', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=50, null=True, blank=True)), + ('message_id', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=50, null=True, blank=True)), + )) + db.send_create_signal(u'stacktach', ['GenericRawData']) + # Adding model 'ImageExists' db.create_table(u'stacktach_imageexists', ( (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), @@ -67,24 +85,6 @@ class Migration(SchemaMigration): )) db.send_create_signal(u'stacktach', ['ImageExists']) - # Adding model 'GenericRawData' - db.create_table(u'stacktach_genericrawdata', ( - (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), - ('deployment', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['stacktach.Deployment'])), - ('tenant', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=50, null=True, blank=True)), - ('json', self.gf('django.db.models.fields.TextField')()), - ('routing_key', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=50, null=True, blank=True)), - ('image_type', self.gf('django.db.models.fields.IntegerField')(default=0, null=True, db_index=True)), - ('when', self.gf('django.db.models.fields.DecimalField')(max_digits=20, decimal_places=6, db_index=True)), - ('publisher', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=100, null=True, blank=True)), - ('event', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=50, null=True, blank=True)), - ('service', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=50, null=True, blank=True)), - ('host', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=100, null=True, blank=True)), - ('instance', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=50, null=True, blank=True)), - ('request_id', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=50, null=True, blank=True)), - )) - db.send_create_signal(u'stacktach', ['GenericRawData']) - def backwards(self, orm): # Deleting model 'ImageDeletes' @@ -96,12 +96,12 @@ class Migration(SchemaMigration): # Deleting model 'ImageUsage' db.delete_table(u'stacktach_imageusage') - # Deleting model 'ImageExists' - db.delete_table(u'stacktach_imageexists') - # Deleting model 'GenericRawData' db.delete_table(u'stacktach_genericrawdata') + # Deleting model 'ImageExists' + db.delete_table(u'stacktach_imageexists') + models = { u'stacktach.deployment': { @@ -115,9 +115,9 @@ class Migration(SchemaMigration): 'event': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), 'host': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '100', 'null': 'True', 'blank': 'True'}), u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'image_type': ('django.db.models.fields.IntegerField', [], {'default': '0', 'null': 'True', 'db_index': 'True'}), 'instance': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), 'json': ('django.db.models.fields.TextField', [], {}), + 'message_id': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), 'publisher': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '100', 'null': 'True', 'blank': 'True'}), 'request_id': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), 'routing_key': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), @@ -213,9 +213,14 @@ class Migration(SchemaMigration): 'instance': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), 'instance_type_id': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), 'launched_at': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}), + 'os_architecture': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'os_distro': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'os_version': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'rax_options': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), 'row_created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), 'row_updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}), - 'source': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '150', 'null': 'True', 'blank': 'True'}) + 'source': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '150', 'null': 'True', 'blank': 'True'}), + 'tenant': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}) }, u'stacktach.instanceusage': { 'Meta': {'object_name': 'InstanceUsage'}, diff --git a/stacktach/models.py b/stacktach/models.py index 0fa2734..5e233bd 100644 --- a/stacktach/models.py +++ b/stacktach/models.py @@ -1,3 +1,18 @@ +# Copyright 2012 - Dark Secret Software Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + from django import forms from django.db import models @@ -16,7 +31,6 @@ class GenericRawData(models.Model): json = models.TextField() routing_key = models.CharField(max_length=50, null=True, blank=True, db_index=True) - image_type = models.IntegerField(null=True, default=0, db_index=True) when = models.DecimalField(max_digits=20, decimal_places=6, db_index=True) publisher = models.CharField(max_length=100, null=True, @@ -31,6 +45,8 @@ class GenericRawData(models.Model): blank=True, db_index=True) request_id = models.CharField(max_length=50, null=True, blank=True, db_index=True) + message_id = models.CharField(max_length=50, null=True, + blank=True, db_index=True) @staticmethod def get_name(): diff --git a/stacktach/notification.py b/stacktach/notification.py index e30a032..2cd1ef8 100644 --- a/stacktach/notification.py +++ b/stacktach/notification.py @@ -72,18 +72,23 @@ class Notification(object): instance = self.payload.get('instance', {}).get('uuid') return instance + @property + def message_id(self): + return self.body.get('message_id', None) + def save(self): return db.create_generic_rawdata(deployment=self.deployment, - routing_key=self.routing_key, - tenant=self.tenant, - json=self.json, - when=self.when, - publisher=self.publisher, - event=self.event, - service=self.service, - host=self.host, - instance=self.instance, - request_id=self.request_id) + routing_key=self.routing_key, + tenant=self.tenant, + json=self.json, + when=self.when, + publisher=self.publisher, + event=self.event, + service=self.service, + host=self.host, + instance=self.instance, + request_id=self.request_id, + message_id=self.message_id) class GlanceNotification(Notification): @@ -179,6 +184,7 @@ class GlanceNotification(Notification): } db.create_image_delete(**values) + class NovaNotification(Notification): def __init__(self, body, deployment, routing_key, json): super(NovaNotification, self).__init__(body, deployment, routing_key, @@ -203,8 +209,6 @@ class NovaNotification(Notification): 'audit_period_beginning', None) self.audit_period_ending = self.payload.get( 'audit_period_ending', None) - self.message = self.payload.get('message', None) - self.message_id = self.body.get('message_id', None) @property def host(self): diff --git a/stacktach/tests.py b/stacktach/tests.py index a4539a2..d3760b6 100644 --- a/stacktach/tests.py +++ b/stacktach/tests.py @@ -142,13 +142,13 @@ class GenericRawDataTestCase(TransactionTestCase): 'tenant': '1234567', 'json': '{}', 'routing_key': 'monitor.info', - 'image_type': 1, 'publisher': 'publisher', 'event': 'event', 'service': 'service', 'host': 'host', 'instance': '1234-5678-9012-3456', - 'request_id': '1234'} + 'request_id': '1234', + 'message_id': 'message_id'} db.create_generic_rawdata(**kwargs) rawdata = GenericRawData.objects.all()[0] diff --git a/stacktach/views.py b/stacktach/views.py index 00f78b0..14c5d08 100644 --- a/stacktach/views.py +++ b/stacktach/views.py @@ -349,7 +349,7 @@ def post_process_glancerawdata(raw, notification): aggregate_glance_usage(raw, notification) -def post_process_genericrawdata(raw, body, notification): +def post_process_genericrawdata(raw, notification): pass diff --git a/tests/unit/test_notification.py b/tests/unit/test_notification.py index 79dc0e9..926d118 100644 --- a/tests/unit/test_notification.py +++ b/tests/unit/test_notification.py @@ -35,6 +35,7 @@ from tests.unit.utils import DUMMY_TIME from tests.unit.utils import TIMESTAMP_1 from tests.unit.utils import TENANT_ID_1 from tests.unit.utils import INSTANCE_ID_1 +from tests.unit.utils import MESSAGE_ID_1 class NovaNotificationTestCase(unittest.TestCase): @@ -422,7 +423,6 @@ class GlanceNotificationTestCase(unittest.TestCase): self.mox.VerifyAll() - class NotificationTestCase(unittest.TestCase): def setUp(self): self.mox = mox.Mox() @@ -437,14 +437,12 @@ class NotificationTestCase(unittest.TestCase): '_context_project_id': TENANT_ID_1, "timestamp": TIMESTAMP_1, "publisher_id": "glance-api01-r2961.global.preprod-ord.ohthree.com", + "message_id": MESSAGE_ID_1, "payload": { 'instance_id': INSTANCE_ID_1, "status": "saving", "container_format": "ovf", - "properties": { - "image_type": "snapshot", - }, - "tenant": "5877054", + "tenant": "5877054" } } deployment = "1" @@ -463,7 +461,8 @@ class NotificationTestCase(unittest.TestCase): service="glance-api01-r2961", host="global.preprod-ord.ohthree.com", instance=INSTANCE_ID_1, - request_id=REQUEST_ID_1).AndReturn(raw) + request_id=REQUEST_ID_1, + message_id=MESSAGE_ID_1).AndReturn(raw) self.mox.ReplayAll() From 974427aa6979dfef70de61b59a051652931649d4 Mon Sep 17 00:00:00 2001 From: Manali Latkar Date: Thu, 18 Jul 2013 15:58:18 +0530 Subject: [PATCH 10/11] Added stacky server api methods show, do_uuid, search and show --- stacktach/models.py | 40 ++++- stacktach/stacky_server.py | 142 ++++++++++++++---- stacktach/tests.py | 83 +++++++++++ stacktach/urls.py | 9 +- tests/unit/test_stacky_server.py | 242 +++++++++++++++++++++++++++++-- 5 files changed, 471 insertions(+), 45 deletions(-) diff --git a/stacktach/models.py b/stacktach/models.py index 5e233bd..410b2a6 100644 --- a/stacktach/models.py +++ b/stacktach/models.py @@ -13,10 +13,15 @@ # License for the specific language governing permissions and limitations # under the License. -from django import forms +import copy from django.db import models +def routing_key_type(key): + if key.endswith('error'): + return 'E' + return ' ' + class Deployment(models.Model): name = models.CharField(max_length=50) @@ -25,6 +30,8 @@ class Deployment(models.Model): class GenericRawData(models.Model): + result_titles = [["#", "?", "When", "Deployment", "Event", "Host", + "Instance", "Request id"]] deployment = models.ForeignKey(Deployment) tenant = models.CharField(max_length=50, null=True, blank=True, db_index=True) @@ -52,8 +59,17 @@ class GenericRawData(models.Model): def get_name(): return GenericRawData.__name__ + def search_results(self, results, when, routing_key_status): + if not results: + results = copy.deepcopy(self.result_titles) + results.append([self.id, routing_key_status, str(when), + self.deployment.name, self.event, self.host, + self.instance, self.request_id]) + return results class RawData(models.Model): + result_titles = [["#", "?", "When", "Deployment", "Event", "Host", + "State", "State'", "Task'"]] deployment = models.ForeignKey(Deployment) tenant = models.CharField(max_length=50, null=True, blank=True, db_index=True) @@ -87,10 +103,22 @@ class RawData(models.Model): def __repr__(self): return "%s %s %s" % (self.event, self.instance, self.state) + @property + def uuid(self): + return self.instance + @staticmethod def get_name(): return RawData.__name__ + def search_results(self, results, when, routing_key_status): + if not results: + results = copy.deepcopy(self.result_titles) + results.append([self.id, routing_key_status, str(when), + self.deployment.name, self.event, self.host, self.state, + self.old_state, self.old_task]) + return results + class RawDataImageMeta(models.Model): raw = models.ForeignKey(RawData, null=False) @@ -273,6 +301,8 @@ class JsonReport(models.Model): class GlanceRawData(models.Model): + result_titles = [["#", "?", "When", "Deployment", "Event", "Host", + "Status"]] ACTIVE = 'active' DELETED = 'deleted' KILLED = 'killed' @@ -317,6 +347,14 @@ class GlanceRawData(models.Model): def get_name(): return GlanceRawData.__name__ + def search_results(self, results, when, routing_key_status): + if not results: + results = copy.deepcopy(self.result_titles) + results.append([self.id, routing_key_status, str(when), + self.deployment.name, self.event, self.host, + self.status]) + return results + class ImageUsage(models.Model): uuid = models.CharField(max_length=50, db_index=True) diff --git a/stacktach/stacky_server.py b/stacktach/stacky_server.py index f42116c..70c732e 100644 --- a/stacktach/stacky_server.py +++ b/stacktach/stacky_server.py @@ -9,13 +9,14 @@ from django.shortcuts import get_object_or_404 import datetime_to_decimal as dt import models import utils +from django.core.exceptions import ObjectDoesNotExist, FieldError SECS_PER_HOUR = 60 * 60 SECS_PER_DAY = SECS_PER_HOUR * 24 -def get_event_names(): - return models.RawData.objects.values('event').distinct() +def get_event_names(service='nova'): + return _model_factory(service).values('event').distinct() def get_host_names(): @@ -104,22 +105,25 @@ def do_hosts(request): return rsp(json.dumps(results)) -def do_uuid(request): +def do_uuid(request, service='nova'): uuid = str(request.GET['uuid']) if not utils.is_uuid_like(uuid): msg = "%s is not uuid-like" % uuid return error_response(400, 'Bad Request', msg) + model = _model_factory(service) + result = {} + param = {} + if service == 'nova' or service == 'generic': + param = {'instance': uuid} + if service == 'glance': + param = {'uuid': uuid} - related = models.RawData.objects.select_related().filter(instance=uuid)\ - .order_by('when') - results = [["#", "?", "When", "Deployment", "Event", "Host", "State", - "State'", "Task'"]] - for e in related: - when = dt.dt_from_decimal(e.when) - results.append([e.id, routing_key_type(e.routing_key), str(when), - e.deployment.name, e.event, e.host, e.state, - e.old_state, e.old_task]) - return rsp(json.dumps(results)) + related = model.select_related().filter(**param).order_by('when') + for event in related: + when = dt.dt_from_decimal(event.when) + routing_key_status = routing_key_type(event.routing_key) + result = event.search_results(result, when, routing_key_status) + return rsp(json.dumps(result)) def do_timings_uuid(request): @@ -202,15 +206,7 @@ def do_request(request): return rsp(json.dumps(results)) -def do_show(request, event_id): - event_id = int(event_id) - results = [] - event = None - try: - event = models.RawData.objects.get(id=event_id) - except models.RawData.ObjectDoesNotExist: - return results - +def append_nova_raw_attributes(event, results): results.append(["Key", "Value"]) results.append(["#", event.id]) when = dt.dt_from_decimal(event.when) @@ -224,16 +220,77 @@ def do_show(request, event_id): results.append(["Host", event.host]) results.append(["UUID", event.instance]) results.append(["Req ID", event.request_id]) - - final = [results, ] - j = json.loads(event.json) - final.append(json.dumps(j, indent=2)) - final.append(event.instance) - - return rsp(json.dumps(final)) + return results -def do_watch(request, deployment_id): +def append_glance_raw_attributes(event, results): + results.append(["Key", "Value"]) + results.append(["#", event.id]) + when = dt.dt_from_decimal(event.when) + results.append(["When", str(when)]) + results.append(["Deployment", event.deployment.name]) + results.append(["Category", event.routing_key]) + results.append(["Publisher", event.publisher]) + results.append(["Status", event.status]) + results.append(["Event", event.event]) + results.append(["Service", event.service]) + results.append(["Host", event.host]) + results.append(["UUID", event.uuid]) + results.append(["Req ID", event.request_id]) + return results + + +def append_generic_raw_attributes(event, results): + results.append(["Key", "Value"]) + results.append(["#", event.id]) + when = dt.dt_from_decimal(event.when) + results.append(["When", str(when)]) + results.append(["Deployment", event.deployment.name]) + results.append(["Category", event.routing_key]) + results.append(["Publisher", event.publisher]) + results.append(["State", event.state]) + results.append(["Event", event.event]) + results.append(["Service", event.service]) + results.append(["Host", event.host]) + results.append(["UUID", event.instance]) + results.append(["Req ID", event.request_id]) + return results + +def _append_raw_attributes(event, results, service): + if service == 'nova': + return append_nova_raw_attributes(event, results) + if service == 'glance': + return append_glance_raw_attributes(event, results) + if service == 'generic': + return append_generic_raw_attributes(event, results) + +def do_show(request, event_id, service='nova'): + event_id = int(event_id) + results = [] + model = _model_factory(service) + try: + event = model.get(id=event_id) + results = _append_raw_attributes(event, results, service) + final = [results, ] + j = json.loads(event.json) + final.append(json.dumps(j, indent=2)) + final.append(event.uuid) + return rsp(json.dumps(final)) + except ObjectDoesNotExist: + return rsp({}) + + +def _model_factory(service): + if service == 'glance': + return models.GlanceRawData.objects + elif service == 'nova': + return models.RawData.objects + elif service == 'generic': + return models.GenericRawData.objects + + +def do_watch(request, deployment_id, service='nova'): + model = _model_factory(service) deployment_id = int(deployment_id) since = request.GET.get('since') event_name = request.GET.get('event_name') @@ -244,7 +301,7 @@ def do_watch(request, deployment_id): events = get_event_names() max_event_width = max([len(event['event']) for event in events]) - base_events = models.RawData.objects.order_by('when') + base_events = model.order_by('when') if deployment_id > 0: base_events = base_events.filter(deployment=deployment_id) @@ -426,3 +483,26 @@ def do_jsonreport(request, report_id): report_id = int(report_id) report = get_object_or_404(models.JsonReport, pk=report_id) return rsp(report.json) + + +def search(request, service): + DEFAULT = 1000 + field = request.GET.get('field') + value = request.GET.get('value') + limit = request.GET.get('limit', DEFAULT) + limit = int(limit) + model = _model_factory(service) + filter_para = {field: value} + results = {} + try: + events = model.filter(**filter_para) + event_len = len(events) + if event_len > limit: + events = events[0:limit] + for event in events: + when = dt.dt_from_decimal(event.when) + routing_key_status = routing_key_type(event.routing_key) + results = event.search_results(results, when, routing_key_status) + return rsp(json.dumps(results)) + except ObjectDoesNotExist or FieldError: + return rsp({}) diff --git a/stacktach/tests.py b/stacktach/tests.py index d3760b6..a2549a7 100644 --- a/stacktach/tests.py +++ b/stacktach/tests.py @@ -27,6 +27,8 @@ from stacktach.models import GenericRawData from stacktach.models import GlanceRawData from stacktach.models import RawData from stacktach.models import get_model_fields +from stacktach import datetime_to_decimal as dt + class RawDataImageMetaDbTestCase(TransactionTestCase): @@ -157,3 +159,84 @@ class GenericRawDataTestCase(TransactionTestCase): if field.name != 'id': self.assertEquals(getattr(rawdata, field.name), kwargs[field.name]) + + +class NovaRawDataSearchTestCase(TransactionTestCase): + def test_search_results_for_nova(self): + expected_result = [['#', '?', 'When', 'Deployment', 'Event', 'Host', + 'State', "State'", "Task'"], [1L, ' ', + '2013-07-17 10:16:10.717219', 'depl', 'event', + 'host', 'state', 'old_state', 'old_task']] + depl = db.get_or_create_deployment('depl')[0] + when = dt.dt_to_decimal(datetime.utcnow()) + raw = db.create_nova_rawdata(deployment=depl, + routing_key='routing_key', + tenant='tenant', + json='json', + when=when, + publisher='publisher', + event='event', + service='nova', + host='host', + instance='instance', + request_id='req-1234', + state='state', + old_state='old_state', + task='task', + old_task='old_task', + os_architecture='arch', + os_distro='distro', + os_version='version', + rax_options=1) + + results = raw.search_results({}, "2013-07-17 10:16:10.717219", ' ') + self.assertEqual(results,expected_result) + + def test_search_results_for_glance(self): + expected_result = [['#', '?', 'When', 'Deployment', 'Event', 'Host', + 'Status'], [1L, ' ', + '2013-07-17 10:16:10.717219', 'depl', 'event', + 'host', 'status']] + depl = db.get_or_create_deployment('depl')[0] + when = dt.dt_to_decimal(datetime.utcnow()) + + glance_raw = db.create_glance_rawdata(deployment=depl, + routing_key='routing_key', + json='json', + when=when, + publisher='publisher', + event='event', + service='glance', + host='host', + uuid='instance', + request_id='req-1234', + status='status', + image_type=1) + + results = glance_raw.search_results({}, "2013-07-17 10:16:10.717219", + ' ') + self.assertEqual(results,expected_result) + + def test_search_results_for_generic(self): + expected_result = [['#', '?', 'When', 'Deployment', 'Event', 'Host', + 'Instance', 'Request id'], [1L, ' ', + '2013-07-17 10:16:10.717219', 'depl', 'event', + 'host', 'instance', 'req-1234']] + depl = db.get_or_create_deployment('depl')[0] + when = dt.dt_to_decimal(datetime.utcnow()) + + generic_raw = db.create_generic_rawdata(deployment=depl, + routing_key='routing_key', + json='json', + when=when, + publisher='publisher', + event='event', + service='glance', + host='host', + instance='instance', + request_id='req-1234', + tenant='tenant') + + results = generic_raw.search_results({}, "2013-07-17 10:16:10.717219", + ' ') + self.assertEqual(results,expected_result) diff --git a/stacktach/urls.py b/stacktach/urls.py index 3670d30..e82788f 100644 --- a/stacktach/urls.py +++ b/stacktach/urls.py @@ -1,4 +1,4 @@ -from django.conf.urls.defaults import patterns, include, url +from django.conf.urls import patterns, url urlpatterns = patterns('', @@ -7,7 +7,7 @@ urlpatterns = patterns('', url(r'stacky/deployments/$', 'stacktach.stacky_server.do_deployments'), url(r'stacky/events/$', 'stacktach.stacky_server.do_events'), url(r'stacky/hosts/$', 'stacktach.stacky_server.do_hosts'), - url(r'stacky/uuid/$', 'stacktach.stacky_server.do_uuid'), + url(r'stacky/uuid/(?P\w+)/$', 'stacktach.stacky_server.do_uuid'), url(r'stacky/timings/$', 'stacktach.stacky_server.do_timings'), url(r'stacky/timings/uuid/$', 'stacktach.stacky_server.do_timings_uuid'), url(r'stacky/summary/$', 'stacktach.stacky_server.do_summary'), @@ -17,8 +17,11 @@ urlpatterns = patterns('', 'stacktach.stacky_server.do_jsonreport'), url(r'stacky/show/(?P\d+)/$', 'stacktach.stacky_server.do_show'), - url(r'stacky/watch/(?P\d+)/$', + url(r'stacky/show/(?P\w+)/(?P\d+)$', + 'stacktach.stacky_server.do_show'), + url(r'stacky/watch/(?P\d+)/(?P\w+)$', 'stacktach.stacky_server.do_watch'), + url(r'stacky/search/(?P\w+)/$', 'stacktach.stacky_server.search'), url(r'stacky/kpi/$', 'stacktach.stacky_server.do_kpi'), url(r'stacky/kpi/(?P\w+)/$', 'stacktach.stacky_server.do_kpi'), url(r'stacky/usage/launches/$', diff --git a/tests/unit/test_stacky_server.py b/tests/unit/test_stacky_server.py index 0b0ee24..96e011b 100644 --- a/tests/unit/test_stacky_server.py +++ b/tests/unit/test_stacky_server.py @@ -40,6 +40,12 @@ class StackyServerTestCase(unittest.TestCase): self.mox.StubOutWithMock(models, 'RawData', use_mock_anything=True) models.RawData.objects = self.mox.CreateMockAnything() self.mox.StubOutWithMock(models, 'Deployment', use_mock_anything=True) + self.mox.StubOutWithMock(models, 'GlanceRawData', + use_mock_anything=True) + models.GlanceRawData.objects = self.mox.CreateMockAnything() + self.mox.StubOutWithMock(models, 'GenericRawData', + use_mock_anything=True) + models.GenericRawData.objects = self.mox.CreateMockAnything() models.Deployment.objects = self.mox.CreateMockAnything() self.mox.StubOutWithMock(models, 'Lifecycle', use_mock_anything=True) models.Lifecycle.objects = self.mox.CreateMockAnything() @@ -65,7 +71,8 @@ class StackyServerTestCase(unittest.TestCase): def _create_raw(self): raw = self.mox.CreateMockAnything() - raw.when = utils.decimal_utc() + raw.when = utils.decimal_utc(datetime.datetime(2013, 7, 17, 10, 16, + 10, 717219)) raw.instance = INSTANCE_ID_1 raw.id = 1 raw.routing_key = 'monitor.info' @@ -80,13 +87,19 @@ class StackyServerTestCase(unittest.TestCase): raw.publisher = "api.example.com" raw.service = 'api' raw.host = 'example.com' + raw.status = 'state' raw.request_id = REQUEST_ID_1 raw.json = '{"key": "value"}' + raw.uuid = 'uuid' + raw.tenant = 'tenant' return raw def test_get_event_names(self): + model = self.mox.CreateMockAnything() result = self.mox.CreateMockAnything() - models.RawData.objects.values('event').AndReturn(result) + self.mox.StubOutWithMock(stacky_server, '_model_factory') + stacky_server._model_factory('nova').AndReturn(model) + model.values('event').AndReturn(result) result.distinct().AndReturn(result) self.mox.ReplayAll() @@ -248,6 +261,10 @@ class StackyServerTestCase(unittest.TestCase): self.mox.VerifyAll() def test_do_uuid(self): + search_result = [["#", "?", "When", "Deployment", "Event", "Host", + "State", "State'", "Task'"], [1, " ", + "2013-07-17 10:16:10.717219", "deployment", + "test.start", "example.com", "active", None, None]] fake_request = self.mox.CreateMockAnything() fake_request.GET = {'uuid': INSTANCE_ID_1} result = self.mox.CreateMockAnything() @@ -256,6 +273,7 @@ class StackyServerTestCase(unittest.TestCase): result.order_by('when').AndReturn(result) raw = self._create_raw() result.__iter__().AndReturn([raw].__iter__()) + raw.search_results({}, mox.IgnoreArg(), ' ').AndReturn(search_result) self.mox.ReplayAll() resp = stacky_server.do_uuid(fake_request) @@ -272,6 +290,36 @@ class StackyServerTestCase(unittest.TestCase): self.assertEqual(json_resp[1], body) self.mox.VerifyAll() + def test_do_uuid_for_glance(self): + search_result = [["#", "?", "When", "Deployment", "Event", "Host", + "Status"], [1, " ", + "2013-07-17 10:16:10.717219", "deployment", + "test.start", "example.com", "state"]] + fake_request = self.mox.CreateMockAnything() + fake_request.GET = {'uuid': INSTANCE_ID_1} + result = self.mox.CreateMockAnything() + models.GlanceRawData.objects.select_related().AndReturn(result) + result.filter(uuid=INSTANCE_ID_1).AndReturn(result) + result.order_by('when').AndReturn(result) + raw = self._create_raw() + result.__iter__().AndReturn([raw].__iter__()) + raw.search_results({}, mox.IgnoreArg(), ' ').AndReturn(search_result) + self.mox.ReplayAll() + + resp = stacky_server.do_uuid(fake_request,'glance') + + self.assertEqual(resp.status_code, 200) + json_resp = json.loads(resp.content) + self.assertEqual(len(json_resp), 2) + header = ["#", "?", "When", "Deployment", "Event", "Host", + "Status"] + self.assertEqual(json_resp[0], header) + datetime = dt.dt_from_decimal(raw.when) + body = [1, " ", str(datetime), "deployment", "test.start", + "example.com", "state"] + self.assertEqual(json_resp[1], body) + self.mox.VerifyAll() + def test_do_uuid_bad_uuid(self): fake_request = self.mox.CreateMockAnything() fake_request.GET = {'uuid': "obviouslybaduuid"} @@ -504,7 +552,9 @@ class StackyServerTestCase(unittest.TestCase): self.assertEqual(resp_json[1], ['Bad Request', msg]) self.mox.VerifyAll() - def _assert_on_show(self, values, raw): + def _assert_on_show_nova(self, json_resp, raw): + self.assertEqual(len(json_resp), 3) + values = json_resp[0] self.assertEqual(len(values), 12) self.assertEqual(values[0], ["Key", "Value"]) self.assertEqual(values[1], ["#", raw.id]) @@ -517,7 +567,25 @@ class StackyServerTestCase(unittest.TestCase): self.assertEqual(values[7], ["Event", raw.event]) self.assertEqual(values[8], ["Service", raw.service]) self.assertEqual(values[9], ["Host", raw.host]) - self.assertEqual(values[10], ["UUID", raw.instance]) + self.assertEqual(values[10],["UUID", raw.instance]) + self.assertEqual(values[11], ["Req ID", raw.request_id]) + + def _assert_on_show_glance(self, json_resp, raw): + self.assertEqual(len(json_resp), 3) + values = json_resp[0] + self.assertEqual(len(values), 12) + self.assertEqual(values[0], ["Key", "Value"]) + self.assertEqual(values[1], ["#", raw.id]) + self.assertEqual(values[2], ["When", + str(dt.dt_from_decimal(raw.when))]) + self.assertEqual(values[3], ["Deployment", raw.deployment.name]) + self.assertEqual(values[4], ["Category", raw.routing_key]) + self.assertEqual(values[5], ["Publisher", raw.publisher]) + self.assertEqual(values[6], ["Status", raw.status]) + self.assertEqual(values[7], ["Event", raw.event]) + self.assertEqual(values[8], ["Service", raw.service]) + self.assertEqual(values[9], ["Host", raw.host]) + self.assertEqual(values[10],["UUID", raw.uuid]) self.assertEqual(values[11], ["Req ID", raw.request_id]) def test_do_show(self): @@ -528,10 +596,81 @@ class StackyServerTestCase(unittest.TestCase): resp = stacky_server.do_show(fake_request, 1) + self.assertEqual(resp.status_code, 200) + json_resp = json.loads(resp.content) + self._assert_on_show_nova(json_resp, raw) + self.mox.VerifyAll() + + def test_do_show_for_glance_rawdata(self): + fake_request = self.mox.CreateMockAnything() + raw = self._create_raw() + models.GlanceRawData.objects.get(id=1).AndReturn(raw) + self.mox.ReplayAll() + + resp = stacky_server.do_show(fake_request, 1, 'glance') + + self.assertEqual(resp.status_code, 200) + json_resp = json.loads(resp.content) + self._assert_on_show_glance(json_resp, raw) + self.mox.VerifyAll() + + def test_do_show_for_generic_rawdata(self): + fake_request = self.mox.CreateMockAnything() + raw = self._create_raw() + models.GenericRawData.objects.get(id=1).AndReturn(raw) + self.mox.ReplayAll() + + resp = stacky_server.do_show(fake_request, 1, 'generic') + + self.assertEqual(resp.status_code, 200) + json_resp = json.loads(resp.content) + self._assert_on_show_nova(json_resp, raw) + self.mox.VerifyAll() + + def test_do_show_should_return_empty_result_on_object_not_found_exception(self): + fake_request = self.mox.CreateMockAnything() + raw = self._create_raw() + models.RawData.objects.get(id=1).AndReturn(raw) + self.mox.ReplayAll() + + resp = stacky_server.do_show(fake_request, 1) + + self.assertEqual(resp.status_code, 200) + json_resp = json.loads(resp.content) + self._assert_on_show_nova(json_resp, raw) + self.mox.VerifyAll() + + def test_do_watch_for_glance(self): + fake_request = self.mox.CreateMockAnything() + fake_request.GET = {} + self.mox.StubOutWithMock(stacky_server, 'get_deployments') + deployment1 = self.mox.CreateMockAnything() + deployment1.id = 1 + deployment1.name = 'dep1' + deployments = [deployment1] + stacky_server.get_deployments().AndReturn(deployments) + self.mox.StubOutWithMock(stacky_server, 'get_event_names') + events = [{'event': 'test.start'}, {'event': 'test.end'}] + stacky_server.get_event_names().AndReturn(events) + results = self.mox.CreateMockAnything() + models.GlanceRawData.objects.order_by('when').AndReturn(results) + results.filter(when__gt=mox.IgnoreArg()).AndReturn(results) + results.filter(when__lte=mox.IgnoreArg()).AndReturn(results) + results.__iter__().AndReturn([self._create_raw()].__iter__()) + self.mox.ReplayAll() + + resp = stacky_server.do_watch(fake_request, 0, 'glance') self.assertEqual(resp.status_code, 200) json_resp = json.loads(resp.content) self.assertEqual(len(json_resp), 3) - self._assert_on_show(json_resp[0], raw) + self.assertEqual(json_resp[0], [10, 1, 15, 20, 10, 36]) + self.assertEqual(json_resp[1][0][0], 1) + self.assertEqual(json_resp[1][0][1], u' ') + time_str = "%s %s" % (json_resp[1][0][2], json_resp[1][0][3]) + datetime.datetime.strptime(time_str, "%Y-%m-%d %H:%M:%S.%f") + self.assertEqual(json_resp[1][0][4], u'dep1') + self.assertEqual(json_resp[1][0][5], u'test.start') + self.assertEqual(json_resp[1][0][6], u'%s' % INSTANCE_ID_1) self.mox.VerifyAll() def test_do_watch(self): @@ -547,7 +686,10 @@ class StackyServerTestCase(unittest.TestCase): events = [{'event': 'test.start'}, {'event': 'test.end'}] stacky_server.get_event_names().AndReturn(events) results = self.mox.CreateMockAnything() - models.RawData.objects.order_by('when').AndReturn(results) + model = self.mox.CreateMockAnything() + self.mox.StubOutWithMock(stacky_server, '_model_factory') + stacky_server._model_factory('nova').AndReturn(model) + model.order_by('when').AndReturn(results) results.filter(when__gt=mox.IgnoreArg()).AndReturn(results) results.filter(when__lte=mox.IgnoreArg()).AndReturn(results) results.__iter__().AndReturn([self._create_raw()].__iter__()) @@ -558,7 +700,6 @@ class StackyServerTestCase(unittest.TestCase): json_resp = json.loads(resp.content) self.assertEqual(len(json_resp), 3) self.assertEqual(json_resp[0], [10, 1, 15, 20, 10, 36]) - print json_resp self.assertEqual(json_resp[1][0][0], 1) self.assertEqual(json_resp[1][0][1], u' ') time_str = "%s %s" % (json_resp[1][0][2], json_resp[1][0][3]) @@ -581,7 +722,11 @@ class StackyServerTestCase(unittest.TestCase): events = [{'event': 'test.start'}, {'event': 'test.end'}] stacky_server.get_event_names().AndReturn(events) results = self.mox.CreateMockAnything() - models.RawData.objects.order_by('when').AndReturn(results) + model = self.mox.CreateMockAnything() + self.mox.StubOutWithMock(stacky_server, '_model_factory') + stacky_server._model_factory('nova').AndReturn(model) + model.order_by('when').AndReturn(results) + results.filter(deployment=1).AndReturn(results) results.filter(when__gt=mox.IgnoreArg()).AndReturn(results) results.filter(when__lte=mox.IgnoreArg()).AndReturn(results) @@ -593,7 +738,6 @@ class StackyServerTestCase(unittest.TestCase): json_resp = json.loads(resp.content) self.assertEqual(len(json_resp), 3) self.assertEqual(json_resp[0], [10, 1, 15, 20, 10, 36]) - print json_resp self.assertEqual(json_resp[1][0][0], 1) self.assertEqual(json_resp[1][0][1], u' ') time_str = "%s %s" % (json_resp[1][0][2], json_resp[1][0][3]) @@ -628,7 +772,6 @@ class StackyServerTestCase(unittest.TestCase): json_resp = json.loads(resp.content) self.assertEqual(len(json_resp), 3) self.assertEqual(json_resp[0], [10, 1, 15, 20, 10, 36]) - print json_resp self.assertEqual(json_resp[1][0][0], 1) self.assertEqual(json_resp[1][0][1], u' ') time_str = "%s %s" % (json_resp[1][0][2], json_resp[1][0][3]) @@ -960,3 +1103,82 @@ class StackyServerTestCase(unittest.TestCase): self.assertEqual(resp_json[1], ['Bad Request', msg]) self.mox.VerifyAll() + def test_model_factory_for_nova(self): + self.mox.UnsetStubs() + nova_model = stacky_server._model_factory('nova') + self.assertEqual(nova_model.model, models.RawData) + + def test_model_factory_for_nova(self): + self.mox.UnsetStubs() + nova_model = stacky_server._model_factory('glance') + self.assertEqual(nova_model.model, models.GlanceRawData) + + def test_model_factory_for_nova(self): + self.mox.UnsetStubs() + nova_model = stacky_server._model_factory('generic') + self.assertEqual(nova_model.model, models.GenericRawData) + + def _assert_on_search_nova(self, json_resp, raw): + title = json_resp[0] + values = json_resp[1] + self.assertEqual(len(values), 9) + self.assertEqual([title[0], values[0]],["#", raw.id] ) + self.assertEqual([title[1], values[1]], ['?', ' ']) + self.assertEqual([title[2], values[2]], ["When", + str(dt.dt_from_decimal(raw.when))]) + self.assertEqual([title[3], values[3]], ["Deployment", raw.deployment.name]) + self.assertEqual([title[4], values[4]], ["Event", raw.event]) + self.assertEqual([title[5], values[5]], ["Host", raw.host]) + self.assertEqual([title[6], values[6]], ["State", raw.state]) + self.assertEqual([title[7], values[7]], ["State'", raw.old_state]) + + def test_search_by_field_for_nova(self): + search_result = [["#", "?", "When", "Deployment", "Event", "Host", + "State", "State'", "Task'"], [1, " ", + "2013-07-17 10:16:10.717219", "deployment", + "test.start", "example.com", "active", None, None]] + fake_request = self.mox.CreateMockAnything() + fake_request.GET = {'field': 'tenant', 'value': 'tenant'} + raw = self._create_raw() + models.RawData.objects.filter(tenant='tenant').AndReturn([raw]) + raw.search_results({}, mox.IgnoreArg(), ' ').AndReturn(search_result) + self.mox.ReplayAll() + + resp = stacky_server.search(fake_request, 'nova') + + self.assertEqual(resp.status_code, 200) + json_resp = json.loads(resp.content) + self._assert_on_search_nova(json_resp, raw) + self.mox.VerifyAll() + + def test_search_by_field_for_nova_with_limit(self): + search_result = [["#", "?", "When", "Deployment", "Event", "Host", + "State", "State'", "Task'"], [1, " ", + "2013-07-17 10:16:10.717219", "deployment", + "test.start", "example.com", "active", None, None]] + search_result_2 = [["#", "?", "When", "Deployment", "Event", "Host", + "State", "State'", "Task'"], [1, " ", + "2013-07-17 10:16:10.717219", "deployment", + "test.start", "example.com", "active", None, None],[2, " ", + "2013-07-17 10:16:10.717219", "deployment", + "test.start", "example.com", "active", None, None]] + fake_request = self.mox.CreateMockAnything() + fake_request.GET = {'field': 'tenant', 'value': 'tenant', 'limit': '2'} + raw1 = self._create_raw() + raw2 = self._create_raw() + raw3 = self._create_raw() + raw2.id = 2 + raw3.id = 3 + models.RawData.objects.filter(tenant='tenant').AndReturn([raw1, raw2, + raw3]) + raw1.search_results({}, mox.IgnoreArg(), ' ').AndReturn(search_result) + raw2.search_results(search_result, mox.IgnoreArg(),' ').AndReturn(search_result_2) + self.mox.ReplayAll() + + resp = stacky_server.search(fake_request, 'nova') + + self.assertEqual(resp.status_code, 200) + json_resp = json.loads(resp.content) + self.assertEqual(len(json_resp), 3) + self._assert_on_search_nova(json_resp, raw1) + self.mox.VerifyAll() \ No newline at end of file From 262a8b17b44572120132b3405149b68128062a09 Mon Sep 17 00:00:00 2001 From: Manali Latkar Date: Tue, 23 Jul 2013 10:26:13 +0530 Subject: [PATCH 11/11] added / to urls, returning a list instead of dict and changing raw.instance to raw.uuid in watch --- stacktach/models.py | 4 ++++ stacktach/stacky_server.py | 8 ++++---- stacktach/urls.py | 4 ++-- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/stacktach/models.py b/stacktach/models.py index 410b2a6..7eaca5f 100644 --- a/stacktach/models.py +++ b/stacktach/models.py @@ -59,6 +59,10 @@ class GenericRawData(models.Model): def get_name(): return GenericRawData.__name__ + @property + def uuid(self): + return self.instance + def search_results(self, results, when, routing_key_status): if not results: results = copy.deepcopy(self.result_titles) diff --git a/stacktach/stacky_server.py b/stacktach/stacky_server.py index 70c732e..c5b0552 100644 --- a/stacktach/stacky_server.py +++ b/stacktach/stacky_server.py @@ -111,7 +111,7 @@ def do_uuid(request, service='nova'): msg = "%s is not uuid-like" % uuid return error_response(400, 'Bad Request', msg) model = _model_factory(service) - result = {} + result = [] param = {} if service == 'nova' or service == 'generic': param = {'instance': uuid} @@ -333,7 +333,7 @@ def do_watch(request, deployment_id, service='nova'): results = [] for raw in events: - uuid = raw.instance + uuid = raw.uuid if not uuid: uuid = "-" typ = routing_key_type(raw.routing_key) @@ -493,7 +493,7 @@ def search(request, service): limit = int(limit) model = _model_factory(service) filter_para = {field: value} - results = {} + results = [] try: events = model.filter(**filter_para) event_len = len(events) @@ -505,4 +505,4 @@ def search(request, service): results = event.search_results(results, when, routing_key_status) return rsp(json.dumps(results)) except ObjectDoesNotExist or FieldError: - return rsp({}) + return rsp([]) diff --git a/stacktach/urls.py b/stacktach/urls.py index e82788f..7057836 100644 --- a/stacktach/urls.py +++ b/stacktach/urls.py @@ -17,9 +17,9 @@ urlpatterns = patterns('', 'stacktach.stacky_server.do_jsonreport'), url(r'stacky/show/(?P\d+)/$', 'stacktach.stacky_server.do_show'), - url(r'stacky/show/(?P\w+)/(?P\d+)$', + url(r'stacky/show/(?P\w+)/(?P\d+)/$', 'stacktach.stacky_server.do_show'), - url(r'stacky/watch/(?P\d+)/(?P\w+)$', + url(r'stacky/watch/(?P\d+)/(?P\w+)/$', 'stacktach.stacky_server.do_watch'), url(r'stacky/search/(?P\w+)/$', 'stacktach.stacky_server.search'), url(r'stacky/kpi/$', 'stacktach.stacky_server.do_kpi'),