x/stacktach
Code Issues Proposed changes

Merge pull request #152 from manalilatkar/RM221-UV

Browse Source 
Rm221 uv
This commit is contained in:
Andrew Melton 2013-07-23 09:45:44 -07:00
commit 0919631c6c
23 changed files with 2652 additions and 652 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
etc/sample_stacktach_worker_config.json
View File

@ -7,7 +7,11 @@
"rabbit_userid": "rabbit",
"rabbit_password": "rabbit",
"rabbit_virtual_host": "/",
"exit_on_exception": true
"exit_on_exception": true,
"topics": {
"nova": ["monitor.info", "monitor.error"],
"glance": ["monitor_glance.info", "monitor_glance.error"]
}
},
{
"name": "east_coast.prod.cell1",
@ -17,6 +21,10 @@
"rabbit_userid": "rabbit",
"rabbit_password": "rabbit",
"rabbit_virtual_host": "/",
"exit_on_exception": false
"exit_on_exception": false,
"topics": {
"nova": ["monitor.info", "monitor.error"],
"glance": ["monitor_glance.info", "monitor_glance.error"]
}
}]
}

2
settings.py
View File

@ -140,6 +140,8 @@ INSTALLED_APPS = (
'south'
)
SOUTH_TESTS_MIGRATE = False
ALLOWED_HOSTS = ['*']
# A sample logging configuration. The only tangible logging

48
stacktach/db.py
View File

@ -20,7 +20,7 @@ def get_or_create_deployment(name):
return models.Deployment.objects.get_or_create(name=name)
def create_rawdata(**kwargs):
def create_nova_rawdata(**kwargs):
imagemeta_fields = ['os_architecture', 'os_version',
'os_distro', 'rax_options']
imagemeta_kwargs = \
@ -35,6 +35,7 @@ def create_rawdata(**kwargs):
return rawdata
def create_lifecycle(**kwargs):
return models.Lifecycle(**kwargs)
@ -88,4 +89,47 @@ def create_instance_exists(**kwargs):
def save(obj):
obj.save()
obj.save()
def create_glance_rawdata(**kwargs):
rawdata = models.GlanceRawData(**kwargs)
rawdata.save()
return rawdata
def create_generic_rawdata(**kwargs):
rawdata = models.GenericRawData(**kwargs)
rawdata.save()
return rawdata
def create_image_usage(**kwargs):
usage = models.ImageUsage(**kwargs)
usage.save()
return usage
def create_image_delete(**kwargs):
delete = models.ImageDeletes(**kwargs)
delete.save()
return delete
def create_image_exists(**kwargs):
exists = models.ImageExists(**kwargs)
exists.save()
return exists
def get_image_delete(**kwargs):
return _safe_get(models.ImageDeletes, **kwargs)
def get_image_usage(**kwargs):
return _safe_get(models.ImageUsage, **kwargs)

9
stacktach/migrations/0003_populate_usage_related_fields_in_rawdataimagemeta_instanceexists_and_instanceusages_from_rawdata.py
View File

@ -1,8 +1,7 @@
# -*- coding: utf-8 -*-
import copy
from south.v2 import DataMigration
from stacktach.notification import Notification
from stacktach.views import NOTIFICATIONS
from stacktach.notification import notification_factory
try:
import ujson as json
@ -43,8 +42,8 @@ class Migration(DataMigration):
json_dict = json.loads(json_message)
routing_key = json_dict[0]
body = json_dict[1]
notification = NOTIFICATIONS[routing_key](body)
return notification
return notification_factory(body, None, routing_key, json_message,
'nova')
def forwards(self, orm):
# Note: Don't use "from appname.models import ModelName".
@ -79,7 +78,7 @@ class Migration(DataMigration):
exists_update_count += 1
print "Updated %s records in InstanceExists" % exists_update_count
print "\nStarted updating records in InstacnceUsages"
print "\nStarted updating records in InstanceUsages"
usages = orm.InstanceUsage.objects.all().values('request_id')
usages_update_count = 0
for usage in usages:

308
stacktach/migrations/0006_create_glance_usage_verification_tables.py Normal file
View File

@ -0,0 +1,308 @@
# -*- coding: utf-8 -*-
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models
class Migration(SchemaMigration):
def forwards(self, orm):
# Adding model 'ImageDeletes'
db.create_table(u'stacktach_imagedeletes', (
(u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
('uuid', self.gf('django.db.models.fields.CharField')(max_length=50, db_index=True)),
('deleted_at', self.gf('django.db.models.fields.DecimalField')(null=True, max_digits=20, decimal_places=6, db_index=True)),
('raw', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['stacktach.GlanceRawData'], null=True)),
))
db.send_create_signal(u'stacktach', ['ImageDeletes'])
# Adding model 'GlanceRawData'
db.create_table(u'stacktach_glancerawdata', (
(u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
('deployment', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['stacktach.Deployment'])),
('owner', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=255, null=True, blank=True)),
('json', self.gf('django.db.models.fields.TextField')()),
('routing_key', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=50, null=True, blank=True)),
('when', self.gf('django.db.models.fields.DecimalField')(max_digits=20, decimal_places=6, db_index=True)),
('publisher', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=100, null=True, blank=True)),
('event', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=50, null=True, blank=True)),
('service', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=50, null=True, blank=True)),
('host', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=100, null=True, blank=True)),
('instance', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=50, null=True, blank=True)),
('request_id', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=50, null=True, blank=True)),
('uuid', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=36, null=True, blank=True)),
('status', self.gf('django.db.models.fields.CharField')(max_length=30, null=True, db_index=True)),
('image_type', self.gf('django.db.models.fields.IntegerField')(default=0, null=True, db_index=True)),
))
db.send_create_signal(u'stacktach', ['GlanceRawData'])
# Adding model 'ImageUsage'
db.create_table(u'stacktach_imageusage', (
(u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
('uuid', self.gf('django.db.models.fields.CharField')(max_length=50, db_index=True)),
('created_at', self.gf('django.db.models.fields.DecimalField')(max_digits=20, decimal_places=6, db_index=True)),
('owner', self.gf('django.db.models.fields.CharField')(max_length=50, db_index=True)),
('size', self.gf('django.db.models.fields.BigIntegerField')(max_length=20)),
('last_raw', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['stacktach.GlanceRawData'], null=True)),
))
db.send_create_signal(u'stacktach', ['ImageUsage'])
# Adding model 'GenericRawData'
db.create_table(u'stacktach_genericrawdata', (
(u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
('deployment', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['stacktach.Deployment'])),
('tenant', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=50, null=True, blank=True)),
('json', self.gf('django.db.models.fields.TextField')()),
('routing_key', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=50, null=True, blank=True)),
('when', self.gf('django.db.models.fields.DecimalField')(max_digits=20, decimal_places=6, db_index=True)),
('publisher', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=100, null=True, blank=True)),
('event', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=50, null=True, blank=True)),
('service', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=50, null=True, blank=True)),
('host', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=100, null=True, blank=True)),
('instance', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=50, null=True, blank=True)),
('request_id', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=50, null=True, blank=True)),
('message_id', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=50, null=True, blank=True)),
))
db.send_create_signal(u'stacktach', ['GenericRawData'])
# Adding model 'ImageExists'
db.create_table(u'stacktach_imageexists', (
(u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
('uuid', self.gf('django.db.models.fields.CharField')(max_length=50, db_index=True)),
('created_at', self.gf('django.db.models.fields.DecimalField')(null=True, max_digits=20, decimal_places=6, db_index=True)),
('deleted_at', self.gf('django.db.models.fields.DecimalField')(null=True, max_digits=20, decimal_places=6, db_index=True)),
('audit_period_beginning', self.gf('django.db.models.fields.DecimalField')(max_digits=20, decimal_places=6, db_index=True)),
('audit_period_ending', self.gf('django.db.models.fields.DecimalField')(max_digits=20, decimal_places=6, db_index=True)),
('status', self.gf('django.db.models.fields.CharField')(default='pending', max_length=50, db_index=True)),
('fail_reason', self.gf('django.db.models.fields.CharField')(max_length=300, null=True)),
('raw', self.gf('django.db.models.fields.related.ForeignKey')(related_name='+', to=orm['stacktach.GlanceRawData'])),
('usage', self.gf('django.db.models.fields.related.ForeignKey')(related_name='+', null=True, to=orm['stacktach.ImageUsage'])),
('delete', self.gf('django.db.models.fields.related.ForeignKey')(related_name='+', null=True, to=orm['stacktach.ImageDeletes'])),
('send_status', self.gf('django.db.models.fields.IntegerField')(default=0, db_index=True)),
('owner', self.gf('django.db.models.fields.CharField')(max_length=255, db_index=True)),
('size', self.gf('django.db.models.fields.BigIntegerField')(max_length=20)),
))
db.send_create_signal(u'stacktach', ['ImageExists'])
def backwards(self, orm):
# Deleting model 'ImageDeletes'
db.delete_table(u'stacktach_imagedeletes')
# Deleting model 'GlanceRawData'
db.delete_table(u'stacktach_glancerawdata')
# Deleting model 'ImageUsage'
db.delete_table(u'stacktach_imageusage')
# Deleting model 'GenericRawData'
db.delete_table(u'stacktach_genericrawdata')
# Deleting model 'ImageExists'
db.delete_table(u'stacktach_imageexists')
models = {
u'stacktach.deployment': {
'Meta': {'object_name': 'Deployment'},
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
},
u'stacktach.genericrawdata': {
'Meta': {'object_name': 'GenericRawData'},
'deployment': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['stacktach.Deployment']"}),
'event': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
'host': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '100', 'null': 'True', 'blank': 'True'}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'instance': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
'json': ('django.db.models.fields.TextField', [], {}),
'message_id': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
'publisher': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '100', 'null': 'True', 'blank': 'True'}),
'request_id': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
'routing_key': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
'service': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
'tenant': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
'when': ('django.db.models.fields.DecimalField', [], {'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'})
},
u'stacktach.glancerawdata': {
'Meta': {'object_name': 'GlanceRawData'},
'deployment': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['stacktach.Deployment']"}),
'event': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
'host': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '100', 'null': 'True', 'blank': 'True'}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'image_type': ('django.db.models.fields.IntegerField', [], {'default': '0', 'null': 'True', 'db_index': 'True'}),
'instance': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
'json': ('django.db.models.fields.TextField', [], {}),
'owner': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '255', 'null': 'True', 'blank': 'True'}),
'publisher': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '100', 'null': 'True', 'blank': 'True'}),
'request_id': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
'routing_key': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
'service': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
'status': ('django.db.models.fields.CharField', [], {'max_length': '30', 'null': 'True', 'db_index': 'True'}),
'uuid': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '36', 'null': 'True', 'blank': 'True'}),
'when': ('django.db.models.fields.DecimalField', [], {'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'})
},
u'stacktach.imagedeletes': {
'Meta': {'object_name': 'ImageDeletes'},
'deleted_at': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'raw': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['stacktach.GlanceRawData']", 'null': 'True'}),
'uuid': ('django.db.models.fields.CharField', [], {'max_length': '50', 'db_index': 'True'})
},
u'stacktach.imageexists': {
'Meta': {'object_name': 'ImageExists'},
'audit_period_beginning': ('django.db.models.fields.DecimalField', [], {'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}),
'audit_period_ending': ('django.db.models.fields.DecimalField', [], {'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}),
'created_at': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}),
'delete': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'null': 'True', 'to': u"orm['stacktach.ImageDeletes']"}),
'deleted_at': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}),
'fail_reason': ('django.db.models.fields.CharField', [], {'max_length': '300', 'null': 'True'}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'owner': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}),
'raw': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'to': u"orm['stacktach.GlanceRawData']"}),
'send_status': ('django.db.models.fields.IntegerField', [], {'default': '0', 'db_index': 'True'}),
'size': ('django.db.models.fields.BigIntegerField', [], {'max_length': '20'}),
'status': ('django.db.models.fields.CharField', [], {'default': "'pending'", 'max_length': '50', 'db_index': 'True'}),
'usage': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'null': 'True', 'to': u"orm['stacktach.ImageUsage']"}),
'uuid': ('django.db.models.fields.CharField', [], {'max_length': '50', 'db_index': 'True'})
},
u'stacktach.imageusage': {
'Meta': {'object_name': 'ImageUsage'},
'created_at': ('django.db.models.fields.DecimalField', [], {'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'last_raw': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['stacktach.GlanceRawData']", 'null': 'True'}),
'owner': ('django.db.models.fields.CharField', [], {'max_length': '50', 'db_index': 'True'}),
'size': ('django.db.models.fields.BigIntegerField', [], {'max_length': '20'}),
'uuid': ('django.db.models.fields.CharField', [], {'max_length': '50', 'db_index': 'True'})
},
u'stacktach.instancedeletes': {
'Meta': {'object_name': 'InstanceDeletes'},
'deleted_at': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'instance': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
'launched_at': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}),
'raw': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['stacktach.RawData']", 'null': 'True'})
},
u'stacktach.instanceexists': {
'Meta': {'object_name': 'InstanceExists'},
'audit_period_beginning': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}),
'audit_period_ending': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}),
'delete': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'null': 'True', 'to': u"orm['stacktach.InstanceDeletes']"}),
'deleted_at': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}),
'fail_reason': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '300', 'null': 'True', 'blank': 'True'}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'instance': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
'instance_type_id': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
'launched_at': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}),
'message_id': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
'os_architecture': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
'os_distro': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
'os_version': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
'raw': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'null': 'True', 'to': u"orm['stacktach.RawData']"}),
'rax_options': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
'send_status': ('django.db.models.fields.IntegerField', [], {'default': '0', 'null': 'True', 'db_index': 'True'}),
'status': ('django.db.models.fields.CharField', [], {'default': "'pending'", 'max_length': '50', 'db_index': 'True'}),
'tenant': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
'usage': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'null': 'True', 'to': u"orm['stacktach.InstanceUsage']"})
},
u'stacktach.instancereconcile': {
'Meta': {'object_name': 'InstanceReconcile'},
'deleted_at': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'instance': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
'instance_type_id': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
'launched_at': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}),
'os_architecture': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
'os_distro': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
'os_version': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
'rax_options': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
'row_created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
'row_updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
'source': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '150', 'null': 'True', 'blank': 'True'}),
'tenant': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'})
},
u'stacktach.instanceusage': {
'Meta': {'object_name': 'InstanceUsage'},
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'instance': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
'instance_type_id': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
'launched_at': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}),
'os_architecture': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
'os_distro': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
'os_version': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
'rax_options': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
'request_id': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
'tenant': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'})
},
u'stacktach.jsonreport': {
'Meta': {'object_name': 'JsonReport'},
'created': ('django.db.models.fields.DecimalField', [], {'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'json': ('django.db.models.fields.TextField', [], {}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '50', 'db_index': 'True'}),
'period_end': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}),
'period_start': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}),
'version': ('django.db.models.fields.IntegerField', [], {'default': '1'})
},
u'stacktach.lifecycle': {
'Meta': {'object_name': 'Lifecycle'},
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'instance': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
'last_raw': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['stacktach.RawData']", 'null': 'True'}),
'last_state': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
'last_task_state': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'})
},
u'stacktach.rawdata': {
'Meta': {'object_name': 'RawData'},
'deployment': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['stacktach.Deployment']"}),
'event': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
'host': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '100', 'null': 'True', 'blank': 'True'}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'image_type': ('django.db.models.fields.IntegerField', [], {'default': '0', 'null': 'True', 'db_index': 'True'}),
'instance': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
'json': ('django.db.models.fields.TextField', [], {}),
'old_state': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '20', 'null': 'True', 'blank': 'True'}),
'old_task': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '30', 'null': 'True', 'blank': 'True'}),
'publisher': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '100', 'null': 'True', 'blank': 'True'}),
'request_id': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
'routing_key': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
'service': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
'state': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '20', 'null': 'True', 'blank': 'True'}),
'task': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '30', 'null': 'True', 'blank': 'True'}),
'tenant': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
'when': ('django.db.models.fields.DecimalField', [], {'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'})
},
u'stacktach.rawdataimagemeta': {
'Meta': {'object_name': 'RawDataImageMeta'},
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'os_architecture': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
'os_distro': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
'os_version': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
'raw': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['stacktach.RawData']"}),
'rax_options': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'})
},
u'stacktach.requesttracker': {
'Meta': {'object_name': 'RequestTracker'},
'completed': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}),
'duration': ('django.db.models.fields.DecimalField', [], {'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'last_timing': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['stacktach.Timing']", 'null': 'True'}),
'lifecycle': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['stacktach.Lifecycle']"}),
'request_id': ('django.db.models.fields.CharField', [], {'max_length': '50', 'db_index': 'True'}),
'start': ('django.db.models.fields.DecimalField', [], {'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'})
},
u'stacktach.timing': {
'Meta': {'object_name': 'Timing'},
'diff': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}),
'end_raw': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'null': 'True', 'to': u"orm['stacktach.RawData']"}),
'end_when': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6'}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'lifecycle': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['stacktach.Lifecycle']"}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '50', 'db_index': 'True'}),
'start_raw': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'null': 'True', 'to': u"orm['stacktach.RawData']"}),
'start_when': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6'})
}
}
complete_apps = ['stacktach']

177
stacktach/models.py
View File

@ -13,10 +13,15 @@
# License for the specific language governing permissions and limitations
# under the License.
from django import forms
import copy
from django.db import models
def routing_key_type(key):
if key.endswith('error'):
return 'E'
return ' '
class Deployment(models.Model):
name = models.CharField(max_length=50)
@ -24,7 +29,51 @@ class Deployment(models.Model):
return self.name
class GenericRawData(models.Model):
result_titles = [["#", "?", "When", "Deployment", "Event", "Host",
"Instance", "Request id"]]
deployment = models.ForeignKey(Deployment)
tenant = models.CharField(max_length=50, null=True, blank=True,
db_index=True)
json = models.TextField()
routing_key = models.CharField(max_length=50, null=True,
blank=True, db_index=True)
when = models.DecimalField(max_digits=20, decimal_places=6,
db_index=True)
publisher = models.CharField(max_length=100, null=True,
blank=True, db_index=True)
event = models.CharField(max_length=50, null=True,
blank=True, db_index=True)
service = models.CharField(max_length=50, null=True,
blank=True, db_index=True)
host = models.CharField(max_length=100, null=True,
blank=True, db_index=True)
instance = models.CharField(max_length=50, null=True,
blank=True, db_index=True)
request_id = models.CharField(max_length=50, null=True,
blank=True, db_index=True)
message_id = models.CharField(max_length=50, null=True,
blank=True, db_index=True)
@staticmethod
def get_name():
return GenericRawData.__name__
@property
def uuid(self):
return self.instance
def search_results(self, results, when, routing_key_status):
if not results:
results = copy.deepcopy(self.result_titles)
results.append([self.id, routing_key_status, str(when),
self.deployment.name, self.event, self.host,
self.instance, self.request_id])
return results
class RawData(models.Model):
result_titles = [["#", "?", "When", "Deployment", "Event", "Host",
"State", "State'", "Task'"]]
deployment = models.ForeignKey(Deployment)
tenant = models.CharField(max_length=50, null=True, blank=True,
db_index=True)
@ -58,6 +107,22 @@ class RawData(models.Model):
def __repr__(self):
return "%s %s %s" % (self.event, self.instance, self.state)
@property
def uuid(self):
return self.instance
@staticmethod
def get_name():
return RawData.__name__
def search_results(self, results, when, routing_key_status):
if not results:
results = copy.deepcopy(self.result_titles)
results.append([self.id, routing_key_status, str(when),
self.deployment.name, self.event, self.host, self.state,
self.old_state, self.old_task])
return results
class RawDataImageMeta(models.Model):
raw = models.ForeignKey(RawData, null=False)
@ -158,6 +223,7 @@ class InstanceExists(models.Model):
(RECONCILED, 'Passed Verification After Reconciliation'),
(FAILED, 'Failed Verification'),
]
instance = models.CharField(max_length=50, null=True,
blank=True, db_index=True)
launched_at = models.DecimalField(null=True, max_digits=20,
@ -238,5 +304,114 @@ class JsonReport(models.Model):
json = models.TextField()
class GlanceRawData(models.Model):
result_titles = [["#", "?", "When", "Deployment", "Event", "Host",
"Status"]]
ACTIVE = 'active'
DELETED = 'deleted'
KILLED = 'killed'
PENDING_DELETE = 'pending_delete'
QUEUED = 'queued'
SAVING = 'saving'
STATUS_CHOICES = [
(ACTIVE, 'Active'),
(DELETED, 'Deleted'),
(KILLED, 'Killed'),
(PENDING_DELETE, 'Pending delete'),
(QUEUED, 'Queued'),
(SAVING, 'Saving'),
]
deployment = models.ForeignKey(Deployment)
owner = models.CharField(max_length=255, null=True, blank=True,
db_index=True)
json = models.TextField()
routing_key = models.CharField(max_length=50, null=True, blank=True,
db_index=True)
when = models.DecimalField(max_digits=20, decimal_places=6, db_index=True)
publisher = models.CharField(max_length=100, null=True,
blank=True, db_index=True)
event = models.CharField(max_length=50, null=True, blank=True,
db_index=True)
service = models.CharField(max_length=50, null=True, blank=True,
db_index=True)
host = models.CharField(max_length=100, null=True, blank=True,
db_index=True)
instance = models.CharField(max_length=50, null=True, blank=True,
db_index=True)
request_id = models.CharField(max_length=50, null=True, blank=True,
db_index=True)
uuid = models.CharField(max_length=36, null=True, blank=True,
db_index=True)
status = models.CharField(max_length=30, db_index=True,
choices=STATUS_CHOICES, null=True)
image_type = models.IntegerField(null=True, default=0, db_index=True)
@staticmethod
def get_name():
return GlanceRawData.__name__
def search_results(self, results, when, routing_key_status):
if not results:
results = copy.deepcopy(self.result_titles)
results.append([self.id, routing_key_status, str(when),
self.deployment.name, self.event, self.host,
self.status])
return results
class ImageUsage(models.Model):
uuid = models.CharField(max_length=50, db_index=True)
created_at = models.DecimalField(max_digits=20,
decimal_places=6, db_index=True)
owner = models.CharField(max_length=50, db_index=True)
size = models.BigIntegerField(max_length=20)
last_raw = models.ForeignKey(GlanceRawData, null=True)
class ImageDeletes(models.Model):
uuid = models.CharField(max_length=50, db_index=True)
deleted_at = models.DecimalField(max_digits=20,
decimal_places=6, db_index=True,
null=True)
raw = models.ForeignKey(GlanceRawData, null=True)
class ImageExists(models.Model):
PENDING = 'pending'
VERIFYING = 'verifying'
VERIFIED = 'verified'
FAILED = 'failed'
STATUS_CHOICES = [
(PENDING, 'Pending Verification'),
(VERIFYING, 'Currently Being Verified'),
(VERIFIED, 'Passed Verification'),
(FAILED, 'Failed Verification'),
]
uuid = models.CharField(max_length=50, db_index=True)
created_at = models.DecimalField(max_digits=20,
decimal_places=6, db_index=True,
null=True)
deleted_at = models.DecimalField(max_digits=20,
decimal_places=6, db_index=True,
null=True)
audit_period_beginning = models.DecimalField(max_digits=20,
decimal_places=6,
db_index=True)
audit_period_ending = models.DecimalField(max_digits=20,
decimal_places=6, db_index=True)
status = models.CharField(max_length=50, db_index=True,
choices=STATUS_CHOICES,
default=PENDING)
fail_reason = models.CharField(max_length=300, null=True)
raw = models.ForeignKey(GlanceRawData, related_name='+')
usage = models.ForeignKey(ImageUsage, related_name='+', null=True)
delete = models.ForeignKey(ImageDeletes, related_name='+', null=True)
send_status = models.IntegerField(default=0, db_index=True)
owner = models.CharField(max_length=255, db_index=True)
size = models.BigIntegerField(max_length=20)
def get_model_fields(model):
return model._meta.fields

250
stacktach/notification.py
View File

@ -1,25 +1,38 @@
# Copyright (c) 2013 - Rackspace Inc.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to
# deal in the Software without restriction, including without limitation the
# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
# sell copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.
from stacktach import utils
from stacktach import stacklog
from stacktach import image_type
from stacktach import db
class Notification(object):
def __init__(self, body):
def __init__(self, body, deployment, routing_key, json):
self.body = body
self.request_id = body.get('_context_request_id', "")
self.deployment = deployment
self.routing_key = routing_key
self.json = json
self.payload = body.get('payload', {})
self.state = self.payload.get('state', "")
self.old_state = self.payload.get('old_state', "")
self.old_task = self.payload.get('old_task_state', "")
self.task = self.payload.get('new_task_state', "")
self.image_type = image_type.get_numeric_code(self.payload)
self.publisher = self.body['publisher_id']
self.event = self.body['event_type']
image_meta = self.payload.get('image_meta', {})
self.os_architecture = image_meta.get('org.openstack__1__architecture',
'')
self.os_distro = image_meta.get('org.openstack__1__os_distro', '')
self.os_version = image_meta.get('org.openstack__1__os_version', '')
self.rax_options = image_meta.get('com.rackspace__1__options', '')
@property
def when(self):
@ -29,30 +42,24 @@ class Notification(object):
when = utils.str_time_to_unix(when)
return when
def rawdata_kwargs(self, deployment, routing_key, json):
return {
'deployment': deployment,
'routing_key': routing_key,
'event': self.event,
'publisher': self.publisher,
'json': json,
'state': self.state,
'old_state': self.old_state,
'task': self.task,
'old_task': self.old_task,
'image_type': self.image_type,
'when': self.when,
'publisher': self.publisher,
'service': self.service,
'host': self.host,
'instance': self.instance,
'request_id': self.request_id,
'tenant': self.tenant,
'os_architecture': self.os_architecture,
'os_distro': self.os_distro,
'os_version': self.os_version,
'rax_options': self.rax_options
}
@property
def service(self):
parts = self.publisher.split('.')
return parts[0]
@property
def host(self):
host = None
parts = self.publisher.split('.')
if len(parts) > 1:
host = ".".join(parts[1:])
return host
@property
def tenant(self):
tenant = self.body.get('_context_project_id', None)
tenant = self.payload.get('tenant_id', tenant)
return tenant
@property
def instance(self):
@ -65,6 +72,144 @@ class Notification(object):
instance = self.payload.get('instance', {}).get('uuid')
return instance
@property
def message_id(self):
return self.body.get('message_id', None)
def save(self):
return db.create_generic_rawdata(deployment=self.deployment,
routing_key=self.routing_key,
tenant=self.tenant,
json=self.json,
when=self.when,
publisher=self.publisher,
event=self.event,
service=self.service,
host=self.host,
instance=self.instance,
request_id=self.request_id,
message_id=self.message_id)
class GlanceNotification(Notification):
def __init__(self, body, deployment, routing_key, json):
super(GlanceNotification, self).__init__(body, deployment,
routing_key, json)
self.properties = self.payload.get('properties', {})
self.image_type = image_type.get_numeric_code(self.payload)
self.status = self.payload.get('status', None)
self.uuid = self.payload.get('id', None)
self.size = self.payload.get('size', None)
created_at = self.payload.get('created_at', None)
self.created_at = created_at and utils.str_time_to_unix(created_at)
audit_period_beginning = self.payload.get(
'audit_period_beginning', None)
self.audit_period_beginning = audit_period_beginning and\
utils.str_time_to_unix(audit_period_beginning)
audit_period_ending = self.payload.get(
'audit_period_ending', None)
self.audit_period_ending = audit_period_ending and \
utils.str_time_to_unix(audit_period_ending)
@property
def owner(self):
return self.payload.get('owner', None)
@property
def instance(self):
return self.properties.get('instance_uuid', None)
@property
def deleted_at(self):
deleted_at = self.body.get('deleted_at', None)
deleted_at = deleted_at or self.payload.get('deleted_at', None)
return deleted_at and utils.str_time_to_unix(deleted_at)
def save(self):
return db.create_glance_rawdata(deployment=self.deployment,
routing_key=self.routing_key,
owner=self.owner,
json=self.json,
when=self.when,
publisher=self.publisher,
event=self.event,
service=self.service,
host=self.host,
instance=self.instance,
request_id=self.request_id,
image_type=self.image_type,
status=self.status,
uuid=self.uuid)
def save_exists(self, raw):
if self.created_at:
values = {
'uuid': self.uuid,
'audit_period_beginning': self.audit_period_beginning,
'audit_period_ending': self.audit_period_ending,
'owner': self.owner,
'size': self.size,
'raw': raw
}
created_at_range = (self.created_at, self.created_at+1)
usage = db.get_image_usage(
uuid=self.uuid, created_at__range=created_at_range)
values['usage'] = usage
values['created_at'] = self.created_at
if self.deleted_at:
delete = db.get_image_delete(
uuid=self.uuid, created_at__range=created_at_range)
values['delete'] = delete
values['deleted_at'] = self.deleted_at
db.create_image_exists(**values)
else:
stacklog.warn("Ignoring exists without created_at. GlanceRawData(%s)"
% raw.id)
def save_usage(self, raw):
values = {
'uuid': self.uuid,
'created_at': self.created_at,
'owner': self.owner,
'size': self.size,
'last_raw': raw
}
db.create_image_usage(**values)
def save_delete(self, raw):
values = {
'uuid': self.uuid,
'raw': raw,
'deleted_at': self.deleted_at
}
db.create_image_delete(**values)
class NovaNotification(Notification):
def __init__(self, body, deployment, routing_key, json):
super(NovaNotification, self).__init__(body, deployment, routing_key,
json)
self.state = self.payload.get('state', '')
self.old_state = self.payload.get('old_state', '')
self.old_task = self.payload.get('old_task_state', '')
self.task = self.payload.get('new_task_state', '')
self.image_type = image_type.get_numeric_code(self.payload)
image_meta = self.payload.get('image_meta', {})
self.os_architecture = \
image_meta.get('org.openstack__1__architecture', '')
self.os_distro = image_meta.get('org.openstack__1__os_distro', '')
self.os_version = image_meta.get('org.openstack__1__os_version', '')
self.rax_options = image_meta.get('com.rackspace__1__options', '')
self.instance_type_id = self.payload.get('instance_type_id', None)
self.new_instance_type_id = \
self.payload.get('new_instance_type_id', None)
self.launched_at = self.payload.get('launched_at', None)
self.deleted_at = self.payload.get('deleted_at', None)
self.audit_period_beginning = self.payload.get(
'audit_period_beginning', None)
self.audit_period_ending = self.payload.get(
'audit_period_ending', None)
@property
def host(self):
host = None
@ -78,8 +223,31 @@ class Notification(object):
parts = self.publisher.split('.')
return parts[0]
@property
def tenant(self):
tenant = self.body.get('_context_project_id', None)
tenant = self.payload.get('tenant_id', tenant)
return tenant
def save(self):
return db.create_nova_rawdata(deployment=self.deployment,
routing_key=self.routing_key,
tenant=self.tenant,
json=self.json,
when=self.when,
publisher=self.publisher,
event=self.event,
service=self.service,
host=self.host,
instance=self.instance,
request_id=self.request_id,
state=self.state,
old_state=self.old_state,
task=self.task,
old_task=self.old_task,
os_architecture=self.os_architecture,
os_distro=self.os_distro,
os_version=self.os_version,
rax_options=self.rax_options)
def notification_factory(body, deployment, routing_key, json, exchange):
if exchange == 'nova':
return NovaNotification(body, deployment, routing_key, json)
if exchange == "glance":
return GlanceNotification(body, deployment, routing_key, json)
return Notification(body, deployment, routing_key, json)

63
stacktach/stacklog.py
View File

@ -36,31 +36,33 @@ def set_default_logger_name(name):
default_logger_name = name
def _make_logger(name):
log = logging.getLogger(__name__)
log.setLevel(logging.DEBUG)
handler = logging.handlers.TimedRotatingFileHandler(default_logger_location % name,
when='midnight', interval=1, backupCount=3)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler.setFormatter(formatter)
log.addHandler(handler)
log.handlers[0].doRollover()
def _logger_factory(exchange, name):
if exchange:
return ExchangeLogger(exchange, name)
else:
logger = logging.getLogger(__name__)
_configure(logger, name)
return logger
def _make_logger(name, exchange=None):
log = _logger_factory(exchange, name)
return log
def init_logger(name=None):
def init_logger(name=None, exchange=None):
global LOGGERS
if name is None:
name = default_logger_name
if name not in LOGGERS:
LOGGERS[name] = _make_logger(name)
LOGGERS[name] = _make_logger(name, exchange)
def get_logger(name=None):
def get_logger(name=None, exchange=None):
global LOGGERS
if name is None:
name = default_logger_name
init_logger(name=name)
init_logger(name=name, exchange=exchange)
return LOGGERS[name]
@ -80,3 +82,38 @@ def info(msg, name=None):
if name is None:
name = default_logger_name
get_logger(name=name).info(msg)
def _configure(logger, name):
logger.setLevel(logging.DEBUG)
handler = logging.handlers.TimedRotatingFileHandler(
default_logger_location % name,
when='midnight', interval=1, backupCount=3)
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler.setFormatter(formatter)
logger.addHandler(handler)
logger.handlers[0].doRollover()
class ExchangeLogger():
def __init__(self, exchange, name='stacktach-default'):
self.logger = logging.getLogger(name)
_configure(self.logger, name)
self.exchange = exchange
def info(self, msg, *args, **kwargs):
msg = self.exchange + ': ' + msg
self.logger.info(msg, *args, **kwargs)
def warn(self, msg, *args, **kwargs):
msg = self.exchange + ': ' + msg
self.logger.warn(msg, *args, **kwargs)
def error(self, msg, *args, **kwargs):
msg = self.exchange + ': ' + msg
self.logger.error(msg, *args, **kwargs)
def exception(self, msg, *args, **kwargs):
msg = self.exchange + ': ' + msg
self.logger.error(msg, *args, **kwargs)

144
stacktach/stacky_server.py
View File

@ -9,13 +9,14 @@ from django.shortcuts import get_object_or_404
import datetime_to_decimal as dt
import models
import utils
from django.core.exceptions import ObjectDoesNotExist, FieldError
SECS_PER_HOUR = 60 * 60
SECS_PER_DAY = SECS_PER_HOUR * 24
def get_event_names():
return models.RawData.objects.values('event').distinct()
def get_event_names(service='nova'):
return _model_factory(service).values('event').distinct()
def get_host_names():
@ -104,22 +105,25 @@ def do_hosts(request):
return rsp(json.dumps(results))
def do_uuid(request):
def do_uuid(request, service='nova'):
uuid = str(request.GET['uuid'])
if not utils.is_uuid_like(uuid):
msg = "%s is not uuid-like" % uuid
return error_response(400, 'Bad Request', msg)
model = _model_factory(service)
result = []
param = {}
if service == 'nova' or service == 'generic':
param = {'instance': uuid}
if service == 'glance':
param = {'uuid': uuid}
related = models.RawData.objects.select_related().filter(instance=uuid)\
.order_by('when')
results = [["#", "?", "When", "Deployment", "Event", "Host", "State",
"State'", "Task'"]]
for e in related:
when = dt.dt_from_decimal(e.when)
results.append([e.id, routing_key_type(e.routing_key), str(when),
e.deployment.name, e.event, e.host, e.state,
e.old_state, e.old_task])
return rsp(json.dumps(results))
related = model.select_related().filter(**param).order_by('when')
for event in related:
when = dt.dt_from_decimal(event.when)
routing_key_status = routing_key_type(event.routing_key)
result = event.search_results(result, when, routing_key_status)
return rsp(json.dumps(result))
def do_timings_uuid(request):
@ -202,15 +206,7 @@ def do_request(request):
return rsp(json.dumps(results))
def do_show(request, event_id):
event_id = int(event_id)
results = []
event = None
try:
event = models.RawData.objects.get(id=event_id)
except models.RawData.ObjectDoesNotExist:
return results
def append_nova_raw_attributes(event, results):
results.append(["Key", "Value"])
results.append(["#", event.id])
when = dt.dt_from_decimal(event.when)
@ -224,16 +220,77 @@ def do_show(request, event_id):
results.append(["Host", event.host])
results.append(["UUID", event.instance])
results.append(["Req ID", event.request_id])
final = [results, ]
j = json.loads(event.json)
final.append(json.dumps(j, indent=2))
final.append(event.instance)
return rsp(json.dumps(final))
return results
def do_watch(request, deployment_id):
def append_glance_raw_attributes(event, results):
results.append(["Key", "Value"])
results.append(["#", event.id])
when = dt.dt_from_decimal(event.when)
results.append(["When", str(when)])
results.append(["Deployment", event.deployment.name])
results.append(["Category", event.routing_key])
results.append(["Publisher", event.publisher])
results.append(["Status", event.status])
results.append(["Event", event.event])
results.append(["Service", event.service])
results.append(["Host", event.host])
results.append(["UUID", event.uuid])
results.append(["Req ID", event.request_id])
return results
def append_generic_raw_attributes(event, results):
results.append(["Key", "Value"])
results.append(["#", event.id])
when = dt.dt_from_decimal(event.when)
results.append(["When", str(when)])
results.append(["Deployment", event.deployment.name])
results.append(["Category", event.routing_key])
results.append(["Publisher", event.publisher])
results.append(["State", event.state])
results.append(["Event", event.event])
results.append(["Service", event.service])
results.append(["Host", event.host])
results.append(["UUID", event.instance])
results.append(["Req ID", event.request_id])
return results
def _append_raw_attributes(event, results, service):
if service == 'nova':
return append_nova_raw_attributes(event, results)
if service == 'glance':
return append_glance_raw_attributes(event, results)
if service == 'generic':
return append_generic_raw_attributes(event, results)
def do_show(request, event_id, service='nova'):
event_id = int(event_id)
results = []
model = _model_factory(service)
try:
event = model.get(id=event_id)
results = _append_raw_attributes(event, results, service)
final = [results, ]
j = json.loads(event.json)
final.append(json.dumps(j, indent=2))
final.append(event.uuid)
return rsp(json.dumps(final))
except ObjectDoesNotExist:
return rsp({})
def _model_factory(service):
if service == 'glance':
return models.GlanceRawData.objects
elif service == 'nova':
return models.RawData.objects
elif service == 'generic':
return models.GenericRawData.objects
def do_watch(request, deployment_id, service='nova'):
model = _model_factory(service)
deployment_id = int(deployment_id)
since = request.GET.get('since')
event_name = request.GET.get('event_name')
@ -244,7 +301,7 @@ def do_watch(request, deployment_id):
events = get_event_names()
max_event_width = max([len(event['event']) for event in events])
base_events = models.RawData.objects.order_by('when')
base_events = model.order_by('when')
if deployment_id > 0:
base_events = base_events.filter(deployment=deployment_id)
@ -276,7 +333,7 @@ def do_watch(request, deployment_id):
results = []
for raw in events:
uuid = raw.instance
uuid = raw.uuid
if not uuid:
uuid = "-"
typ = routing_key_type(raw.routing_key)
@ -426,3 +483,26 @@ def do_jsonreport(request, report_id):
report_id = int(report_id)
report = get_object_or_404(models.JsonReport, pk=report_id)
return rsp(report.json)
def search(request, service):
DEFAULT = 1000
field = request.GET.get('field')
value = request.GET.get('value')
limit = request.GET.get('limit', DEFAULT)
limit = int(limit)
model = _model_factory(service)
filter_para = {field: value}
results = []
try:
events = model.filter(**filter_para)
event_len = len(events)
if event_len > limit:
events = events[0:limit]
for event in events:
when = dt.dt_from_decimal(event.when)
routing_key_status = routing_key_type(event.routing_key)
results = event.search_results(results, when, routing_key_status)
return rsp(json.dumps(results))
except ObjectDoesNotExist or FieldError:
return rsp([])

211
stacktach/tests.py
View File

@ -19,39 +19,224 @@
# IN THE SOFTWARE.
from datetime import datetime
import unittest
from django.test import TransactionTestCase
import db
from stacktach.datetime_to_decimal import dt_to_decimal
from stacktach.models import RawDataImageMeta
from stacktach.models import RawDataImageMeta, ImageUsage, ImageDeletes
from stacktach.models import GenericRawData
from stacktach.models import GlanceRawData
from stacktach.models import RawData
from stacktach.models import get_model_fields
from stacktach import datetime_to_decimal as dt
class RawDataImageMetaDbTestCase(unittest.TestCase):
class RawDataImageMetaDbTestCase(TransactionTestCase):
def test_create_raw_data_should_populate_rawdata_and_rawdata_imagemeta(self):
deployment = db.get_or_create_deployment('deployment1')[0]
kwargs = {
'deployment': deployment,
'when': dt_to_decimal(datetime.utcnow()),
'tenant': '1', 'json': '{}', 'routing_key': 'monitor.info',
'state': 'verifying', 'old_state': 'pending',
'old_task': '', 'task': '', 'image_type': 1,
'publisher': '', 'event': 'compute.instance.exists',
'service': '', 'host': '', 'instance': '1234-5678-9012-3456',
'request_id': '1234', 'os_architecture': 'x86', 'os_version': '1',
'os_distro': 'windows', 'rax_options': '2'}
'tenant': '1',
'json': '{}',
'routing_key': 'monitor.info',
'state': 'verifying',
'old_state': 'pending',
'old_task': 'building',
'task': 'saving',
'image_type': 1,
'publisher': 'publisher',
'event': 'compute.instance.exists',
'service': 'compute',
'host': 'host',
'instance': '1234-5678-9012-3456',
'request_id': '1234',
'os_architecture': 'x86',
'os_version': '1',
'os_distro': 'windows',
'rax_options': '2'}
rawdata = db.create_rawdata(**kwargs)
rawdata = db.create_nova_rawdata(**kwargs)
for field in get_model_fields(RawData):
if field.name != 'id':
self.assertEquals(getattr(rawdata, field.name),
kwargs[field.name])
raw_image_meta = RawDataImageMeta.objects.all()[0]
self.assertEquals(raw_image_meta.raw, rawdata)
raw_image_meta = RawDataImageMeta.objects.filter(raw_id=rawdata.id)[0]
self.assertEquals(raw_image_meta.os_architecture,
kwargs['os_architecture'])
self.assertEquals(raw_image_meta.os_version, kwargs['os_version'])
self.assertEquals(raw_image_meta.os_distro, kwargs['os_distro'])
self.assertEquals(raw_image_meta.rax_options, kwargs['rax_options'])
class GlanceTestCase(TransactionTestCase):
def _create_glance_rawdata(self):
deployment = db.get_or_create_deployment('deployment1')[0]
kwargs = {
'deployment': deployment,
'when': dt_to_decimal(datetime.utcnow()),
'owner': '1234567',
'json': '{}',
'routing_key': 'glance_monitor.info',
'image_type': 1,
'publisher': 'publisher',
'event': 'event',
'service': 'service',
'host': 'host',
'instance': '1234-5678-9012-3456',
'request_id': '1234',
'uuid': '1234-5678-0912-3456',
'status': 'active',
}
db.create_glance_rawdata(**kwargs)
rawdata = GlanceRawData.objects.all()[0]
return kwargs, rawdata
def test_create_rawdata_should_populate_glance_rawdata(self):
kwargs, rawdata = self._create_glance_rawdata()
for field in get_model_fields(GlanceRawData):
if field.name != 'id':
self.assertEquals(getattr(rawdata, field.name),
kwargs[field.name])
def test_create_glance_usage_should_populate_image_usage(self):
_, rawdata = self._create_glance_rawdata()
kwargs = {
'uuid': '1',
'created_at': dt_to_decimal(datetime.utcnow()),
'owner': '1234567',
'size': 12345,
'last_raw': rawdata
}
db.create_image_usage(**kwargs)
usage = ImageUsage.objects.all()[0]
for field in get_model_fields(ImageUsage):
if field.name != 'id':
self.assertEquals(getattr(usage, field.name),
kwargs[field.name])
def test_create_image_delete_should_populate_image_delete(self):
_, rawdata = self._create_glance_rawdata()
kwargs = {
'uuid': '1',
'raw': rawdata,
'deleted_at': dt_to_decimal(datetime.utcnow())
}
db.create_image_delete(**kwargs)
image_delete = ImageDeletes.objects.all()[0]
for field in get_model_fields(ImageDeletes):
if field.name != 'id':
self.assertEquals(getattr(image_delete, field.name),
kwargs[field.name])
class GenericRawDataTestCase(TransactionTestCase):
def test_create_generic_rawdata_should_populate_generic_rawdata(self):
deployment = db.get_or_create_deployment('deployment1')[0]
kwargs = {
'deployment': deployment,
'when': dt_to_decimal(datetime.utcnow()),
'tenant': '1234567',
'json': '{}',
'routing_key': 'monitor.info',
'publisher': 'publisher',
'event': 'event',
'service': 'service',
'host': 'host',
'instance': '1234-5678-9012-3456',
'request_id': '1234',
'message_id': 'message_id'}
db.create_generic_rawdata(**kwargs)
rawdata = GenericRawData.objects.all()[0]
for field in get_model_fields(GenericRawData):
if field.name != 'id':
self.assertEquals(getattr(rawdata, field.name),
kwargs[field.name])
class NovaRawDataSearchTestCase(TransactionTestCase):
def test_search_results_for_nova(self):
expected_result = [['#', '?', 'When', 'Deployment', 'Event', 'Host',
'State', "State'", "Task'"], [1L, ' ',
'2013-07-17 10:16:10.717219', 'depl', 'event',
'host', 'state', 'old_state', 'old_task']]
depl = db.get_or_create_deployment('depl')[0]
when = dt.dt_to_decimal(datetime.utcnow())
raw = db.create_nova_rawdata(deployment=depl,
routing_key='routing_key',
tenant='tenant',
json='json',
when=when,
publisher='publisher',
event='event',
service='nova',
host='host',
instance='instance',
request_id='req-1234',
state='state',
old_state='old_state',
task='task',
old_task='old_task',
os_architecture='arch',
os_distro='distro',
os_version='version',
rax_options=1)
results = raw.search_results({}, "2013-07-17 10:16:10.717219", ' ')
self.assertEqual(results,expected_result)
def test_search_results_for_glance(self):
expected_result = [['#', '?', 'When', 'Deployment', 'Event', 'Host',
'Status'], [1L, ' ',
'2013-07-17 10:16:10.717219', 'depl', 'event',
'host', 'status']]
depl = db.get_or_create_deployment('depl')[0]
when = dt.dt_to_decimal(datetime.utcnow())
glance_raw = db.create_glance_rawdata(deployment=depl,
routing_key='routing_key',
json='json',
when=when,
publisher='publisher',
event='event',
service='glance',
host='host',
uuid='instance',
request_id='req-1234',
status='status',
image_type=1)
results = glance_raw.search_results({}, "2013-07-17 10:16:10.717219",
' ')
self.assertEqual(results,expected_result)
def test_search_results_for_generic(self):
expected_result = [['#', '?', 'When', 'Deployment', 'Event', 'Host',
'Instance', 'Request id'], [1L, ' ',
'2013-07-17 10:16:10.717219', 'depl', 'event',
'host', 'instance', 'req-1234']]
depl = db.get_or_create_deployment('depl')[0]
when = dt.dt_to_decimal(datetime.utcnow())
generic_raw = db.create_generic_rawdata(deployment=depl,
routing_key='routing_key',
json='json',
when=when,
publisher='publisher',
event='event',
service='glance',
host='host',
instance='instance',
request_id='req-1234',
tenant='tenant')
results = generic_raw.search_results({}, "2013-07-17 10:16:10.717219",
' ')
self.assertEqual(results,expected_result)

9
stacktach/urls.py
View File

@ -1,4 +1,4 @@
from django.conf.urls.defaults import patterns, include, url
from django.conf.urls import patterns, url
urlpatterns = patterns('',
@ -7,7 +7,7 @@ urlpatterns = patterns('',
url(r'stacky/deployments/$', 'stacktach.stacky_server.do_deployments'),
url(r'stacky/events/$', 'stacktach.stacky_server.do_events'),
url(r'stacky/hosts/$', 'stacktach.stacky_server.do_hosts'),
url(r'stacky/uuid/$', 'stacktach.stacky_server.do_uuid'),
url(r'stacky/uuid/(?P<service>\w+)/$', 'stacktach.stacky_server.do_uuid'),
url(r'stacky/timings/$', 'stacktach.stacky_server.do_timings'),
url(r'stacky/timings/uuid/$', 'stacktach.stacky_server.do_timings_uuid'),
url(r'stacky/summary/$', 'stacktach.stacky_server.do_summary'),
@ -17,8 +17,11 @@ urlpatterns = patterns('',
'stacktach.stacky_server.do_jsonreport'),
url(r'stacky/show/(?P<event_id>\d+)/$',
'stacktach.stacky_server.do_show'),
url(r'stacky/watch/(?P<deployment_id>\d+)/$',
url(r'stacky/show/(?P<service>\w+)/(?P<event_id>\d+)/$',
'stacktach.stacky_server.do_show'),
url(r'stacky/watch/(?P<deployment_id>\d+)/(?P<service>\w+)/$',
'stacktach.stacky_server.do_watch'),
url(r'stacky/search/(?P<service>\w+)/$', 'stacktach.stacky_server.search'),
url(r'stacky/kpi/$', 'stacktach.stacky_server.do_kpi'),
url(r'stacky/kpi/(?P<tenant_id>\w+)/$', 'stacktach.stacky_server.do_kpi'),
url(r'stacky/usage/launches/$',

156
stacktach/views.py
View File

@ -12,7 +12,7 @@ from stacktach import db as stackdb
from stacktach import models
from stacktach import stacklog
from stacktach import utils
from stacktach.notification import Notification
from stacktach import notification
STACKDB = stackdb
@ -25,13 +25,6 @@ def log_warn(msg):
LOG.warn(msg)
# routing_key : handler
NOTIFICATIONS = {
'monitor.info': Notification,
'monitor.error': Notification}
def start_kpi_tracking(lifecycle, raw):
"""Start the clock for kpi timings when we see an instance.update
coming in from an api node."""
@ -170,17 +163,16 @@ INSTANCE_EVENT = {
}
def _process_usage_for_new_launch(raw, body):
payload = body['payload']
def _process_usage_for_new_launch(raw, notification):
values = {}
values['instance'] = payload['instance_id']
values['request_id'] = body['_context_request_id']
values['instance'] = notification.instance
values['request_id'] = notification.request_id
(usage, new) = STACKDB.get_or_create_instance_usage(**values)
if raw.event in [INSTANCE_EVENT['create_start'],
INSTANCE_EVENT['rebuild_start']]:
usage.instance_type_id = payload['instance_type_id']
usage.instance_type_id = notification.instance_type_id
if raw.event in [INSTANCE_EVENT['rebuild_start'],
INSTANCE_EVENT['resize_prep_start'],
@ -190,27 +182,23 @@ def _process_usage_for_new_launch(raw, body):
# we will have a launch record corresponding to the exists.
# We don't want to override a launched_at if it is already set
# though, because we may have already received the end event
usage.launched_at = utils.str_time_to_unix(payload['launched_at'])
usage.launched_at = utils.str_time_to_unix(notification.launched_at)
usage.tenant = payload['tenant_id']
image_meta = payload.get('image_meta', {})
usage.rax_options = image_meta.get('com.rackspace__1__options', '')
usage.os_architecture = image_meta.get('org.openstack__1__architecture',
'')
usage.os_version = image_meta.get('org.openstack__1__os_version', '')
usage.os_distro = image_meta.get('org.openstack__1__os_distro', '')
usage.tenant = notification.tenant
usage.rax_options = notification.rax_options
usage.os_architecture = notification.os_architecture
usage.os_version = notification.os_version
usage.os_distro = notification.os_distro
STACKDB.save(usage)
def _process_usage_for_updates(raw, body):
payload = body['payload']
def _process_usage_for_updates(raw, notification):
if raw.event == INSTANCE_EVENT['create_end']:
if 'message' in payload and payload['message'] != 'Success':
if notification.message and notification.message != 'Success':
return
instance_id = payload['instance_id']
request_id = body['_context_request_id']
instance_id = notification.instance
request_id = notification.request_id
(usage, new) = STACKDB.get_or_create_instance_usage(instance=instance_id,
request_id=request_id)
@ -218,28 +206,25 @@ def _process_usage_for_updates(raw, body):
INSTANCE_EVENT['rebuild_end'],
INSTANCE_EVENT['resize_finish_end'],
INSTANCE_EVENT['resize_revert_end']]:
usage.launched_at = utils.str_time_to_unix(payload['launched_at'])
usage.launched_at = utils.str_time_to_unix(notification.launched_at)
if raw.event == INSTANCE_EVENT['resize_revert_end']:
usage.instance_type_id = payload['instance_type_id']
usage.instance_type_id = notification.instance_type_id
elif raw.event == INSTANCE_EVENT['resize_prep_end']:
usage.instance_type_id = payload['new_instance_type_id']
usage.instance_type_id = notification.new_instance_type_id
usage.tenant = payload['tenant_id']
image_meta = payload.get('image_meta', {})
usage.rax_options = image_meta.get('com.rackspace__1__options', '')
usage.os_architecture = image_meta.get('org.openstack__1__architecture',
'')
usage.os_version = image_meta.get('org.openstack__1__os_version', '')
usage.os_distro = image_meta.get('org.openstack__1__os_distro', '')
usage.tenant = notification.tenant
usage.rax_options = notification.rax_options
usage.os_architecture = notification.os_architecture
usage.os_version = notification.os_version
usage.os_distro = notification.os_distro
STACKDB.save(usage)
def _process_delete(raw, body):
payload = body['payload']
instance_id = payload['instance_id']
deleted_at = utils.str_time_to_unix(payload['deleted_at'])
def _process_delete(raw, notification):
instance_id = notification.instance
deleted_at = utils.str_time_to_unix(notification.deleted_at)
values = {
'instance': instance_id,
'deleted_at': deleted_at,
@ -247,7 +232,7 @@ def _process_delete(raw, body):
(delete, new) = STACKDB.get_or_create_instance_delete(**values)
delete.raw = raw
launched_at = payload.get('launched_at')
launched_at = notification.launched_at
if launched_at and launched_at != '':
launched_at = utils.str_time_to_unix(launched_at)
delete.launched_at = launched_at
@ -255,37 +240,33 @@ def _process_delete(raw, body):
STACKDB.save(delete)
def _process_exists(raw, body):
payload = body['payload']
instance_id = payload['instance_id']
launched_at_str = payload.get('launched_at')
def _process_exists(raw, notification):
instance_id = notification.instance
launched_at_str = notification.launched_at
if launched_at_str is not None and launched_at_str != '':
launched_at = utils.str_time_to_unix(payload['launched_at'])
launched_at = utils.str_time_to_unix(notification.launched_at)
launched_range = (launched_at, launched_at+1)
usage = STACKDB.get_instance_usage(instance=instance_id,
launched_at__range=launched_range)
values = {}
values['message_id'] = body['message_id']
values['message_id'] = notification.message_id
values['instance'] = instance_id
values['launched_at'] = launched_at
beginning = utils.str_time_to_unix(payload['audit_period_beginning'])
beginning = utils.str_time_to_unix(notification.audit_period_beginning)
values['audit_period_beginning'] = beginning
ending = utils.str_time_to_unix(payload['audit_period_ending'])
ending = utils.str_time_to_unix(notification.audit_period_ending)
values['audit_period_ending'] = ending
values['instance_type_id'] = payload['instance_type_id']
values['instance_type_id'] = notification.instance_type_id
if usage:
values['usage'] = usage
values['raw'] = raw
values['tenant'] = payload['tenant_id']
image_meta = payload.get('image_meta', {})
values['rax_options'] = image_meta.get('com.rackspace__1__options', '')
os_arch = image_meta.get('org.openstack__1__architecture', '')
values['os_architecture'] = os_arch
os_version = image_meta.get('org.openstack__1__os_version', '')
values['os_version'] = os_version
values['os_distro'] = image_meta.get('org.openstack__1__os_distro', '')
values['tenant'] = notification.tenant
values['rax_options'] = notification.rax_options
values['os_architecture'] = notification.os_architecture
values['os_version'] = notification.os_version
values['os_distro'] = notification.os_distro
deleted_at = payload.get('deleted_at')
deleted_at = notification.deleted_at
if deleted_at and deleted_at != '':
# We only want to pre-populate the 'delete' if we know this is in
# fact an exist event for a deleted instance. Otherwise, there
@ -304,6 +285,16 @@ def _process_exists(raw, body):
stacklog.warn("Ignoring exists without launched_at. RawData(%s)" % raw.id)
def _process_glance_usage(raw, notification):
notification.save_usage(raw)
def _process_glance_delete(raw, notification):
notification.save_delete(raw)
def _process_glance_exists(raw, notification):
notification.save_exists(raw)
USAGE_PROCESS_MAPPING = {
INSTANCE_EVENT['create_start']: _process_usage_for_new_launch,
INSTANCE_EVENT['rebuild_start']: _process_usage_for_new_launch,
@ -315,36 +306,51 @@ USAGE_PROCESS_MAPPING = {
INSTANCE_EVENT['resize_finish_end']: _process_usage_for_updates,
INSTANCE_EVENT['resize_revert_end']: _process_usage_for_updates,
INSTANCE_EVENT['delete_end']: _process_delete,
INSTANCE_EVENT['exists']: _process_exists,
}
INSTANCE_EVENT['exists']: _process_exists
}
GLANCE_USAGE_PROCESS_MAPPING = {
'image.activate': _process_glance_usage,
'image.delete': _process_glance_delete,
'image.exists': _process_glance_exists
}
def aggregate_usage(raw, body):
def aggregate_usage(raw, notification):
if not raw.instance:
return
if raw.event in USAGE_PROCESS_MAPPING:
USAGE_PROCESS_MAPPING[raw.event](raw, body)
USAGE_PROCESS_MAPPING[raw.event](raw, notification)
def process_raw_data(deployment, args, json_args):
def aggregate_glance_usage(raw, body):
if raw.event in GLANCE_USAGE_PROCESS_MAPPING.keys():
GLANCE_USAGE_PROCESS_MAPPING[raw.event](raw, body)
def process_raw_data(deployment, args, json_args, exchange):
"""This is called directly by the worker to add the event to the db."""
db.reset_queries()
routing_key, body = args
record = None
notification = NOTIFICATIONS[routing_key](body)
if notification:
values = notification.rawdata_kwargs(deployment, routing_key, json_args)
if not values:
return record
record = STACKDB.create_rawdata(**values)
return record
notif = notification.notification_factory(body, deployment, routing_key,
json_args, exchange)
raw = notif.save()
return raw, notif
def post_process(raw, body):
def post_process_rawdata(raw, notification):
aggregate_lifecycle(raw)
aggregate_usage(raw, body)
aggregate_usage(raw, notification)
def post_process_glancerawdata(raw, notification):
aggregate_glance_usage(raw, notification)
def post_process_genericrawdata(raw, notification):
pass
def _post_process_raw_data(rows, highlight=None):

33
tests/unit/test_models.py Normal file
View File

@ -0,0 +1,33 @@
# Copyright (c) 2013 - Rackspace Inc.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to
# deal in the Software without restriction, including without limitation the
# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
# sell copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.
import unittest
from stacktach.models import RawData, GlanceRawData, GenericRawData
class ModelsTestCase(unittest.TestCase):
def test_get_name_for_rawdata(self):
self.assertEquals(RawData.get_name(), 'RawData')
def test_get_name_for_glancerawdata(self):
self.assertEquals(GlanceRawData.get_name(), 'GlanceRawData')
def test_get_name_for_genericrawdata(self):
self.assertEquals(GenericRawData.get_name(), 'GenericRawData')

585
tests/unit/test_notification.py
View File

@ -18,163 +18,454 @@
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.
from decimal import Decimal
import unittest
import mox
from stacktach import notification
from stacktach import utils
from stacktach.notification import Notification
from tests.unit.utils import REQUEST_ID_1, TENANT_ID_1, INSTANCE_ID_1
from stacktach.notification import NovaNotification
from stacktach.notification import GlanceNotification
from stacktach import db
from tests.unit.utils import REQUEST_ID_1
from tests.unit.utils import DECIMAL_DUMMY_TIME
from tests.unit.utils import DUMMY_TIME
from tests.unit.utils import TIMESTAMP_1
from tests.unit.utils import TENANT_ID_1
from tests.unit.utils import INSTANCE_ID_1
from tests.unit.utils import MESSAGE_ID_1
class NovaNotificationTestCase(unittest.TestCase):
def setUp(self):
self.mox = mox.Mox()
def tearDown(self):
self.mox.UnsetStubs()
def test_factory_should_return_nova_notification_for_nova_exchange(
self):
body = {}
deployment = "1"
json = "{}"
routing_key = "monitor.info"
self.mox.StubOutWithMock(notification, 'NovaNotification')
notification.NovaNotification(body, deployment, routing_key, json)
self.mox.ReplayAll()
notification.notification_factory(body, deployment, routing_key, json,
'nova')
self.mox.VerifyAll()
def test_factory_should_return_glance_notification_for_glance_exchange(
self):
body = {}
deployment = "1"
json = "{}"
routing_key = "monitor_glance.info"
self.mox.StubOutWithMock(notification, 'GlanceNotification')
notification.GlanceNotification(body, deployment, routing_key, json)
self.mox.ReplayAll()
notification.notification_factory(body, deployment, routing_key, json,
'glance')
self.mox.VerifyAll()
def test_factory_should_return_notification_for_unknown_exchange(
self):
body = {}
deployment = "1"
json = "{}"
routing_key = "unknown.info"
self.mox.StubOutWithMock(notification, 'Notification')
notification.Notification(body, deployment, routing_key, json)
self.mox.ReplayAll()
notification.notification_factory(body, deployment, routing_key, json,
'unknown_exchange')
self.mox.VerifyAll()
def test_save_should_persist_nova_rawdata_to_database(self):
body = {
"event_type": "compute.instance.exists",
'_context_request_id': REQUEST_ID_1,
'_context_project_id': TENANT_ID_1,
"timestamp": TIMESTAMP_1,
"publisher_id": "compute.global.preprod-ord.ohthree.com",
"payload": {
'instance_id': INSTANCE_ID_1,
"status": "saving",
"container_format": "ovf",
"properties": {
"image_type": "snapshot",
},
"tenant": "5877054",
"old_state": 'old_state',
"old_task_state": 'old_task',
"image_meta": {
"org.openstack__1__architecture": 'os_arch',
"org.openstack__1__os_distro": 'os_distro',
"org.openstack__1__os_version": 'os_version',
"com.rackspace__1__options": 'rax_opt',
},
"state": 'state',
"new_task_state": 'task'
}
}
deployment = "1"
routing_key = "monitor.info"
json = '{["routing_key", {%s}]}' % body
raw = self.mox.CreateMockAnything()
self.mox.StubOutWithMock(db, 'create_nova_rawdata')
db.create_nova_rawdata(
deployment="1",
tenant=TENANT_ID_1,
json=json,
routing_key=routing_key,
when=utils.str_time_to_unix(TIMESTAMP_1),
publisher="compute.global.preprod-ord.ohthree.com",
event="compute.instance.exists",
service="compute",
host="global.preprod-ord.ohthree.com",
instance=INSTANCE_ID_1,
request_id=REQUEST_ID_1,
old_state='old_state',
old_task='old_task',
os_architecture='os_arch',
os_distro='os_distro',
os_version='os_version',
rax_options='rax_opt',
state='state',
task='task').AndReturn(raw)
self.mox.ReplayAll()
notification = NovaNotification(body, deployment, routing_key, json)
self.assertEquals(notification.save(), raw)
self.mox.VerifyAll()
class GlanceNotificationTestCase(unittest.TestCase):
def setUp(self):
self.mox = mox.Mox()
def tearDown(self):
self.mox.UnsetStubs()
def test_save_should_persist_glance_rawdata_to_database(self):
body = {
"event_type": "image.upload",
"timestamp": "2013-06-20 17:31:57.939614",
"publisher_id": "glance-api01-r2961.global.preprod-ord.ohthree.com",
"payload": {
"status": "saving",
"properties": {
"image_type": "snapshot",
"instance_uuid": INSTANCE_ID_1,
},
"owner": TENANT_ID_1,
"id": "2df2ccf6-bc1b-4853-aab0-25fda346b3bb",
}
}
deployment = "1"
routing_key = "glance_monitor.info"
json = '{["routing_key", {%s}]}' % body
raw = self.mox.CreateMockAnything()
self.mox.StubOutWithMock(db, 'create_glance_rawdata')
db.create_glance_rawdata(
deployment="1",
owner=TENANT_ID_1,
json=json,
routing_key=routing_key,
when=utils.str_time_to_unix("2013-06-20 17:31:57.939614"),
publisher="glance-api01-r2961.global.preprod-ord.ohthree.com",
event="image.upload",
service="glance-api01-r2961",
host="global.preprod-ord.ohthree.com",
instance=INSTANCE_ID_1,
request_id='',
image_type=0,
status="saving",
uuid="2df2ccf6-bc1b-4853-aab0-25fda346b3bb").AndReturn(raw)
self.mox.ReplayAll()
notification = GlanceNotification(body, deployment, routing_key,
json)
self.assertEquals(notification.save(), raw)
self.mox.VerifyAll()
def test_save_image_exists(self):
raw = self.mox.CreateMockAnything()
audit_period_beginning = "2013-05-20 17:31:57.939614"
audit_period_ending = "2013-06-20 17:31:57.939614"
size = 123
uuid = "2df2ccf6-bc1b-4853-aab0-25fda346b3bb"
body = {
"event_type": "image.upload",
"timestamp": "2013-06-20 18:31:57.939614",
"publisher_id": "glance-api01-r2961.global.preprod-ord.ohthree.com",
"payload": {
"created_at": str(DUMMY_TIME),
"status": "saving",
"audit_period_beginning": audit_period_beginning,
"audit_period_ending": audit_period_ending,
"properties": {
"image_type": "snapshot",
"instance_uuid": INSTANCE_ID_1,
},
"size": size,
"owner": TENANT_ID_1,
"id": uuid
}
}
deployment = "1"
routing_key = "glance_monitor.info"
json = '{["routing_key", {%s}]}' % body
self.mox.StubOutWithMock(db, 'create_image_exists')
self.mox.StubOutWithMock(db, 'get_image_usage')
created_at_range = (DECIMAL_DUMMY_TIME, DECIMAL_DUMMY_TIME+1)
db.get_image_usage(created_at__range=created_at_range,
uuid=uuid).AndReturn(None)
db.create_image_exists(
created_at=utils.str_time_to_unix(str(DUMMY_TIME)),
owner=TENANT_ID_1,
raw=raw,
audit_period_beginning=utils.str_time_to_unix(audit_period_beginning),
audit_period_ending=utils.str_time_to_unix(audit_period_ending),
size=size,
uuid=uuid,
usage=None).AndReturn(raw)
self.mox.ReplayAll()
notification = GlanceNotification(body, deployment, routing_key,
json)
notification.save_exists(raw)
self.mox.VerifyAll()
def test_save_image_exists_with_delete_not_none(self):
raw = self.mox.CreateMockAnything()
delete = self.mox.CreateMockAnything()
audit_period_beginning = "2013-05-20 17:31:57.939614"
audit_period_ending = "2013-06-20 17:31:57.939614"
size = 123
uuid = "2df2ccf6-bc1b-4853-aab0-25fda346b3bb"
deleted_at = "2013-06-20 14:31:57.939614"
body = {
"event_type": "image.upload",
"timestamp": "2013-06-20 18:31:57.939614",
"publisher_id": "glance-api01-r2961.global.preprod-ord.ohthree.com",
"payload": {
"created_at": str(DUMMY_TIME),
"status": "saving",
"audit_period_beginning": audit_period_beginning,
"audit_period_ending": audit_period_ending,
"properties": {
"image_type": "snapshot",
"instance_uuid": INSTANCE_ID_1,
},
"deleted_at": deleted_at,
"size": size,
"owner": TENANT_ID_1,
"id": "2df2ccf6-bc1b-4853-aab0-25fda346b3bb",
}
}
deployment = "1"
routing_key = "glance_monitor.info"
json = '{["routing_key", {%s}]}' % body
self.mox.StubOutWithMock(db, 'create_image_exists')
self.mox.StubOutWithMock(db, 'get_image_usage')
self.mox.StubOutWithMock(db, 'get_image_delete')
created_at_range = (DECIMAL_DUMMY_TIME, DECIMAL_DUMMY_TIME+1)
db.get_image_usage(created_at__range=created_at_range,
uuid=uuid).AndReturn(None)
db.get_image_delete(created_at__range=created_at_range,
uuid=uuid).AndReturn(delete)
db.create_image_exists(
created_at=utils.str_time_to_unix(str(DUMMY_TIME)),
owner=TENANT_ID_1,
raw=raw,
audit_period_beginning=utils.str_time_to_unix(audit_period_beginning),
audit_period_ending=utils.str_time_to_unix(audit_period_ending),
size=size,
uuid=uuid,
usage=None,
delete=delete,
deleted_at=utils.str_time_to_unix(str(deleted_at))).AndReturn(raw)
self.mox.ReplayAll()
notification = GlanceNotification(body, deployment, routing_key,
json)
notification.save_exists(raw)
self.mox.VerifyAll()
def test_save_image_exists_with_usage_not_none(self):
raw = self.mox.CreateMockAnything()
usage = self.mox.CreateMockAnything()
audit_period_beginning = "2013-05-20 17:31:57.939614"
audit_period_ending = "2013-06-20 17:31:57.939614"
size = 123
uuid = "2df2ccf6-bc1b-4853-aab0-25fda346b3bb"
body = {
"event_type": "image.upload",
"timestamp": "2013-06-20 18:31:57.939614",
"publisher_id": "glance-api01-r2961.global.preprod-ord.ohthree.com",
"payload": {
"created_at": str(DUMMY_TIME),
"status": "saving",
"audit_period_beginning": audit_period_beginning,
"audit_period_ending": audit_period_ending,
"properties": {
"image_type": "snapshot",
"instance_uuid": INSTANCE_ID_1,
},
"size": size,
"owner": TENANT_ID_1,
"id": "2df2ccf6-bc1b-4853-aab0-25fda346b3bb",
}
}
deployment = "1"
routing_key = "glance_monitor.info"
json = '{["routing_key", {%s}]}' % body
self.mox.StubOutWithMock(db, 'create_image_exists')
self.mox.StubOutWithMock(db, 'get_image_usage')
self.mox.StubOutWithMock(db, 'get_image_delete')
created_at_range = (DECIMAL_DUMMY_TIME, DECIMAL_DUMMY_TIME+1)
db.get_image_usage(created_at__range=created_at_range,
uuid=uuid).AndReturn(usage)
db.create_image_exists(
created_at=utils.str_time_to_unix(str(DUMMY_TIME)),
owner=TENANT_ID_1,
raw=raw,
audit_period_beginning=utils.str_time_to_unix(audit_period_beginning),
audit_period_ending=utils.str_time_to_unix(audit_period_ending),
size=size,
uuid=uuid,
usage=usage).AndReturn(raw)
self.mox.ReplayAll()
notification = GlanceNotification(body, deployment, routing_key,
json)
notification.save_exists(raw)
self.mox.VerifyAll()
def test_save_usage_should_persist_image_usage(self):
raw = self.mox.CreateMockAnything()
size = 123
uuid = "2df2ccf6-bc1b-4853-aab0-25fda346b3bb"
body = {
"event_type": "image.upload",
"timestamp": "2013-06-20 18:31:57.939614",
"publisher_id": "glance-api01-r2961.global.preprod-ord.ohthree.com",
"payload": {
"created_at": str(DUMMY_TIME),
"size": size,
"owner": TENANT_ID_1,
"id": "2df2ccf6-bc1b-4853-aab0-25fda346b3bb",
}
}
deployment = "1"
routing_key = "glance_monitor.info"
json = '{["routing_key", {%s}]}' % body
self.mox.StubOutWithMock(db, 'create_image_usage')
db.create_image_usage(
created_at=utils.str_time_to_unix(str(DUMMY_TIME)),
owner=TENANT_ID_1,
last_raw=raw,
size=size,
uuid=uuid).AndReturn(raw)
self.mox.ReplayAll()
notification = GlanceNotification(body, deployment, routing_key, json)
notification.save_usage(raw)
self.mox.VerifyAll()
def test_save_delete_should_persist_image_delete(self):
raw = self.mox.CreateMockAnything()
uuid = "2df2ccf6-bc1b-4853-aab0-25fda346b3bb"
deleted_at = "2013-06-20 14:31:57.939614"
body = {
"event_type": "image.delete",
"publisher_id": "glance-api01-r2961.global.preprod-ord.ohthree.com",
"payload": {
"id": "2df2ccf6-bc1b-4853-aab0-25fda346b3bb",
"deleted_at": deleted_at
}
}
deployment = "1"
routing_key = "glance_monitor.info"
json = '{["routing_key", {%s}]}' % body
self.mox.StubOutWithMock(db, 'create_image_delete')
db.create_image_delete(
raw=raw,
uuid=uuid,
deleted_at=utils.str_time_to_unix(deleted_at)).AndReturn(raw)
self.mox.ReplayAll()
notification = GlanceNotification(body, deployment, routing_key, json)
notification.save_delete(raw)
self.mox.VerifyAll()
class NotificationTestCase(unittest.TestCase):
def setUp(self):
self.mox = mox.Mox()
def test_rawdata_kwargs(self):
message = {
'event_type': 'compute.instance.create.start',
'publisher_id': 'compute.cpu1-n01.example.com',
def tearDown(self):
self.mox.UnsetStubs()
def test_save_should_persist_generic_rawdata_to_database(self):
body = {
"event_type": "image.upload",
'_context_request_id': REQUEST_ID_1,
'_context_project_id': TENANT_ID_1,
'timestamp': '2013-06-12 06:30:52.790476',
'payload': {
"timestamp": TIMESTAMP_1,
"publisher_id": "glance-api01-r2961.global.preprod-ord.ohthree.com",
"message_id": MESSAGE_ID_1,
"payload": {
'instance_id': INSTANCE_ID_1,
'state': 'active',
'old_state': 'building',
'old_task_state': 'build',
"new_task_state": 'rebuild_spawning',
'image_meta': {
'image_type': 'base',
'org.openstack__1__architecture': 'x64',
'org.openstack__1__os_distro': 'com.microsoft.server',
'org.openstack__1__os_version': '2008.2',
'com.rackspace__1__options': '36'
}
"status": "saving",
"container_format": "ovf",
"tenant": "5877054"
}
}
kwargs = Notification(message).rawdata_kwargs('1', 'monitor.info', 'json')
deployment = "1"
routing_key = "generic_monitor.info"
json = '{["routing_key", {%s}]}' % body
raw = self.mox.CreateMockAnything()
self.mox.StubOutWithMock(db, 'create_generic_rawdata')
db.create_generic_rawdata(
deployment="1",
tenant=TENANT_ID_1,
json=json,
routing_key=routing_key,
when=utils.str_time_to_unix(TIMESTAMP_1),
publisher="glance-api01-r2961.global.preprod-ord.ohthree.com",
event="image.upload",
service="glance-api01-r2961",
host="global.preprod-ord.ohthree.com",
instance=INSTANCE_ID_1,
request_id=REQUEST_ID_1,
message_id=MESSAGE_ID_1).AndReturn(raw)
self.assertEquals(kwargs['host'], 'cpu1-n01.example.com')
self.assertEquals(kwargs['deployment'], '1')
self.assertEquals(kwargs['routing_key'], 'monitor.info')
self.assertEquals(kwargs['tenant'], TENANT_ID_1)
self.assertEquals(kwargs['json'], 'json')
self.assertEquals(kwargs['state'], 'active')
self.assertEquals(kwargs['old_state'], 'building')
self.assertEquals(kwargs['old_task'], 'build')
self.assertEquals(kwargs['task'], 'rebuild_spawning')
self.assertEquals(kwargs['image_type'], 1)
self.assertEquals(kwargs['when'], Decimal('1371018652.790476'))
self.assertEquals(kwargs['publisher'], 'compute.cpu1-n01.example.com')
self.assertEquals(kwargs['event'], 'compute.instance.create.start')
self.assertEquals(kwargs['request_id'], REQUEST_ID_1)
self.mox.ReplayAll()
def test_rawdata_kwargs_missing_image_meta(self):
message = {
'event_type': 'compute.instance.create.start',
'publisher_id': 'compute.cpu1-n01.example.com',
'_context_request_id': REQUEST_ID_1,
'_context_project_id': TENANT_ID_1,
'timestamp': '2013-06-12 06:30:52.790476',
'payload': {
'instance_id': INSTANCE_ID_1,
'state': 'active',
'old_state': 'building',
'old_task_state': 'build',
"new_task_state": 'rebuild_spawning',
'image_meta': {
'image_type': 'base',
}
}
}
kwargs = Notification(message).rawdata_kwargs('1', 'monitor.info', 'json')
self.assertEquals(kwargs['host'], 'cpu1-n01.example.com')
self.assertEquals(kwargs['deployment'], '1')
self.assertEquals(kwargs['routing_key'], 'monitor.info')
self.assertEquals(kwargs['tenant'], TENANT_ID_1)
self.assertEquals(kwargs['json'], 'json')
self.assertEquals(kwargs['state'], 'active')
self.assertEquals(kwargs['old_state'], 'building')
self.assertEquals(kwargs['old_task'], 'build')
self.assertEquals(kwargs['task'], 'rebuild_spawning')
self.assertEquals(kwargs['image_type'], 1)
self.assertEquals(kwargs['when'], Decimal('1371018652.790476'))
self.assertEquals(kwargs['publisher'], 'compute.cpu1-n01.example.com')
self.assertEquals(kwargs['event'], 'compute.instance.create.start')
self.assertEquals(kwargs['request_id'], REQUEST_ID_1)
def test_rawdata_kwargs_for_message_with_no_host(self):
message = {
'event_type': 'compute.instance.create.start',
'publisher_id': 'compute',
'_context_request_id': REQUEST_ID_1,
'_context_project_id': TENANT_ID_1,
'timestamp': '2013-06-12 06:30:52.790476',
'payload': {
'instance_id': INSTANCE_ID_1,
'state': 'active',
'old_state': 'building',
'old_task_state': 'build',
"new_task_state": 'rebuild_spawning',
'image_meta': {
'image_type': 'base',
'org.openstack__1__architecture': 'x64',
'org.openstack__1__os_distro': 'com.microsoft.server',
'org.openstack__1__os_version': '2008.2',
'com.rackspace__1__options': '36'
}
}
}
kwargs = Notification(message).rawdata_kwargs('1', 'monitor.info', 'json')
self.assertEquals(kwargs['host'], None)
self.assertEquals(kwargs['deployment'], '1')
self.assertEquals(kwargs['routing_key'], 'monitor.info')
self.assertEquals(kwargs['tenant'], TENANT_ID_1)
self.assertEquals(kwargs['json'], 'json')
self.assertEquals(kwargs['state'], 'active')
self.assertEquals(kwargs['old_state'], 'building')
self.assertEquals(kwargs['old_task'], 'build')
self.assertEquals(kwargs['task'], 'rebuild_spawning')
self.assertEquals(kwargs['image_type'], 1)
self.assertEquals(kwargs['when'], Decimal('1371018652.790476'))
self.assertEquals(kwargs['publisher'], 'compute')
self.assertEquals(kwargs['event'], 'compute.instance.create.start')
self.assertEquals(kwargs['request_id'], REQUEST_ID_1)
def test_rawdata_kwargs_for_message_with_exception(self):
message = {
'event_type': 'compute.instance.create.start',
'publisher_id': 'compute.cpu1-n01.example.com',
'_context_request_id': REQUEST_ID_1,
'_context_project_id': TENANT_ID_1,
'timestamp': '2013-06-12 06:30:52.790476',
'payload': {
'exception': {'kwargs':{'uuid': INSTANCE_ID_1}},
'instance_id': INSTANCE_ID_1,
'state': 'active',
'old_state': 'building',
'old_task_state': 'build',
"new_task_state": 'rebuild_spawning',
'image_meta': {
'image_type': 'base',
'org.openstack__1__architecture': 'x64',
'org.openstack__1__os_distro': 'com.microsoft.server',
'org.openstack__1__os_version': '2008.2',
'com.rackspace__1__options': '36'
}
}
}
kwargs = Notification(message).rawdata_kwargs('1', 'monitor.info', 'json')
self.assertEquals(kwargs['host'], 'cpu1-n01.example.com')
self.assertEquals(kwargs['deployment'], '1')
self.assertEquals(kwargs['routing_key'], 'monitor.info')
self.assertEquals(kwargs['tenant'], TENANT_ID_1)
self.assertEquals(kwargs['json'], 'json')
self.assertEquals(kwargs['state'], 'active')
self.assertEquals(kwargs['old_state'], 'building')
self.assertEquals(kwargs['old_task'], 'build')
self.assertEquals(kwargs['task'], 'rebuild_spawning')
self.assertEquals(kwargs['image_type'], 1)
self.assertEquals(kwargs['when'], Decimal('1371018652.790476'))
self.assertEquals(kwargs['publisher'], 'compute.cpu1-n01.example.com')
self.assertEquals(kwargs['event'], 'compute.instance.create.start')
self.assertEquals(kwargs['request_id'], REQUEST_ID_1)
notification = Notification(body, deployment, routing_key, json)
self.assertEquals(notification.save(), raw)
self.mox.VerifyAll()

102
tests/unit/test_stacklog.py Normal file
View File

@ -0,0 +1,102 @@
import glob
import logging
import os
from unittest import TestCase
import mox
from stacktach import stacklog
import __builtin__
from stacktach.stacklog import ExchangeLogger
class StacklogTestCase(TestCase):
def setUp(self):
self.mox = mox.Mox()
def tearDown(self):
self.mox.UnsetStubs()
def test_get_logger_should_get_exchange_logger_if_exchange_provided(self):
filename = 'filename'
logger = stacklog.get_logger(filename, 'nova')
self.assertIsInstance(logger, ExchangeLogger)
for file in glob.glob('{0}.log*'.format(filename)):
os.remove(file)
def test_get_logger_should_get_default_logger_if_exchange_not_provided(self):
filename = 'default_logger'
logger = stacklog.get_logger(filename)
self.assertIsInstance(logger, logging.Logger)
for file in glob.glob('{0}.log*'.format(filename)):
os.remove(file)
class ExchangeLoggerTestCase(TestCase):
def setUp(self):
self.mox = mox.Mox()
def tearDown(self):
self.mox.UnsetStubs()
def _setup_logger_mocks(self, name='name'):
mock_logger = self.mox.CreateMockAnything()
self.mox.StubOutWithMock(logging, 'getLogger')
logging.getLogger(name).AndReturn(mock_logger)
mock_logger.setLevel(logging.DEBUG)
self.mox.StubOutClassWithMocks(logging.handlers,
'TimedRotatingFileHandler')
filename = "{0}.log".format(name)
handler = logging.handlers.TimedRotatingFileHandler(
filename, backupCount=3, interval=1, when='midnight')
self.mox.StubOutClassWithMocks(logging, 'Formatter')
mock_formatter = logging.Formatter(
"%(asctime)s - %(name)s - %(levelname)s - %(message)s")
handler.setFormatter(mock_formatter)
mock_logger.addHandler(handler)
mock_logger.handlers = [handler]
handler.doRollover()
return mock_logger
def test_exchange_logger_should_append_exchange_name_to_info(self):
mock_logger = self._setup_logger_mocks()
mock_logger.info('exchange: Log %s', 'args', xyz='xyz')
self.mox.ReplayAll()
log = ExchangeLogger('exchange', 'name')
log.info("Log %s", 'args', xyz='xyz')
self.mox.VerifyAll()
def test_exchange_logger_should_append_exchange_name_to_warn(self):
mock_logger = self._setup_logger_mocks()
mock_logger.warn('exchange: Log %s', 'args', xyz='xyz')
self.mox.ReplayAll()
logger = ExchangeLogger('exchange', 'name')
logger.warn("Log %s", 'args', xyz='xyz')
self.mox.VerifyAll()
def test_exchange_logger_should_append_exchange_name_to_error(self):
mock_logger = self._setup_logger_mocks()
mock_logger.error('exchange: Log %s', 'args', xyz='xyz')
self.mox.ReplayAll()
logger = ExchangeLogger('exchange', 'name')
logger.error("Log %s", 'args', xyz='xyz')
self.mox.VerifyAll()
def test_exchange_logger_should_append_exchange_name_to_exception(self):
mock_logger = self._setup_logger_mocks()
mock_logger.error('exchange: Log %s', 'args', xyz='xyz')
self.mox.ReplayAll()
logger = ExchangeLogger('exchange', 'name')
logger.exception("Log %s", 'args', xyz='xyz')
self.mox.VerifyAll()
def test_exchange_logger_should_use_default_name_if_not_provided(self):
self._setup_logger_mocks('stacktach-default')
self.mox.ReplayAll()
ExchangeLogger('exchange')
self.mox.VerifyAll()

564
tests/unit/test_stacktach.py
View File

@ -36,7 +36,9 @@ from utils import TENANT_ID_1
from utils import INSTANCE_TYPE_ID_1
from utils import DUMMY_TIME
from utils import INSTANCE_TYPE_ID_2
from utils import IMAGE_UUID_1
from stacktach import stacklog
from stacktach import notification
from stacktach import views
@ -59,54 +61,45 @@ class StacktachRawParsingTestCase(unittest.TestCase):
dict = {
'timestamp': when,
}
args = ('monitor.info', dict)
routing_key = 'monitor.info'
args = (routing_key, dict)
json_args = json.dumps(args)
raw_values = {
'deployment': deployment,
'when': utils.decimal_utc(datetime.datetime.strptime(when, '%Y-%m-%d %H:%M:%S.%f')),
'host': 'api',
'routing_key': 'monitor.info',
'json': json_args
}
old_info_handler = views.NOTIFICATIONS['monitor.info']
mock_record = self.mox.CreateMockAnything()
mock_notification = self.mox.CreateMockAnything()
mock_notification.rawdata_kwargs(deployment, 'monitor.info', json_args).AndReturn(raw_values)
views.NOTIFICATIONS['monitor.info'] = lambda message_body: mock_notification
views.STACKDB.create_rawdata(**raw_values)
mock_notification.save().AndReturn(mock_record)
self.mox.StubOutWithMock(notification, 'notification_factory')
exchange = 'nova'
notification.notification_factory(dict, deployment, routing_key,
json_args, exchange).AndReturn(
mock_notification)
self.mox.ReplayAll()
views.process_raw_data(deployment, args, json_args)
self.mox.VerifyAll()
views.NOTIFICATIONS['monitor.info'] = old_info_handler
self.assertEquals(
views.process_raw_data(deployment, args, json_args, exchange),
(mock_record, mock_notification))
self.mox.VerifyAll()
def test_process_raw_data_old_timestamp(self):
deployment = self.mox.CreateMockAnything()
when = '2013-1-25T13:38:23.123'
dict = {
'_context_timestamp': when,
}
}
routing_key = 'monitor.info'
args = ('monitor.info', dict)
json_args = json.dumps(args[1])
raw_values = {
'deployment': deployment,
'when': utils.decimal_utc(datetime.datetime.strptime(when, '%Y-%m-%dT%H:%M:%S.%f')),
'host': 'api',
'routing_key': 'monitor.info',
'json': json_args
}
old_info_handler = views.NOTIFICATIONS['monitor.info']
mock_notification = self.mox.CreateMockAnything()
mock_notification.rawdata_kwargs(deployment, 'monitor.info', json_args).AndReturn(raw_values)
views.NOTIFICATIONS['monitor.info'] = lambda message_body: mock_notification
views.STACKDB.create_rawdata(**raw_values)
mock_notification = self.mox.CreateMockAnything()
mock_notification.save()
self.mox.StubOutWithMock(notification, 'notification_factory')
exchange = 'nova'
notification.notification_factory(dict, deployment, routing_key,
json_args, exchange).AndReturn(mock_notification)
self.mox.ReplayAll()
views.process_raw_data(deployment, args, json_args)
views.process_raw_data(deployment, args, json_args, exchange)
self.mox.VerifyAll()
views.NOTIFICATIONS['monitor.info'] = old_info_handler
class StacktachLifecycleTestCase(unittest.TestCase):
def setUp(self):
@ -311,15 +304,30 @@ class StacktachUsageParsingTestCase(unittest.TestCase):
stacklog.get_logger(name=name).AndReturn(self.log)
def test_process_usage_for_new_launch_create_start(self):
kwargs = {'launched': str(DUMMY_TIME), 'tenant_id': TENANT_ID_1, 'rax_options': RAX_OPTIONS_1,
'os_architecture': OS_ARCH_1, 'os_version': OS_VERSION_1, 'os_distro': OS_DISTRO_1 }
notification = utils.create_nova_notif(request_id=REQUEST_ID_1, **kwargs)
event = 'compute.instance.create.start'
raw, usage = self._setup_process_usage_mocks(event, notification)
notification = self.mox.CreateMockAnything()
notification.launched_at = str(DUMMY_TIME)
notification.tenant = TENANT_ID_1
notification.rax_options = RAX_OPTIONS_1
notification.os_architecture = OS_ARCH_1
notification.os_version = OS_VERSION_1
notification.os_distro = OS_DISTRO_1
notification.instance = INSTANCE_ID_1
notification.request_id = REQUEST_ID_1
notification.instance_type_id = INSTANCE_TYPE_ID_1
views._process_usage_for_new_launch(raw, notification[1])
raw = self.mox.CreateMockAnything()
raw.event = 'compute.instance.create.start'
self.assertEquals(usage.instance_type_id, '1')
usage = self.mox.CreateMockAnything()
views.STACKDB.get_or_create_instance_usage(instance=INSTANCE_ID_1,
request_id=REQUEST_ID_1) \
.AndReturn((usage, True))
views.STACKDB.save(usage)
self.mox.ReplayAll()
views._process_usage_for_new_launch(raw, notification)
self.assertEquals(usage.instance_type_id, INSTANCE_TYPE_ID_1)
self.assertEquals(usage.tenant, TENANT_ID_1)
self.assertEquals(usage.os_architecture, OS_ARCH_1)
self.assertEquals(usage.os_version, OS_VERSION_1)
@ -329,15 +337,29 @@ class StacktachUsageParsingTestCase(unittest.TestCase):
self.mox.VerifyAll()
def test_process_usage_for_new_launch_rebuild_start(self):
kwargs = {'launched': str(DUMMY_TIME), 'tenant_id': TENANT_ID_1, 'rax_options': RAX_OPTIONS_1,
'os_architecture': OS_ARCH_1, 'os_version': OS_VERSION_1, 'os_distro': OS_DISTRO_1 }
notification = utils.create_nova_notif(request_id=REQUEST_ID_1, **kwargs)
event = 'compute.instance.rebuild.start'
raw, usage = self._setup_process_usage_mocks(event, notification)
notification = self.mox.CreateMockAnything()
notification.launched_at = str(DUMMY_TIME)
notification.tenant = TENANT_ID_1
notification.rax_options = RAX_OPTIONS_1
notification.os_architecture = OS_ARCH_1
notification.os_version = OS_VERSION_1
notification.os_distro = OS_DISTRO_1
notification.instance = INSTANCE_ID_1
notification.request_id = REQUEST_ID_1
notification.instance_type_id = INSTANCE_TYPE_ID_1
views._process_usage_for_new_launch(raw, notification[1])
raw = self.mox.CreateMockAnything()
raw.event = 'compute.instance.rebuild.start'
usage = self.mox.CreateMockAnything()
views.STACKDB.get_or_create_instance_usage(instance=INSTANCE_ID_1,
request_id=REQUEST_ID_1) \
.AndReturn((usage, True))
views.STACKDB.save(usage)
self.mox.ReplayAll()
self.assertEquals(usage.instance_type_id, '1')
views._process_usage_for_new_launch(raw, notification)
self.assertEquals(usage.instance_type_id, INSTANCE_TYPE_ID_1)
self.assertEquals(usage.tenant, TENANT_ID_1)
self.assertEquals(usage.os_architecture, OS_ARCH_1)
self.assertEquals(usage.os_version, OS_VERSION_1)
@ -346,14 +368,29 @@ class StacktachUsageParsingTestCase(unittest.TestCase):
self.mox.VerifyAll()
def test_process_usage_for_new_launch_rebuild_start_when_no_launched_at_in_db(self):
kwargs = {'launched': str(DUMMY_TIME), 'tenant_id': TENANT_ID_1, 'rax_options': RAX_OPTIONS_1,
'os_architecture': OS_ARCH_1, 'os_version': OS_VERSION_1, 'os_distro': OS_DISTRO_1 }
notification = utils.create_nova_notif(request_id=REQUEST_ID_1, **kwargs)
event = 'compute.instance.rebuild.start'
raw, usage = self._setup_process_usage_mocks(event, notification)
usage.launched_at = None
notification = self.mox.CreateMockAnything()
notification.launched_at = str(DUMMY_TIME)
notification.tenant = TENANT_ID_1
notification.rax_options = RAX_OPTIONS_1
notification.os_architecture = OS_ARCH_1
notification.os_version = OS_VERSION_1
notification.os_distro = OS_DISTRO_1
notification.instance = INSTANCE_ID_1
notification.request_id = REQUEST_ID_1
notification.instance_type_id = INSTANCE_TYPE_ID_1
views._process_usage_for_new_launch(raw, notification[1])
raw = self.mox.CreateMockAnything()
raw.event = 'compute.instance.rebuild.start'
usage = self.mox.CreateMockAnything()
usage.launched_at = None
views.STACKDB.get_or_create_instance_usage(instance=INSTANCE_ID_1,
request_id=REQUEST_ID_1) \
.AndReturn((usage, True))
views.STACKDB.save(usage)
self.mox.ReplayAll()
views._process_usage_for_new_launch(raw, notification)
self.assertEqual(usage.launched_at, utils.decimal_utc(DUMMY_TIME))
self.assertEquals(usage.tenant, TENANT_ID_1)
@ -365,14 +402,31 @@ class StacktachUsageParsingTestCase(unittest.TestCase):
self.mox.VerifyAll()
def test_process_usage_for_new_launch_resize_prep_start_when_no_launched_at_in_db(self):
kwargs = {'launched': str(DUMMY_TIME), 'tenant_id': TENANT_ID_1, 'rax_options': RAX_OPTIONS_1,
'os_architecture': OS_ARCH_1, 'os_version': OS_VERSION_1, 'os_distro': OS_DISTRO_1 }
notification = utils.create_nova_notif(request_id=REQUEST_ID_1, **kwargs)
event = 'compute.instance.resize.prep.start'
raw, usage = self._setup_process_usage_mocks(event, notification)
notification = self.mox.CreateMockAnything()
notification.launched_at = str(DUMMY_TIME)
notification.tenant = TENANT_ID_1
notification.rax_options = RAX_OPTIONS_1
notification.os_architecture = OS_ARCH_1
notification.os_version = OS_VERSION_1
notification.os_distro = OS_DISTRO_1
notification.instance = INSTANCE_ID_1
notification.request_id = REQUEST_ID_1
notification.instance_type_id = INSTANCE_TYPE_ID_1
raw = self.mox.CreateMockAnything()
raw.event = 'compute.instance.resize.prep.start'
usage = self.mox.CreateMockAnything()
usage.launched_at = None
views.STACKDB.get_or_create_instance_usage(instance=INSTANCE_ID_1,
request_id=REQUEST_ID_1) \
.AndReturn((usage, True))
views.STACKDB.save(usage)
self.mox.ReplayAll()
usage.launched_at = None
views._process_usage_for_new_launch(raw, notification[1])
views._process_usage_for_new_launch(raw, notification)
self.assertEqual(usage.launched_at, utils.decimal_utc(DUMMY_TIME))
self.assertEquals(usage.tenant, TENANT_ID_1)
@ -384,14 +438,29 @@ class StacktachUsageParsingTestCase(unittest.TestCase):
self.mox.VerifyAll()
def test_process_usage_for_new_launch_resize_revert_start_when_no_launched_at_in_db(self):
kwargs = {'launched': str(DUMMY_TIME), 'tenant_id': TENANT_ID_1,'rax_options': RAX_OPTIONS_1,
'os_architecture': OS_ARCH_1, 'os_version': OS_VERSION_1, 'os_distro': OS_DISTRO_1 }
notification = utils.create_nova_notif(request_id=REQUEST_ID_1, **kwargs)
event = 'compute.instance.resize.revert.start'
raw, usage = self._setup_process_usage_mocks(event, notification)
usage.launched_at = None
notification = self.mox.CreateMockAnything()
notification.launched_at = str(DUMMY_TIME)
notification.tenant = TENANT_ID_1
notification.rax_options = RAX_OPTIONS_1
notification.os_architecture = OS_ARCH_1
notification.os_version = OS_VERSION_1
notification.os_distro = OS_DISTRO_1
notification.instance = INSTANCE_ID_1
notification.request_id = REQUEST_ID_1
notification.instance_type_id = INSTANCE_TYPE_ID_1
views._process_usage_for_new_launch(raw, notification[1])
raw = self.mox.CreateMockAnything()
raw.event = 'compute.instance.resize.revert.start'
usage = self.mox.CreateMockAnything()
usage.launched_at = None
views.STACKDB.get_or_create_instance_usage(instance=INSTANCE_ID_1,
request_id=REQUEST_ID_1) \
.AndReturn((usage, True))
views.STACKDB.save(usage)
self.mox.ReplayAll()
views._process_usage_for_new_launch(raw, notification)
self.assertEquals(usage.tenant, TENANT_ID_1)
self.assertEqual(usage.launched_at, utils.decimal_utc(DUMMY_TIME))
@ -403,17 +472,30 @@ class StacktachUsageParsingTestCase(unittest.TestCase):
self.mox.VerifyAll()
def test_process_usage_for_new_launch_resize_prep_start_when_launched_at_in_db(self):
kwargs = {'launched': str(DUMMY_TIME), 'tenant_id': TENANT_ID_1,
'rax_options': RAX_OPTIONS_1, 'os_architecture': OS_ARCH_1,
'os_version': OS_VERSION_1, 'os_distro': OS_DISTRO_1 }
notification = utils.create_nova_notif(request_id=REQUEST_ID_1,
**kwargs)
event = 'compute.instance.resize.prep.start'
raw, usage = self._setup_process_usage_mocks(event, notification)
orig_launched_at = utils.decimal_utc(DUMMY_TIME - datetime.timedelta(days=1))
usage.launched_at = orig_launched_at
notification = self.mox.CreateMockAnything()
notification.launched_at = str(DUMMY_TIME)
notification.tenant = TENANT_ID_1
notification.rax_options = RAX_OPTIONS_1
notification.os_architecture = OS_ARCH_1
notification.os_version = OS_VERSION_1
notification.os_distro = OS_DISTRO_1
notification.instance = INSTANCE_ID_1
notification.request_id = REQUEST_ID_1
notification.instance_type_id = INSTANCE_TYPE_ID_1
views._process_usage_for_new_launch(raw, notification[1])
raw = self.mox.CreateMockAnything()
raw.event = 'compute.instance.resize.prep.start'
orig_launched_at = utils.decimal_utc(DUMMY_TIME - datetime.timedelta(days=1))
usage = self.mox.CreateMockAnything()
usage.launched_at = orig_launched_at
views.STACKDB.get_or_create_instance_usage(instance=INSTANCE_ID_1,
request_id=REQUEST_ID_1) \
.AndReturn((usage, True))
views.STACKDB.save(usage)
self.mox.ReplayAll()
views._process_usage_for_new_launch(raw, notification)
self.assertEqual(usage.launched_at, orig_launched_at)
self.assertEqual(usage.tenant, TENANT_ID_1)
@ -425,16 +507,30 @@ class StacktachUsageParsingTestCase(unittest.TestCase):
self.mox.VerifyAll()
def test_process_usage_for_updates_create_end(self):
kwargs = {'launched': str(DUMMY_TIME),
'tenant_id': TENANT_ID_1, 'rax_options': RAX_OPTIONS_1,
'os_architecture': OS_ARCH_1, 'os_version': OS_VERSION_1,
'os_distro': OS_DISTRO_1 }
notification = utils.create_nova_notif(request_id=REQUEST_ID_1,
**kwargs)
event = 'compute.instance.create.end'
raw, usage = self._setup_process_usage_mocks(event, notification)
notification = self.mox.CreateMockAnything()
notification.launched_at = str(DUMMY_TIME)
notification.tenant = TENANT_ID_1
notification.rax_options = RAX_OPTIONS_1
notification.os_architecture = OS_ARCH_1
notification.os_version = OS_VERSION_1
notification.os_distro = OS_DISTRO_1
notification.instance = INSTANCE_ID_1
notification.request_id = REQUEST_ID_1
notification.instance_type_id = INSTANCE_TYPE_ID_1
notification.message = None
views._process_usage_for_updates(raw, notification[1])
raw = self.mox.CreateMockAnything()
raw.event = 'compute.instance.create.end'
usage = self.mox.CreateMockAnything()
usage.launched_at = None
views.STACKDB.get_or_create_instance_usage(instance=INSTANCE_ID_1,
request_id=REQUEST_ID_1) \
.AndReturn((usage, True))
views.STACKDB.save(usage)
self.mox.ReplayAll()
views._process_usage_for_updates(raw, notification)
self.assertEqual(usage.launched_at, utils.decimal_utc(DUMMY_TIME))
self.assertEqual(usage.tenant, TENANT_ID_1)
@ -446,17 +542,30 @@ class StacktachUsageParsingTestCase(unittest.TestCase):
self.mox.VerifyAll()
def test_process_usage_for_updates_create_end_success_message(self):
kwargs = {'launched': str(DUMMY_TIME),
'tenant_id': TENANT_ID_1, 'rax_options': RAX_OPTIONS_1,
'os_architecture': OS_ARCH_1, 'os_version': OS_VERSION_1,
'os_distro': OS_DISTRO_1 }
notification = utils.create_nova_notif(request_id=REQUEST_ID_1,
**kwargs)
notification[1]['payload']['message'] = "Success"
event = 'compute.instance.create.end'
raw, usage = self._setup_process_usage_mocks(event, notification)
notification = self.mox.CreateMockAnything()
notification.launched_at = str(DUMMY_TIME)
notification.tenant = TENANT_ID_1
notification.rax_options = RAX_OPTIONS_1
notification.os_architecture = OS_ARCH_1
notification.os_version = OS_VERSION_1
notification.os_distro = OS_DISTRO_1
notification.instance = INSTANCE_ID_1
notification.request_id = REQUEST_ID_1
notification.instance_type_id = INSTANCE_TYPE_ID_1
notification.message = 'Success'
views._process_usage_for_updates(raw, notification[1])
raw = self.mox.CreateMockAnything()
raw.event = 'compute.instance.create.end'
usage = self.mox.CreateMockAnything()
usage.launched_at = None
views.STACKDB.get_or_create_instance_usage(instance=INSTANCE_ID_1,
request_id=REQUEST_ID_1) \
.AndReturn((usage, True))
views.STACKDB.save(usage)
self.mox.ReplayAll()
views._process_usage_for_updates(raw, notification)
self.assertEqual(usage.launched_at, utils.decimal_utc(DUMMY_TIME))
self.assertEqual(usage.tenant, TENANT_ID_1)
@ -468,37 +577,42 @@ class StacktachUsageParsingTestCase(unittest.TestCase):
self.mox.VerifyAll()
def test_process_usage_for_updates_create_end_error_message(self):
kwargs = {'launched': str(DUMMY_TIME),
'tenant_id': TENANT_ID_1, 'rax_options': RAX_OPTIONS_1,
'os_architecture': OS_ARCH_1, 'os_version': OS_VERSION_1,
'os_distro': OS_DISTRO_1 }
notification = utils.create_nova_notif(request_id=REQUEST_ID_1,
**kwargs)
notification[1]['payload']['message'] = "Error"
event = 'compute.instance.create.end'
when_time = DUMMY_TIME
when_decimal = utils.decimal_utc(when_time)
json_str = json.dumps(notification)
raw = utils.create_raw(self.mox, when_decimal, event=event,
json_str=json_str)
notification = self.mox.CreateMockAnything()
notification.message = 'Error'
raw = self.mox.CreateMockAnything()
raw.event = 'compute.instance.create.end'
self.mox.ReplayAll()
views._process_usage_for_updates(raw, notification[1])
views._process_usage_for_updates(raw, notification)
self.mox.VerifyAll()
def test_process_usage_for_updates_revert_end(self):
kwargs = {'launched': str(DUMMY_TIME),
'type_id': INSTANCE_TYPE_ID_1,
'tenant_id': TENANT_ID_1, 'rax_options': RAX_OPTIONS_1,
'os_architecture': OS_ARCH_1, 'os_version': OS_VERSION_1,
'os_distro': OS_DISTRO_1 }
notification = utils.create_nova_notif(request_id=REQUEST_ID_1,
**kwargs)
event = 'compute.instance.resize.revert.end'
raw, usage = self._setup_process_usage_mocks(event, notification)
notification = self.mox.CreateMockAnything()
notification.launched_at = str(DUMMY_TIME)
notification.tenant = TENANT_ID_1
notification.rax_options = RAX_OPTIONS_1
notification.os_architecture = OS_ARCH_1
notification.os_version = OS_VERSION_1
notification.os_distro = OS_DISTRO_1
notification.instance = INSTANCE_ID_1
notification.request_id = REQUEST_ID_1
notification.instance_type_id = INSTANCE_TYPE_ID_1
notification.message = None
views._process_usage_for_updates(raw, notification[1])
raw = self.mox.CreateMockAnything()
raw.event = 'compute.instance.resize.revert.end'
usage = self.mox.CreateMockAnything()
usage.launched_at = None
views.STACKDB.get_or_create_instance_usage(instance=INSTANCE_ID_1,
request_id=REQUEST_ID_1) \
.AndReturn((usage, True))
views.STACKDB.save(usage)
self.mox.ReplayAll()
views._process_usage_for_updates(raw, notification)
self.assertEqual(usage.instance_type_id, INSTANCE_TYPE_ID_1)
self.assertEqual(usage.launched_at, utils.decimal_utc(DUMMY_TIME))
@ -511,17 +625,30 @@ class StacktachUsageParsingTestCase(unittest.TestCase):
self.mox.VerifyAll()
def test_process_usage_for_updates_prep_end(self):
kwargs = {'launched': str(DUMMY_TIME),
'new_type_id': INSTANCE_TYPE_ID_2,
'tenant_id': TENANT_ID_1, 'rax_options': RAX_OPTIONS_1,
'os_architecture': OS_ARCH_1, 'os_version': OS_VERSION_1,
'os_distro': OS_DISTRO_1 }
notification = utils.create_nova_notif(request_id=REQUEST_ID_1,
**kwargs)
event = 'compute.instance.resize.prep.end'
raw, usage = self._setup_process_usage_mocks(event, notification)
notification = self.mox.CreateMockAnything()
notification.launched_at = str(DUMMY_TIME)
notification.tenant = TENANT_ID_1
notification.rax_options = RAX_OPTIONS_1
notification.os_architecture = OS_ARCH_1
notification.os_version = OS_VERSION_1
notification.os_distro = OS_DISTRO_1
notification.instance = INSTANCE_ID_1
notification.request_id = REQUEST_ID_1
notification.new_instance_type_id = INSTANCE_TYPE_ID_2
notification.message = None
views._process_usage_for_updates(raw, notification[1])
raw = self.mox.CreateMockAnything()
raw.event = 'compute.instance.resize.prep.end'
usage = self.mox.CreateMockAnything()
usage.launched_at = None
views.STACKDB.get_or_create_instance_usage(instance=INSTANCE_ID_1,
request_id=REQUEST_ID_1) \
.AndReturn((usage, True))
views.STACKDB.save(usage)
self.mox.ReplayAll()
views._process_usage_for_updates(raw, notification)
self.assertEqual(usage.instance_type_id, INSTANCE_TYPE_ID_2)
self.assertEquals(usage.tenant, TENANT_ID_1)
@ -532,43 +659,29 @@ class StacktachUsageParsingTestCase(unittest.TestCase):
self.mox.VerifyAll()
def _setup_process_usage_mocks(self, event, notification):
when_time = DUMMY_TIME
when_decimal = utils.decimal_utc(when_time)
json_str = json.dumps(notification)
raw = utils.create_raw(self.mox, when_decimal, event=event,
json_str=json_str)
usage = self.mox.CreateMockAnything()
views.STACKDB.get_or_create_instance_usage(instance=INSTANCE_ID_1,
request_id=REQUEST_ID_1) \
.AndReturn((usage, True))
views.STACKDB.save(usage)
self.mox.ReplayAll()
return raw, usage
def test_process_delete(self):
delete_time = datetime.datetime.utcnow()
launch_time = delete_time-datetime.timedelta(days=1)
launch_decimal = utils.decimal_utc(launch_time)
delete_decimal = utils.decimal_utc(delete_time)
notif = utils.create_nova_notif(request_id=REQUEST_ID_1,
launched=str(launch_time),
deleted=str(delete_time))
json_str = json.dumps(notif)
event = 'compute.instance.delete.end'
raw = utils.create_raw(self.mox, delete_decimal, event=event,
json_str=json_str)
notification = self.mox.CreateMockAnything()
notification.instance = INSTANCE_ID_1
notification.deleted_at = str(delete_time)
notification.launched_at = str(launch_time)
raw = self.mox.CreateMockAnything()
delete = self.mox.CreateMockAnything()
delete.instance = INSTANCE_ID_1
delete.launched_at = launch_decimal
delete.deleted_at = delete_decimal
views.STACKDB.get_or_create_instance_delete(instance=INSTANCE_ID_1,
deleted_at=delete_decimal)\
.AndReturn((delete, True))
views.STACKDB.get_or_create_instance_delete(
instance=INSTANCE_ID_1, deleted_at=delete_decimal)\
.AndReturn((delete, True))
views.STACKDB.save(delete)
self.mox.ReplayAll()
views._process_delete(raw, notif[1])
views._process_delete(raw, notification)
self.assertEqual(delete.instance, INSTANCE_ID_1)
self.assertEqual(delete.launched_at, launch_decimal)
self.assertEqual(delete.deleted_at, delete_decimal)
@ -576,47 +689,50 @@ class StacktachUsageParsingTestCase(unittest.TestCase):
def test_process_delete_no_launch(self):
delete_time = datetime.datetime.utcnow()
launch_time = delete_time-datetime.timedelta(days=1)
delete_decimal = utils.decimal_utc(delete_time)
notif = utils.create_nova_notif(request_id=REQUEST_ID_1,
deleted=str(delete_time))
json_str = json.dumps(notif)
event = 'compute.instance.delete.end'
raw = utils.create_raw(self.mox, delete_decimal, event=event,
json_str=json_str)
notification = self.mox.CreateMockAnything()
notification.instance = INSTANCE_ID_1
notification.deleted_at = str(delete_time)
notification.launched_at = str(launch_time)
raw = self.mox.CreateMockAnything()
delete = self.mox.CreateMockAnything()
delete.instance = INSTANCE_ID_1
delete.deleted_at = delete_decimal
views.STACKDB.get_or_create_instance_delete(instance=INSTANCE_ID_1,
deleted_at=delete_decimal)\
.AndReturn((delete, True))
views.STACKDB.get_or_create_instance_delete(
instance=INSTANCE_ID_1, deleted_at=delete_decimal)\
.AndReturn((delete, True))
views.STACKDB.save(delete)
self.mox.ReplayAll()
views._process_delete(raw, notif[1])
views._process_delete(raw, notification)
self.assertEqual(delete.instance, INSTANCE_ID_1)
self.assertEqual(delete.deleted_at, delete_decimal)
self.mox.VerifyAll()
def test_process_exists(self):
notification = self.mox.CreateMockAnything()
current_time = datetime.datetime.utcnow()
launch_time = current_time - datetime.timedelta(hours=23)
launch_decimal = utils.decimal_utc(launch_time)
current_decimal = utils.decimal_utc(current_time)
audit_beginning = current_time - datetime.timedelta(hours=20)
audit_beginning_decimal = utils.decimal_utc(audit_beginning)
audit_ending_decimal = utils.decimal_utc(current_time)
notif = utils.create_nova_notif(launched=str(launch_time),
audit_period_beginning=str(audit_beginning),
audit_period_ending=str(current_time),
tenant_id=TENANT_ID_1,
os_architecture=OS_ARCH_1,
os_version=OS_VERSION_1,
os_distro=OS_DISTRO_1,
rax_options=RAX_OPTIONS_1)
json_str = json.dumps(notif)
event = 'compute.instance.exists'
raw = utils.create_raw(self.mox, current_decimal, event=event,
json_str=json_str)
notification.launched_at = str(launch_time)
notification.audit_period_beginning = str(audit_beginning)
notification.audit_period_ending = str(current_time)
notification.tenant = TENANT_ID_1
notification.os_architecture = OS_ARCH_1
notification.os_version = OS_VERSION_1
notification.os_distro = OS_DISTRO_1
notification.rax_options = RAX_OPTIONS_1
notification.instance = INSTANCE_ID_1
notification.deleted_at = ''
notification.instance_type_id = INSTANCE_TYPE_ID_1
notification.message_id = MESSAGE_ID_1
raw = self.mox.CreateMockAnything()
usage = self.mox.CreateMockAnything()
launched_range = (launch_decimal, launch_decimal+1)
views.STACKDB.get_instance_usage(instance=INSTANCE_ID_1,
@ -628,7 +744,7 @@ class StacktachUsageParsingTestCase(unittest.TestCase):
'launched_at': launch_decimal,
'audit_period_beginning': audit_beginning_decimal,
'audit_period_ending': audit_ending_decimal,
'instance_type_id': '1',
'instance_type_id': INSTANCE_TYPE_ID_1,
'usage': usage,
'raw': raw,
'tenant': TENANT_ID_1,
@ -641,50 +757,45 @@ class StacktachUsageParsingTestCase(unittest.TestCase):
views.STACKDB.create_instance_exists(**exists_values).AndReturn(exists)
views.STACKDB.save(exists)
self.mox.ReplayAll()
views._process_exists(raw, notif[1])
views._process_exists(raw, notification)
self.mox.VerifyAll()
def test_process_exists_no_launched_at(self):
current_time = datetime.datetime.utcnow()
current_decimal = utils.decimal_utc(current_time)
audit_beginning = current_time - datetime.timedelta(hours=20)
notif = utils.create_nova_notif(audit_period_beginning=str(audit_beginning),
audit_period_ending=str(current_time),
tenant_id=TENANT_ID_1)
json_str = json.dumps(notif)
event = 'compute.instance.exists'
raw = utils.create_raw(self.mox, current_decimal, event=event,
json_str=json_str)
raw.id = 1
notification = self.mox.CreateMockAnything()
notification.instance = INSTANCE_ID_1
notification.launched_at = None
raw = self.mox.CreateMockAnything()
raw.id = '1'
self.setup_mock_log()
self.log.warn('Ignoring exists without launched_at. RawData(1)')
self.mox.ReplayAll()
views._process_exists(raw, notif[1])
views._process_exists(raw, notification)
self.mox.VerifyAll()
def test_process_exists_with_deleted_at(self):
notification = self.mox.CreateMockAnything()
current_time = datetime.datetime.utcnow()
launch_time = current_time - datetime.timedelta(hours=23)
launch_decimal = utils.decimal_utc(launch_time)
deleted_time = current_time - datetime.timedelta(hours=12)
deleted_decimal = utils.decimal_utc(deleted_time)
current_decimal = utils.decimal_utc(current_time)
delete_time = datetime.datetime.utcnow()
deleted_decimal = utils.decimal_utc(delete_time)
audit_beginning = current_time - datetime.timedelta(hours=20)
audit_beginning_decimal = utils.decimal_utc(audit_beginning)
audit_ending_decimal = utils.decimal_utc(current_time)
notif = utils.create_nova_notif(launched=str(launch_time),
deleted=str(deleted_time),
audit_period_beginning=str(audit_beginning),
audit_period_ending=str(current_time),
tenant_id=TENANT_ID_1,
os_architecture=OS_ARCH_1,
os_version=OS_VERSION_1,
os_distro=OS_DISTRO_1,
rax_options=RAX_OPTIONS_1)
json_str = json.dumps(notif)
event = 'compute.instance.exists'
raw = utils.create_raw(self.mox, current_decimal, event=event,
json_str=json_str)
notification.launched_at = str(launch_time)
notification.audit_period_beginning = str(audit_beginning)
notification.audit_period_ending = str(current_time)
notification.tenant = TENANT_ID_1
notification.os_architecture = OS_ARCH_1
notification.os_version = OS_VERSION_1
notification.os_distro = OS_DISTRO_1
notification.rax_options = RAX_OPTIONS_1
notification.instance = INSTANCE_ID_1
notification.instance_type_id = INSTANCE_TYPE_ID_1
notification.message_id = MESSAGE_ID_1
notification.deleted_at = str(delete_time)
raw = self.mox.CreateMockAnything()
usage = self.mox.CreateMockAnything()
launched_range = (launch_decimal, launch_decimal+1)
views.STACKDB.get_instance_usage(instance=INSTANCE_ID_1,
@ -701,7 +812,7 @@ class StacktachUsageParsingTestCase(unittest.TestCase):
'deleted_at': deleted_decimal,
'audit_period_beginning': audit_beginning_decimal,
'audit_period_ending': audit_ending_decimal,
'instance_type_id': '1',
'instance_type_id': INSTANCE_TYPE_ID_1,
'usage': usage,
'delete': delete,
'raw': raw,
@ -715,6 +826,41 @@ class StacktachUsageParsingTestCase(unittest.TestCase):
views.STACKDB.create_instance_exists(**exists_values).AndReturn(exists)
views.STACKDB.save(exists)
self.mox.ReplayAll()
views._process_exists(raw, notif[1])
views._process_exists(raw, notification)
self.mox.VerifyAll()
class StacktachImageUsageParsingTestCase(unittest.TestCase):
def setUp(self):
self.mox = mox.Mox()
views.STACKDB = self.mox.CreateMockAnything()
def tearDown(self):
self.mox.UnsetStubs()
def test_save_image_usage(self):
raw = self.mox.CreateMockAnything()
notification = self.mox.CreateMockAnything()
notification.save_usage(raw)
self.mox.ReplayAll()
views._process_glance_usage(raw, notification)
self.mox.VerifyAll()
def test_save_image_delete(self):
raw = self.mox.CreateMockAnything()
notification = self.mox.CreateMockAnything()
notification.save_delete(raw)
self.mox.ReplayAll()
views._process_glance_delete(raw, notification)
self.mox.VerifyAll()
def test_save_image_exists(self):
raw = self.mox.CreateMockAnything()
notification = self.mox.CreateMockAnything()
notification.save_exists(raw)
self.mox.ReplayAll()
views._process_glance_exists(raw, notification)
self.mox.VerifyAll()

242
tests/unit/test_stacky_server.py
View File

@ -40,6 +40,12 @@ class StackyServerTestCase(unittest.TestCase):
self.mox.StubOutWithMock(models, 'RawData', use_mock_anything=True)
models.RawData.objects = self.mox.CreateMockAnything()
self.mox.StubOutWithMock(models, 'Deployment', use_mock_anything=True)
self.mox.StubOutWithMock(models, 'GlanceRawData',
use_mock_anything=True)
models.GlanceRawData.objects = self.mox.CreateMockAnything()
self.mox.StubOutWithMock(models, 'GenericRawData',
use_mock_anything=True)
models.GenericRawData.objects = self.mox.CreateMockAnything()
models.Deployment.objects = self.mox.CreateMockAnything()
self.mox.StubOutWithMock(models, 'Lifecycle', use_mock_anything=True)
models.Lifecycle.objects = self.mox.CreateMockAnything()
@ -65,7 +71,8 @@ class StackyServerTestCase(unittest.TestCase):
def _create_raw(self):
raw = self.mox.CreateMockAnything()
raw.when = utils.decimal_utc()
raw.when = utils.decimal_utc(datetime.datetime(2013, 7, 17, 10, 16,
10, 717219))
raw.instance = INSTANCE_ID_1
raw.id = 1
raw.routing_key = 'monitor.info'
@ -80,13 +87,19 @@ class StackyServerTestCase(unittest.TestCase):
raw.publisher = "api.example.com"
raw.service = 'api'
raw.host = 'example.com'
raw.status = 'state'
raw.request_id = REQUEST_ID_1
raw.json = '{"key": "value"}'
raw.uuid = 'uuid'
raw.tenant = 'tenant'
return raw
def test_get_event_names(self):
model = self.mox.CreateMockAnything()
result = self.mox.CreateMockAnything()
models.RawData.objects.values('event').AndReturn(result)
self.mox.StubOutWithMock(stacky_server, '_model_factory')
stacky_server._model_factory('nova').AndReturn(model)
model.values('event').AndReturn(result)
result.distinct().AndReturn(result)
self.mox.ReplayAll()
@ -248,6 +261,10 @@ class StackyServerTestCase(unittest.TestCase):
self.mox.VerifyAll()
def test_do_uuid(self):
search_result = [["#", "?", "When", "Deployment", "Event", "Host",
"State", "State'", "Task'"], [1, " ",
"2013-07-17 10:16:10.717219", "deployment",
"test.start", "example.com", "active", None, None]]
fake_request = self.mox.CreateMockAnything()
fake_request.GET = {'uuid': INSTANCE_ID_1}
result = self.mox.CreateMockAnything()
@ -256,6 +273,7 @@ class StackyServerTestCase(unittest.TestCase):
result.order_by('when').AndReturn(result)
raw = self._create_raw()
result.__iter__().AndReturn([raw].__iter__())
raw.search_results({}, mox.IgnoreArg(), ' ').AndReturn(search_result)
self.mox.ReplayAll()
resp = stacky_server.do_uuid(fake_request)
@ -272,6 +290,36 @@ class StackyServerTestCase(unittest.TestCase):
self.assertEqual(json_resp[1], body)
self.mox.VerifyAll()
def test_do_uuid_for_glance(self):
search_result = [["#", "?", "When", "Deployment", "Event", "Host",
"Status"], [1, " ",
"2013-07-17 10:16:10.717219", "deployment",
"test.start", "example.com", "state"]]
fake_request = self.mox.CreateMockAnything()
fake_request.GET = {'uuid': INSTANCE_ID_1}
result = self.mox.CreateMockAnything()
models.GlanceRawData.objects.select_related().AndReturn(result)
result.filter(uuid=INSTANCE_ID_1).AndReturn(result)
result.order_by('when').AndReturn(result)
raw = self._create_raw()
result.__iter__().AndReturn([raw].__iter__())
raw.search_results({}, mox.IgnoreArg(), ' ').AndReturn(search_result)
self.mox.ReplayAll()
resp = stacky_server.do_uuid(fake_request,'glance')
self.assertEqual(resp.status_code, 200)
json_resp = json.loads(resp.content)
self.assertEqual(len(json_resp), 2)
header = ["#", "?", "When", "Deployment", "Event", "Host",
"Status"]
self.assertEqual(json_resp[0], header)
datetime = dt.dt_from_decimal(raw.when)
body = [1, " ", str(datetime), "deployment", "test.start",
"example.com", "state"]
self.assertEqual(json_resp[1], body)
self.mox.VerifyAll()
def test_do_uuid_bad_uuid(self):
fake_request = self.mox.CreateMockAnything()
fake_request.GET = {'uuid': "obviouslybaduuid"}
@ -504,7 +552,9 @@ class StackyServerTestCase(unittest.TestCase):
self.assertEqual(resp_json[1], ['Bad Request', msg])
self.mox.VerifyAll()
def _assert_on_show(self, values, raw):
def _assert_on_show_nova(self, json_resp, raw):
self.assertEqual(len(json_resp), 3)
values = json_resp[0]
self.assertEqual(len(values), 12)
self.assertEqual(values[0], ["Key", "Value"])
self.assertEqual(values[1], ["#", raw.id])
@ -517,7 +567,25 @@ class StackyServerTestCase(unittest.TestCase):
self.assertEqual(values[7], ["Event", raw.event])
self.assertEqual(values[8], ["Service", raw.service])
self.assertEqual(values[9], ["Host", raw.host])
self.assertEqual(values[10], ["UUID", raw.instance])
self.assertEqual(values[10],["UUID", raw.instance])
self.assertEqual(values[11], ["Req ID", raw.request_id])
def _assert_on_show_glance(self, json_resp, raw):
self.assertEqual(len(json_resp), 3)
values = json_resp[0]
self.assertEqual(len(values), 12)
self.assertEqual(values[0], ["Key", "Value"])
self.assertEqual(values[1], ["#", raw.id])
self.assertEqual(values[2], ["When",
str(dt.dt_from_decimal(raw.when))])
self.assertEqual(values[3], ["Deployment", raw.deployment.name])
self.assertEqual(values[4], ["Category", raw.routing_key])
self.assertEqual(values[5], ["Publisher", raw.publisher])
self.assertEqual(values[6], ["Status", raw.status])
self.assertEqual(values[7], ["Event", raw.event])
self.assertEqual(values[8], ["Service", raw.service])
self.assertEqual(values[9], ["Host", raw.host])
self.assertEqual(values[10],["UUID", raw.uuid])
self.assertEqual(values[11], ["Req ID", raw.request_id])
def test_do_show(self):
@ -528,10 +596,81 @@ class StackyServerTestCase(unittest.TestCase):
resp = stacky_server.do_show(fake_request, 1)
self.assertEqual(resp.status_code, 200)
json_resp = json.loads(resp.content)
self._assert_on_show_nova(json_resp, raw)
self.mox.VerifyAll()
def test_do_show_for_glance_rawdata(self):
fake_request = self.mox.CreateMockAnything()
raw = self._create_raw()
models.GlanceRawData.objects.get(id=1).AndReturn(raw)
self.mox.ReplayAll()
resp = stacky_server.do_show(fake_request, 1, 'glance')
self.assertEqual(resp.status_code, 200)
json_resp = json.loads(resp.content)
self._assert_on_show_glance(json_resp, raw)
self.mox.VerifyAll()
def test_do_show_for_generic_rawdata(self):
fake_request = self.mox.CreateMockAnything()
raw = self._create_raw()
models.GenericRawData.objects.get(id=1).AndReturn(raw)
self.mox.ReplayAll()
resp = stacky_server.do_show(fake_request, 1, 'generic')
self.assertEqual(resp.status_code, 200)
json_resp = json.loads(resp.content)
self._assert_on_show_nova(json_resp, raw)
self.mox.VerifyAll()
def test_do_show_should_return_empty_result_on_object_not_found_exception(self):
fake_request = self.mox.CreateMockAnything()
raw = self._create_raw()
models.RawData.objects.get(id=1).AndReturn(raw)
self.mox.ReplayAll()
resp = stacky_server.do_show(fake_request, 1)
self.assertEqual(resp.status_code, 200)
json_resp = json.loads(resp.content)
self._assert_on_show_nova(json_resp, raw)
self.mox.VerifyAll()
def test_do_watch_for_glance(self):
fake_request = self.mox.CreateMockAnything()
fake_request.GET = {}
self.mox.StubOutWithMock(stacky_server, 'get_deployments')
deployment1 = self.mox.CreateMockAnything()
deployment1.id = 1
deployment1.name = 'dep1'
deployments = [deployment1]
stacky_server.get_deployments().AndReturn(deployments)
self.mox.StubOutWithMock(stacky_server, 'get_event_names')
events = [{'event': 'test.start'}, {'event': 'test.end'}]
stacky_server.get_event_names().AndReturn(events)
results = self.mox.CreateMockAnything()
models.GlanceRawData.objects.order_by('when').AndReturn(results)
results.filter(when__gt=mox.IgnoreArg()).AndReturn(results)
results.filter(when__lte=mox.IgnoreArg()).AndReturn(results)
results.__iter__().AndReturn([self._create_raw()].__iter__())
self.mox.ReplayAll()
resp = stacky_server.do_watch(fake_request, 0, 'glance')
self.assertEqual(resp.status_code, 200)
json_resp = json.loads(resp.content)
self.assertEqual(len(json_resp), 3)
self._assert_on_show(json_resp[0], raw)
self.assertEqual(json_resp[0], [10, 1, 15, 20, 10, 36])
self.assertEqual(json_resp[1][0][0], 1)
self.assertEqual(json_resp[1][0][1], u' ')
time_str = "%s %s" % (json_resp[1][0][2], json_resp[1][0][3])
datetime.datetime.strptime(time_str, "%Y-%m-%d %H:%M:%S.%f")
self.assertEqual(json_resp[1][0][4], u'dep1')
self.assertEqual(json_resp[1][0][5], u'test.start')
self.assertEqual(json_resp[1][0][6], u'%s' % INSTANCE_ID_1)
self.mox.VerifyAll()
def test_do_watch(self):
@ -547,7 +686,10 @@ class StackyServerTestCase(unittest.TestCase):
events = [{'event': 'test.start'}, {'event': 'test.end'}]
stacky_server.get_event_names().AndReturn(events)
results = self.mox.CreateMockAnything()
models.RawData.objects.order_by('when').AndReturn(results)
model = self.mox.CreateMockAnything()
self.mox.StubOutWithMock(stacky_server, '_model_factory')
stacky_server._model_factory('nova').AndReturn(model)
model.order_by('when').AndReturn(results)
results.filter(when__gt=mox.IgnoreArg()).AndReturn(results)
results.filter(when__lte=mox.IgnoreArg()).AndReturn(results)
results.__iter__().AndReturn([self._create_raw()].__iter__())
@ -558,7 +700,6 @@ class StackyServerTestCase(unittest.TestCase):
json_resp = json.loads(resp.content)
self.assertEqual(len(json_resp), 3)
self.assertEqual(json_resp[0], [10, 1, 15, 20, 10, 36])
print json_resp
self.assertEqual(json_resp[1][0][0], 1)
self.assertEqual(json_resp[1][0][1], u' ')
time_str = "%s %s" % (json_resp[1][0][2], json_resp[1][0][3])
@ -581,7 +722,11 @@ class StackyServerTestCase(unittest.TestCase):
events = [{'event': 'test.start'}, {'event': 'test.end'}]
stacky_server.get_event_names().AndReturn(events)
results = self.mox.CreateMockAnything()
models.RawData.objects.order_by('when').AndReturn(results)
model = self.mox.CreateMockAnything()
self.mox.StubOutWithMock(stacky_server, '_model_factory')
stacky_server._model_factory('nova').AndReturn(model)
model.order_by('when').AndReturn(results)
results.filter(deployment=1).AndReturn(results)
results.filter(when__gt=mox.IgnoreArg()).AndReturn(results)
results.filter(when__lte=mox.IgnoreArg()).AndReturn(results)
@ -593,7 +738,6 @@ class StackyServerTestCase(unittest.TestCase):
json_resp = json.loads(resp.content)
self.assertEqual(len(json_resp), 3)
self.assertEqual(json_resp[0], [10, 1, 15, 20, 10, 36])
print json_resp
self.assertEqual(json_resp[1][0][0], 1)
self.assertEqual(json_resp[1][0][1], u' ')
time_str = "%s %s" % (json_resp[1][0][2], json_resp[1][0][3])
@ -628,7 +772,6 @@ class StackyServerTestCase(unittest.TestCase):
json_resp = json.loads(resp.content)
self.assertEqual(len(json_resp), 3)
self.assertEqual(json_resp[0], [10, 1, 15, 20, 10, 36])
print json_resp
self.assertEqual(json_resp[1][0][0], 1)
self.assertEqual(json_resp[1][0][1], u' ')
time_str = "%s %s" % (json_resp[1][0][2], json_resp[1][0][3])
@ -960,3 +1103,82 @@ class StackyServerTestCase(unittest.TestCase):
self.assertEqual(resp_json[1], ['Bad Request', msg])
self.mox.VerifyAll()
def test_model_factory_for_nova(self):
self.mox.UnsetStubs()
nova_model = stacky_server._model_factory('nova')
self.assertEqual(nova_model.model, models.RawData)
def test_model_factory_for_nova(self):
self.mox.UnsetStubs()
nova_model = stacky_server._model_factory('glance')
self.assertEqual(nova_model.model, models.GlanceRawData)
def test_model_factory_for_nova(self):
self.mox.UnsetStubs()
nova_model = stacky_server._model_factory('generic')
self.assertEqual(nova_model.model, models.GenericRawData)
def _assert_on_search_nova(self, json_resp, raw):
title = json_resp[0]
values = json_resp[1]
self.assertEqual(len(values), 9)
self.assertEqual([title[0], values[0]],["#", raw.id] )
self.assertEqual([title[1], values[1]], ['?', ' '])
self.assertEqual([title[2], values[2]], ["When",
str(dt.dt_from_decimal(raw.when))])
self.assertEqual([title[3], values[3]], ["Deployment", raw.deployment.name])
self.assertEqual([title[4], values[4]], ["Event", raw.event])
self.assertEqual([title[5], values[5]], ["Host", raw.host])
self.assertEqual([title[6], values[6]], ["State", raw.state])
self.assertEqual([title[7], values[7]], ["State'", raw.old_state])
def test_search_by_field_for_nova(self):
search_result = [["#", "?", "When", "Deployment", "Event", "Host",
"State", "State'", "Task'"], [1, " ",
"2013-07-17 10:16:10.717219", "deployment",
"test.start", "example.com", "active", None, None]]
fake_request = self.mox.CreateMockAnything()
fake_request.GET = {'field': 'tenant', 'value': 'tenant'}
raw = self._create_raw()
models.RawData.objects.filter(tenant='tenant').AndReturn([raw])
raw.search_results({}, mox.IgnoreArg(), ' ').AndReturn(search_result)
self.mox.ReplayAll()
resp = stacky_server.search(fake_request, 'nova')
self.assertEqual(resp.status_code, 200)
json_resp = json.loads(resp.content)
self._assert_on_search_nova(json_resp, raw)
self.mox.VerifyAll()
def test_search_by_field_for_nova_with_limit(self):
search_result = [["#", "?", "When", "Deployment", "Event", "Host",
"State", "State'", "Task'"], [1, " ",
"2013-07-17 10:16:10.717219", "deployment",
"test.start", "example.com", "active", None, None]]
search_result_2 = [["#", "?", "When", "Deployment", "Event", "Host",
"State", "State'", "Task'"], [1, " ",
"2013-07-17 10:16:10.717219", "deployment",
"test.start", "example.com", "active", None, None],[2, " ",
"2013-07-17 10:16:10.717219", "deployment",
"test.start", "example.com", "active", None, None]]
fake_request = self.mox.CreateMockAnything()
fake_request.GET = {'field': 'tenant', 'value': 'tenant', 'limit': '2'}
raw1 = self._create_raw()
raw2 = self._create_raw()
raw3 = self._create_raw()
raw2.id = 2
raw3.id = 3
models.RawData.objects.filter(tenant='tenant').AndReturn([raw1, raw2,
raw3])
raw1.search_results({}, mox.IgnoreArg(), ' ').AndReturn(search_result)
raw2.search_results(search_result, mox.IgnoreArg(),' ').AndReturn(search_result_2)
self.mox.ReplayAll()
resp = stacky_server.search(fake_request, 'nova')
self.assertEqual(resp.status_code, 200)
json_resp = json.loads(resp.content)
self.assertEqual(len(json_resp), 3)
self._assert_on_search_nova(json_resp, raw1)
self.mox.VerifyAll()

98
tests/unit/test_worker.py
View File

@ -22,15 +22,14 @@ import json
import unittest
import kombu
import kombu.entity
import kombu.connection
import mox
from stacktach import db, views
from stacktach import db
from stacktach import views
import worker.worker as worker
class NovaConsumerTestCase(unittest.TestCase):
class ConsumerTestCase(unittest.TestCase):
def setUp(self):
self.mox = mox.Mox()
@ -47,9 +46,10 @@ class NovaConsumerTestCase(unittest.TestCase):
consumer = self.mox.CreateMockAnything()
created_consumers.append(consumer)
return consumer
self.mox.StubOutWithMock(worker.NovaConsumer, '_create_exchange')
self.mox.StubOutWithMock(worker.NovaConsumer, '_create_queue')
consumer = worker.NovaConsumer('test', None, None, True, {})
self.mox.StubOutWithMock(worker.Consumer, '_create_exchange')
self.mox.StubOutWithMock(worker.Consumer, '_create_queue')
consumer = worker.Consumer('test', None, None, True, {}, "nova",
["monitor.info", "monitor.error"])
exchange = self.mox.CreateMockAnything()
consumer._create_exchange('nova', 'topic').AndReturn(exchange)
info_queue = self.mox.CreateMockAnything()
@ -71,7 +71,8 @@ class NovaConsumerTestCase(unittest.TestCase):
def test_create_exchange(self):
args = {'key': 'value'}
consumer = worker.NovaConsumer('test', None, None, True, args)
consumer = worker.Consumer('test', None, None, True, args, 'nova',
["monitor.info", "monitor.error"])
self.mox.StubOutClassWithMocks(kombu.entity, 'Exchange')
exchange = kombu.entity.Exchange('nova', type='topic', exclusive=False,
@ -87,7 +88,8 @@ class NovaConsumerTestCase(unittest.TestCase):
queue = kombu.Queue('name', exchange, auto_delete=False, durable=True,
exclusive=False, routing_key='routing.key',
queue_arguments={})
consumer = worker.NovaConsumer('test', None, None, True, {})
consumer = worker.Consumer('test', None, None, True, {}, 'nova',
["monitor.info", "monitor.error"])
self.mox.ReplayAll()
actual_queue = consumer._create_queue('name', exchange, 'routing.key',
exclusive=False,
@ -103,7 +105,8 @@ class NovaConsumerTestCase(unittest.TestCase):
queue = kombu.Queue('name', exchange, auto_delete=False, durable=True,
exclusive=False, routing_key='routing.key',
queue_arguments=queue_args)
consumer = worker.NovaConsumer('test', None, None, True, queue_args)
consumer = worker.Consumer('test', None, None, True, queue_args,
'nova', ["monitor.info", "monitor.error"])
self.mox.ReplayAll()
actual_queue = consumer._create_queue('name', exchange, 'routing.key',
exclusive=False,
@ -114,21 +117,30 @@ class NovaConsumerTestCase(unittest.TestCase):
def test_process(self):
deployment = self.mox.CreateMockAnything()
raw = self.mox.CreateMockAnything()
raw.get_name().AndReturn('RawData')
message = self.mox.CreateMockAnything()
consumer = worker.NovaConsumer('test', None, deployment, True, {})
exchange = 'nova'
consumer = worker.Consumer('test', None, deployment, True, {},
exchange, ["monitor.info", "monitor.error"])
routing_key = 'monitor.info'
message.delivery_info = {'routing_key': routing_key}
body_dict = {u'key': u'value'}
message.body = json.dumps(body_dict)
mock_notification = self.mox.CreateMockAnything()
mock_post_process_method = self.mox.CreateMockAnything()
mock_post_process_method(raw, mock_notification)
old_handler = worker.POST_PROCESS_METHODS
worker.POST_PROCESS_METHODS["RawData"] = mock_post_process_method
self.mox.StubOutWithMock(views, 'process_raw_data',
use_mock_anything=True)
args = (routing_key, body_dict)
views.process_raw_data(deployment, args, json.dumps(args))\
.AndReturn(raw)
views.process_raw_data(deployment, args, json.dumps(args), exchange) \
.AndReturn((raw, mock_notification))
message.ack()
self.mox.StubOutWithMock(views, 'post_process')
views.post_process(raw, body_dict)
self.mox.StubOutWithMock(consumer, '_check_memory',
use_mock_anything=True)
consumer._check_memory()
@ -136,29 +148,7 @@ class NovaConsumerTestCase(unittest.TestCase):
consumer._process(message)
self.assertEqual(consumer.processed, 1)
self.mox.VerifyAll()
def test_process_no_raw_dont_ack(self):
deployment = self.mox.CreateMockAnything()
raw = self.mox.CreateMockAnything()
message = self.mox.CreateMockAnything()
consumer = worker.NovaConsumer('test', None, deployment, True, {})
routing_key = 'monitor.info'
message.delivery_info = {'routing_key': routing_key}
body_dict = {u'key': u'value'}
message.body = json.dumps(body_dict)
self.mox.StubOutWithMock(views, 'process_raw_data',
use_mock_anything=True)
args = (routing_key, body_dict)
views.process_raw_data(deployment, args, json.dumps(args))\
.AndReturn(None)
self.mox.StubOutWithMock(consumer, '_check_memory',
use_mock_anything=True)
consumer._check_memory()
self.mox.ReplayAll()
consumer._process(message)
self.assertEqual(consumer.processed, 0)
self.mox.VerifyAll()
worker.POST_PROCESS_METHODS["RawData"] = old_handler
def test_run(self):
config = {
@ -168,7 +158,9 @@ class NovaConsumerTestCase(unittest.TestCase):
'rabbit_port': 5672,
'rabbit_userid': 'rabbit',
'rabbit_password': 'rabbit',
'rabbit_virtual_host': '/'
'rabbit_virtual_host': '/',
"services": ["nova"],
"topics": {"nova": ["monitor.info", "monitor.error"]}
}
self.mox.StubOutWithMock(db, 'get_or_create_deployment')
deployment = self.mox.CreateMockAnything()
@ -187,13 +179,15 @@ class NovaConsumerTestCase(unittest.TestCase):
kombu.connection.BrokerConnection(**params).AndReturn(conn)
conn.__enter__().AndReturn(conn)
conn.__exit__(None, None, None).AndReturn(None)
self.mox.StubOutClassWithMocks(worker, 'NovaConsumer')
consumer = worker.NovaConsumer(config['name'], conn, deployment,
config['durable_queue'], {})
self.mox.StubOutClassWithMocks(worker, 'Consumer')
exchange = 'nova'
consumer = worker.Consumer(config['name'], conn, deployment,
config['durable_queue'], {}, exchange,
["monitor.info", "monitor.error"])
consumer.run()
worker.continue_running().AndReturn(False)
self.mox.ReplayAll()
worker.run(config)
worker.run(config, exchange)
self.mox.VerifyAll()
def test_run_queue_args(self):
@ -205,7 +199,9 @@ class NovaConsumerTestCase(unittest.TestCase):
'rabbit_userid': 'rabbit',
'rabbit_password': 'rabbit',
'rabbit_virtual_host': '/',
'queue_arguments': {'x-ha-policy': 'all'}
'queue_arguments': {'x-ha-policy': 'all'},
"services": ["nova"],
"topics": {"nova": ["monitor.info", "monitor.error"]}
}
self.mox.StubOutWithMock(db, 'get_or_create_deployment')
deployment = self.mox.CreateMockAnything()
@ -224,12 +220,14 @@ class NovaConsumerTestCase(unittest.TestCase):
kombu.connection.BrokerConnection(**params).AndReturn(conn)
conn.__enter__().AndReturn(conn)
conn.__exit__(None, None, None).AndReturn(None)
self.mox.StubOutClassWithMocks(worker, 'NovaConsumer')
consumer = worker.NovaConsumer(config['name'], conn, deployment,
config['durable_queue'],
config['queue_arguments'])
self.mox.StubOutClassWithMocks(worker, 'Consumer')
exchange = 'nova'
consumer = worker.Consumer(config['name'], conn, deployment,
config['durable_queue'],
config['queue_arguments'], exchange,
["monitor.info", "monitor.error"])
consumer.run()
worker.continue_running().AndReturn(False)
self.mox.ReplayAll()
worker.run(config)
self.mox.VerifyAll()
worker.run(config, exchange)
self.mox.VerifyAll()

4
tests/unit/utils.py
View File

@ -25,6 +25,8 @@ TENANT_ID_2 = 'testtenantid2'
from stacktach import datetime_to_decimal as dt
IMAGE_UUID_1 = "1"
INSTANCE_ID_1 = "08f685d9-6352-4dbc-8271-96cc54bf14cd"
INSTANCE_ID_2 = "515adf96-41d3-b86d-5467-e584edc61dab"
@ -32,6 +34,7 @@ INSTANCE_TYPE_ID_1 = "12345"
INSTANCE_TYPE_ID_2 = '54321'
DUMMY_TIME = datetime.datetime.utcnow()
DECIMAL_DUMMY_TIME = dt.dt_to_decimal(DUMMY_TIME)
MESSAGE_ID_1 = "7f28f81b-29a2-43f2-9ba1-ccb3e53ab6c8"
MESSAGE_ID_2 = "4d596126-0f04-4329-865f-7b9a7bd69bcf"
@ -52,6 +55,7 @@ OS_ARCH_2 = "x64"
OS_VERSION_1 = "1"
OS_VERSION_2 = "2"
TIMESTAMP_1 = "2013-06-20 17:31:57.939614"
def decimal_utc(t = datetime.datetime.utcnow()):
return dt.dt_to_decimal(t)

145
util/glance_usage_seed.py Normal file
View File

@ -0,0 +1,145 @@
# Copyright (c) 2013 - Rackspace Inc.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to
# deal in the Software without restriction, including without limitation the
# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
# sell copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.
"""
Usage: python glance_usage_seed.py [period_length] [sql_connection]
python glance_usage_seed.py hour mysql://user:password@nova-db.example
.com/nova?charset=utf8
The idea behind glance_usage seeding is to take the current state of all
active, deleted and pending_delete images from glance and insert that
data into Stacktach's image_usage and image_deletes tables.
"""
import __builtin__
setattr(__builtin__, '_', lambda x: x)
import datetime
import os
import sys
from oslo.config import cfg
CONF = cfg.CONF
if __name__ == '__main__':
if len(sys.argv) != 3:
print "Proper Usage: glance_usage_seed.py [period_length] [" \
"sql_connection]"
sys.exit(1)
CONF.sql_connection = sys.argv[2]
import glance.context
import glance.db.sqlalchemy.api as db_api
from sqlalchemy import or_
from sqlalchemy import and_
import glance.db.sqlalchemy.api as db_api
from glance.db.sqlalchemy import models as glancemodels
POSSIBLE_TOPDIR = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]),
os.pardir, os.pardir))
if os.path.exists(os.path.join(POSSIBLE_TOPDIR, 'stacktach')):
sys.path.insert(0, POSSIBLE_TOPDIR)
from stacktach import datetime_to_decimal as dt
from stacktach import models
# start yanked from reports/nova_usage_audit.py
def get_period_start(time, period_length):
if period_length == 'day':
last_period = time - datetime.timedelta(days=1)
start = datetime.datetime(year=last_period.year,
month=last_period.month,
day=last_period.day)
return start
elif period_length == 'hour':
last_period = time - datetime.timedelta(hours=1)
start = datetime.datetime(year=last_period.year,
month=last_period.month,
day=last_period.day,
hour=last_period.hour)
return start
# end yanked from reports/nova_usage_audit.py
def _usage_for_image(image):
return {
'uuid': image.id,
'owner': image.owner,
'created_at': dt.dt_to_decimal(image.created_at),
'owner': image.owner,
'size': image.size,
'last_raw_id': None
}
def _delete_for_image(image):
return {
'uuid': image.id,
'deleted_at': dt.dt_to_decimal(image.deleted_at),
'raw_id': None
}
def _get_usages(start, session):
usage_filter = (glancemodels.Image.status == 'active',
glancemodels.Image.deleted_at > start)
query = session.query(glancemodels.Image)
images = query.filter(or_(*usage_filter)).all()
return [_usage_for_image(image) for image in images]
def _get_deletes(start, session):
delete_filter = (glancemodels.Image.status == 'deleted',
glancemodels.Image.deleted_at > start)
query = session.query(glancemodels.Image)
images = query.filter(and_(*delete_filter)).all()
return [_delete_for_image(image) for image in images]
def seed(period_length):
start = get_period_start(datetime.datetime.utcnow(), period_length)
db_api.configure_db()
session = db_api.get_session()
print "Populating active image usages"
usages = _get_usages(start, session)
if usages:
print "Saving active image images"
active_images = map(lambda x: models.ImageUsage(**x), usages)
models.ImageUsage.objects.bulk_create(active_images, batch_size=100)
print "Populating image deletes"
deletes = _get_deletes(start, session)
if deletes:
print "Saving image deletes"
deleted_images = map(lambda x: models.ImageDeletes(**x), deletes)
models.ImageDeletes.objects.bulk_create(deleted_images, batch_size=100)
print "Seeding completed"
return len(usages), len(deletes)
if __name__ == '__main__':
msg = ("Seeded system with: \n"
"%s Active images \n"
"%s Deleted images \n")
period = sys.argv[1]
print msg % seed(period)

43
worker/config.py Normal file
View File

@ -0,0 +1,43 @@
# Copyright (c) 2013 - Rackspace Inc.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to
# deal in the Software without restriction, including without limitation the
# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
# sell copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.
import json
import os
config_filename = os.environ.get('STACKTACH_DEPLOYMENTS_FILE',
'stacktach_worker_config.json')
try:
from local_settings import *
config_filename = STACKTACH_DEPLOYMENTS_FILE
except ImportError:
pass
config = None
with open(config_filename, "r") as f:
config = json.load(f)
def deployments():
return config['deployments']
def topics():
return config['topics']

27
worker/start_workers.py
View File

@ -1,4 +1,3 @@
import json
import os
import signal
import sys
@ -11,14 +10,7 @@ if os.path.exists(os.path.join(POSSIBLE_TOPDIR, 'stacktach')):
sys.path.insert(0, POSSIBLE_TOPDIR)
import worker.worker as worker
config_filename = os.environ.get('STACKTACH_DEPLOYMENTS_FILE',
'stacktach_worker_config.json')
try:
from local_settings import *
config_filename = STACKTACH_DEPLOYMENTS_FILE
except ImportError:
pass
from worker import config
processes = []
@ -35,18 +27,15 @@ def kill_time(signal, frame):
if __name__ == '__main__':
config = None
with open(config_filename, "r") as f:
config = json.load(f)
deployments = config['deployments']
for deployment in deployments:
for deployment in config.deployments():
if deployment.get('enabled', True):
process = Process(target=worker.run, args=(deployment,))
process.daemon = True
process.start()
processes.append(process)
for exchange in deployment.get('topics').keys():
process = Process(target=worker.run, args=(deployment,
exchange,))
process.daemon = True
process.start()
processes.append(process)
signal.signal(signal.SIGINT, kill_time)
signal.signal(signal.SIGTERM, kill_time)
signal.pause()

72
worker/worker.py
View File

@ -17,12 +17,13 @@
# to set TENANT_ID and URL to point to your StackTach web server.
import datetime
import kombu
import kombu.entity
import kombu.mixins
import sys
import time
import kombu
import kombu.mixins
try:
import ujson as json
except ImportError:
@ -41,8 +42,9 @@ stacklog.set_default_logger_name('worker')
LOG = stacklog.get_logger()
class NovaConsumer(kombu.mixins.ConsumerMixin):
def __init__(self, name, connection, deployment, durable, queue_arguments):
class Consumer(kombu.mixins.ConsumerMixin):
def __init__(self, name, connection, deployment, durable, queue_arguments,
exchange, topics):
self.connection = connection
self.deployment = deployment
self.durable = durable
@ -52,6 +54,8 @@ class NovaConsumer(kombu.mixins.ConsumerMixin):
self.pmi = None
self.processed = 0
self.total_processed = 0
self.topics = topics
self.exchange = exchange
def _create_exchange(self, name, type, exclusive=False, auto_delete=False):
return kombu.entity.Exchange(name, type=type, exclusive=exclusive,
@ -66,14 +70,12 @@ class NovaConsumer(kombu.mixins.ConsumerMixin):
routing_key=routing_key)
def get_consumers(self, Consumer, channel):
nova_exchange = self._create_exchange("nova", "topic")
exchange = self._create_exchange(self.exchange, "topic")
nova_queues = [
self._create_queue('monitor.info', nova_exchange, 'monitor.info'),
self._create_queue('monitor.error', nova_exchange, 'monitor.error')
]
queues = [self._create_queue(topic, exchange, topic)
for topic in self.topics]
return [Consumer(queues=nova_queues, callbacks=[self.on_nova])]
return [Consumer(queues=queues, callbacks=[self.on_nova])]
def _process(self, message):
routing_key = message.delivery_info['routing_key']
@ -81,14 +83,13 @@ class NovaConsumer(kombu.mixins.ConsumerMixin):
body = str(message.body)
args = (routing_key, json.loads(body))
asJson = json.dumps(args)
# save raw and ack the message
raw = views.process_raw_data(self.deployment, args, asJson)
raw, notif = views.process_raw_data(
self.deployment, args, asJson, self.exchange)
if raw:
self.processed += 1
message.ack()
views.post_process(raw, args[1])
self.processed += 1
message.ack()
POST_PROCESS_METHODS[raw.get_name()](raw, notif)
self._check_memory()
@ -113,9 +114,9 @@ class NovaConsumer(kombu.mixins.ConsumerMixin):
per_message = 0
if self.total_processed:
per_message = idiff / self.total_processed
LOG.debug("%20s %6dk/%6dk ram, "
LOG.debug("%20s %20s %6dk/%6dk ram, "
"%3d/%4d msgs @ %6dk/msg" %
(self.name, diff, idiff, self.processed,
(self.name, self.exchange, diff, idiff, self.processed,
self.total_processed, per_message))
self.last_vsz = self.pmi.vsz
self.processed = 0
@ -140,7 +141,7 @@ def exit_or_sleep(exit=False):
time.sleep(5)
def run(deployment_config):
def run(deployment_config, exchange):
name = deployment_config['name']
host = deployment_config.get('rabbit_host', 'localhost')
port = deployment_config.get('rabbit_port', 5672)
@ -150,11 +151,13 @@ def run(deployment_config):
durable = deployment_config.get('durable_queue', True)
queue_arguments = deployment_config.get('queue_arguments', {})
exit_on_exception = deployment_config.get('exit_on_exception', False)
topics = deployment_config.get('topics', {})
deployment, new = db.get_or_create_deployment(name)
print "Starting worker for '%s'" % name
LOG.info("%s: %s %s %s %s" % (name, host, port, user_id, virtual_host))
print "Starting worker for '%s %s'" % (name, exchange)
LOG.info("%s: %s %s %s %s %s" % (name, exchange, host, port, user_id,
virtual_host))
params = dict(hostname=host,
port=port,
@ -166,21 +169,30 @@ def run(deployment_config):
# continue_running() is used for testing
while continue_running():
try:
LOG.debug("Processing on '%s'" % name)
LOG.debug("Processing on '%s %s'" % (name, exchange))
with kombu.connection.BrokerConnection(**params) as conn:
try:
consumer = NovaConsumer(name, conn, deployment, durable,
queue_arguments)
consumer = Consumer(name, conn, deployment, durable,
queue_arguments, exchange,
topics[exchange])
consumer.run()
except Exception as e:
LOG.error("!!!!Exception!!!!")
LOG.exception("name=%s, exception=%s. Reconnecting in 5s" %
(name, e))
LOG.exception("name=%s, exchange=%s, exception=%s. "
"Reconnecting in 5s" %
(name, exchange, e))
exit_or_sleep(exit_on_exception)
LOG.debug("Completed processing on '%s'" % name)
LOG.debug("Completed processing on '%s %s'" % (name, exchange))
except:
LOG.error("!!!!Exception!!!!")
e = sys.exc_info()[0]
msg = "Uncaught exception: deployment=%s, exception=%s. Retrying in 5s"
LOG.exception(msg % (name, e))
msg = "Uncaught exception: deployment=%s, exchange=%s, " \
"exception=%s. Retrying in 5s"
LOG.exception(msg % (name, exchange, e))
exit_or_sleep(exit_on_exception)
POST_PROCESS_METHODS = {
'RawData': views.post_process_rawdata,
'GlanceRawData': views.post_process_glancerawdata,
'GenericRawData': views.post_process_genericrawdata
}