- Moved config.py to worker

- Removed exchange from config and started reading them from topics
- Fixed lengths of GlanceRawData fields to match Glance fields
- Made uuid and status accept null values as the image.send notification does not contain those values

stacktach/config.py

@@ -0,0 +1,43 @@
+# Copyright (c) 2013 - Rackspace Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to
+# deal in the Software without restriction, including without limitation the
+# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+# sell copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+import json
+import os
+
+config_filename = os.environ.get('STACKTACH_DEPLOYMENTS_FILE',
+                                 'stacktach_worker_config.json')
+try:
+    from local_settings import *
+    config_filename = STACKTACH_DEPLOYMENTS_FILE
+except ImportError:
+    pass
+
+config = None
+with open(config_filename, "r") as f:
+    config = json.load(f)
+
+
+def deployments():
+    return config['deployments']
+
+
+def topics():
+    return config['topics']
+
+

stacktach/migrations/0003_populate_usage_related_fields_in_rawdataimagemeta_instanceexists_and_instanceusages_from_rawdata.py

@@ -42,7 +42,8 @@ class Migration(DataMigration):
         json_dict = json.loads(json_message)
         routing_key = json_dict[0]
         body = json_dict[1]
-        return notification_factory(body, None, routing_key, json_message, None)
+        return notification_factory(body, None, routing_key, json_message,
+                                    'nova')
 
     def forwards(self, orm):
         # Note: Don't use "from appname.models import ModelName".

stacktach/migrations/0005_create_glancerawdata_and_genericrawdata.py

@@ -12,7 +12,7 @@ class Migration(SchemaMigration):
         db.create_table(u'stacktach_glancerawdata', (
             (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
             ('deployment', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['stacktach.Deployment'])),
-            ('owner', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=50, null=True, blank=True)),
+            ('owner', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=255, null=True, blank=True)),
             ('json', self.gf('django.db.models.fields.TextField')()),
             ('routing_key', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=50, null=True, blank=True)),
             ('when', self.gf('django.db.models.fields.DecimalField')(max_digits=20, decimal_places=6, db_index=True)),
@@ -22,8 +22,8 @@ class Migration(SchemaMigration):
             ('host', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=100, null=True, blank=True)),
             ('instance', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=50, null=True, blank=True)),
             ('request_id', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=50, null=True, blank=True)),
-            ('uuid', self.gf('django.db.models.fields.CharField')(max_length=50)),
+            ('uuid', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=36, null=True, blank=True)),
-            ('status', self.gf('django.db.models.fields.CharField')(default='queued', max_length=50, db_index=True)),
+            ('status', self.gf('django.db.models.fields.CharField')(max_length=30, null=True, db_index=True)),
             ('image_type', self.gf('django.db.models.fields.IntegerField')(default=0, null=True, db_index=True)),
         ))
         db.send_create_signal(u'stacktach', ['GlanceRawData'])
@@ -86,13 +86,13 @@ class Migration(SchemaMigration):
             'image_type': ('django.db.models.fields.IntegerField', [], {'default': '0', 'null': 'True', 'db_index': 'True'}),
             'instance': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
             'json': ('django.db.models.fields.TextField', [], {}),
-            'owner': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
+            'owner': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '255', 'null': 'True', 'blank': 'True'}),
             'publisher': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '100', 'null': 'True', 'blank': 'True'}),
             'request_id': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
             'routing_key': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
             'service': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
-            'status': ('django.db.models.fields.CharField', [], {'default': "'queued'", 'max_length': '50', 'db_index': 'True'}),
+            'status': ('django.db.models.fields.CharField', [], {'max_length': '30', 'null': 'True', 'db_index': 'True'}),
-            'uuid': ('django.db.models.fields.CharField', [], {'max_length': '50'}),
+            'uuid': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '36', 'null': 'True', 'blank': 'True'}),
             'when': ('django.db.models.fields.DecimalField', [], {'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'})
         },
         u'stacktach.instancedeletes': {
@@ -125,6 +125,17 @@ class Migration(SchemaMigration):
             'tenant': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
             'usage': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'null': 'True', 'to': u"orm['stacktach.InstanceUsage']"})
         },
+        u'stacktach.instancereconcile': {
+            'Meta': {'object_name': 'InstanceReconcile'},
+            'deleted_at': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'instance': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
+            'instance_type_id': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}),
+            'launched_at': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}),
+            'row_created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'row_updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+            'source': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '150', 'null': 'True', 'blank': 'True'})
+        },
         u'stacktach.instanceusage': {
             'Meta': {'object_name': 'InstanceUsage'},
             u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),

stacktach/models.py

@@ -1,3 +1,18 @@
+# Copyright 2012 - Dark Secret Software Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
 from django import forms
 from django.db import models
 
@@ -273,8 +288,8 @@ class GlanceRawData(models.Model):
     ]
 
     deployment = models.ForeignKey(Deployment)
-    owner = models.CharField(max_length=50, null=True, blank=True,
+    owner = models.CharField(max_length=255, null=True, blank=True,
-                              db_index=True)
+                             db_index=True)
     json = models.TextField()
     routing_key = models.CharField(max_length=50, null=True, blank=True,
                                    db_index=True)
@@ -291,9 +306,10 @@ class GlanceRawData(models.Model):
                                 db_index=True)
     request_id = models.CharField(max_length=50, null=True, blank=True,
                                   db_index=True)
-    uuid = models.CharField(max_length=50)
+    uuid = models.CharField(max_length=36, null=True, blank=True,
-    status = models.CharField(max_length=50, db_index=True,
+                            db_index=True)
-                              choices=STATUS_CHOICES, default=QUEUED)
+    status = models.CharField(max_length=30, db_index=True,
+                              choices=STATUS_CHOICES, null=True)
     image_type = models.IntegerField(null=True, default=0, db_index=True)
 
     @staticmethod

worker/start_workers.py

@@ -3,7 +3,6 @@ import signal
 import sys
 
 from multiprocessing import Process
-from worker import config
 
 POSSIBLE_TOPDIR = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]),
                                    os.pardir, os.pardir))
@@ -11,6 +10,7 @@ if os.path.exists(os.path.join(POSSIBLE_TOPDIR, 'stacktach')):
     sys.path.insert(0, POSSIBLE_TOPDIR)
 
 import worker.worker as worker
+from worker import config
 
 processes = []
 

worker/worker.py

@@ -114,9 +114,9 @@ class Consumer(kombu.mixins.ConsumerMixin):
             per_message = 0
             if self.total_processed:
                 per_message = idiff / self.total_processed
-            LOG.debug("%20s %6dk/%6dk ram, "
+            LOG.debug("%20s %20s %6dk/%6dk ram, "
                       "%3d/%4d msgs @ %6dk/msg" %
-                      (self.name, diff, idiff, self.processed,
+                      (self.name, self.exchange, diff, idiff, self.processed,
                       self.total_processed, per_message))
             self.last_vsz = self.pmi.vsz
             self.processed = 0