From 3b1e086a26eae748c217867cd18dcd3d1b97498d Mon Sep 17 00:00:00 2001 From: Anuj Mathur Date: Tue, 2 Jul 2013 14:53:43 +0530 Subject: [PATCH] - Moved config.py to worker - Removed exchange from config and started reading them from topics - Fixed lengths of GlanceRawData fields to match Glance fields - Made uuid and status accept null values as the image.send notification does not contain those values --- stacktach/config.py | 43 +++++++++++++++++++ ...eexists_and_instanceusages_from_rawdata.py | 3 +- ...reate_glancerawdata_and_genericrawdata.py} | 23 +++++++--- stacktach/models.py | 26 ++++++++--- worker/start_workers.py | 2 +- worker/worker.py | 4 +- 6 files changed, 86 insertions(+), 15 deletions(-) create mode 100644 stacktach/config.py rename stacktach/migrations/{0004_create_glancerawdata_and_genericrawdata.py => 0005_create_glancerawdata_and_genericrawdata.py} (92%) diff --git a/stacktach/config.py b/stacktach/config.py new file mode 100644 index 0000000..28c8b5e --- /dev/null +++ b/stacktach/config.py @@ -0,0 +1,43 @@ +# Copyright (c) 2013 - Rackspace Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +import json +import os + +config_filename = os.environ.get('STACKTACH_DEPLOYMENTS_FILE', + 'stacktach_worker_config.json') +try: + from local_settings import * + config_filename = STACKTACH_DEPLOYMENTS_FILE +except ImportError: + pass + +config = None +with open(config_filename, "r") as f: + config = json.load(f) + + +def deployments(): + return config['deployments'] + + +def topics(): + return config['topics'] + + diff --git a/stacktach/migrations/0003_populate_usage_related_fields_in_rawdataimagemeta_instanceexists_and_instanceusages_from_rawdata.py b/stacktach/migrations/0003_populate_usage_related_fields_in_rawdataimagemeta_instanceexists_and_instanceusages_from_rawdata.py index 86ae331..afa045e 100644 --- a/stacktach/migrations/0003_populate_usage_related_fields_in_rawdataimagemeta_instanceexists_and_instanceusages_from_rawdata.py +++ b/stacktach/migrations/0003_populate_usage_related_fields_in_rawdataimagemeta_instanceexists_and_instanceusages_from_rawdata.py @@ -42,7 +42,8 @@ class Migration(DataMigration): json_dict = json.loads(json_message) routing_key = json_dict[0] body = json_dict[1] - return notification_factory(body, None, routing_key, json_message, None) + return notification_factory(body, None, routing_key, json_message, + 'nova') def forwards(self, orm): # Note: Don't use "from appname.models import ModelName". diff --git a/stacktach/migrations/0004_create_glancerawdata_and_genericrawdata.py b/stacktach/migrations/0005_create_glancerawdata_and_genericrawdata.py similarity index 92% rename from stacktach/migrations/0004_create_glancerawdata_and_genericrawdata.py rename to stacktach/migrations/0005_create_glancerawdata_and_genericrawdata.py index 277c5ca..172c5bc 100644 --- a/stacktach/migrations/0004_create_glancerawdata_and_genericrawdata.py +++ b/stacktach/migrations/0005_create_glancerawdata_and_genericrawdata.py @@ -12,7 +12,7 @@ class Migration(SchemaMigration): db.create_table(u'stacktach_glancerawdata', ( (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), ('deployment', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['stacktach.Deployment'])), - ('owner', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=50, null=True, blank=True)), + ('owner', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=255, null=True, blank=True)), ('json', self.gf('django.db.models.fields.TextField')()), ('routing_key', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=50, null=True, blank=True)), ('when', self.gf('django.db.models.fields.DecimalField')(max_digits=20, decimal_places=6, db_index=True)), @@ -22,8 +22,8 @@ class Migration(SchemaMigration): ('host', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=100, null=True, blank=True)), ('instance', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=50, null=True, blank=True)), ('request_id', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=50, null=True, blank=True)), - ('uuid', self.gf('django.db.models.fields.CharField')(max_length=50)), - ('status', self.gf('django.db.models.fields.CharField')(default='queued', max_length=50, db_index=True)), + ('uuid', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=36, null=True, blank=True)), + ('status', self.gf('django.db.models.fields.CharField')(max_length=30, null=True, db_index=True)), ('image_type', self.gf('django.db.models.fields.IntegerField')(default=0, null=True, db_index=True)), )) db.send_create_signal(u'stacktach', ['GlanceRawData']) @@ -86,13 +86,13 @@ class Migration(SchemaMigration): 'image_type': ('django.db.models.fields.IntegerField', [], {'default': '0', 'null': 'True', 'db_index': 'True'}), 'instance': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), 'json': ('django.db.models.fields.TextField', [], {}), - 'owner': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'owner': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '255', 'null': 'True', 'blank': 'True'}), 'publisher': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '100', 'null': 'True', 'blank': 'True'}), 'request_id': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), 'routing_key': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), 'service': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), - 'status': ('django.db.models.fields.CharField', [], {'default': "'queued'", 'max_length': '50', 'db_index': 'True'}), - 'uuid': ('django.db.models.fields.CharField', [], {'max_length': '50'}), + 'status': ('django.db.models.fields.CharField', [], {'max_length': '30', 'null': 'True', 'db_index': 'True'}), + 'uuid': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '36', 'null': 'True', 'blank': 'True'}), 'when': ('django.db.models.fields.DecimalField', [], {'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}) }, u'stacktach.instancedeletes': { @@ -125,6 +125,17 @@ class Migration(SchemaMigration): 'tenant': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), 'usage': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'null': 'True', 'to': u"orm['stacktach.InstanceUsage']"}) }, + u'stacktach.instancereconcile': { + 'Meta': {'object_name': 'InstanceReconcile'}, + 'deleted_at': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'instance': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'instance_type_id': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'launched_at': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}), + 'row_created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'row_updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}), + 'source': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '150', 'null': 'True', 'blank': 'True'}) + }, u'stacktach.instanceusage': { 'Meta': {'object_name': 'InstanceUsage'}, u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), diff --git a/stacktach/models.py b/stacktach/models.py index 6980ee4..f2d3df4 100644 --- a/stacktach/models.py +++ b/stacktach/models.py @@ -1,3 +1,18 @@ +# Copyright 2012 - Dark Secret Software Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + from django import forms from django.db import models @@ -273,8 +288,8 @@ class GlanceRawData(models.Model): ] deployment = models.ForeignKey(Deployment) - owner = models.CharField(max_length=50, null=True, blank=True, - db_index=True) + owner = models.CharField(max_length=255, null=True, blank=True, + db_index=True) json = models.TextField() routing_key = models.CharField(max_length=50, null=True, blank=True, db_index=True) @@ -291,9 +306,10 @@ class GlanceRawData(models.Model): db_index=True) request_id = models.CharField(max_length=50, null=True, blank=True, db_index=True) - uuid = models.CharField(max_length=50) - status = models.CharField(max_length=50, db_index=True, - choices=STATUS_CHOICES, default=QUEUED) + uuid = models.CharField(max_length=36, null=True, blank=True, + db_index=True) + status = models.CharField(max_length=30, db_index=True, + choices=STATUS_CHOICES, null=True) image_type = models.IntegerField(null=True, default=0, db_index=True) @staticmethod diff --git a/worker/start_workers.py b/worker/start_workers.py index e7fc3d5..0558b57 100644 --- a/worker/start_workers.py +++ b/worker/start_workers.py @@ -3,7 +3,6 @@ import signal import sys from multiprocessing import Process -from worker import config POSSIBLE_TOPDIR = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), os.pardir, os.pardir)) @@ -11,6 +10,7 @@ if os.path.exists(os.path.join(POSSIBLE_TOPDIR, 'stacktach')): sys.path.insert(0, POSSIBLE_TOPDIR) import worker.worker as worker +from worker import config processes = [] diff --git a/worker/worker.py b/worker/worker.py index c3b584e..11efb98 100644 --- a/worker/worker.py +++ b/worker/worker.py @@ -114,9 +114,9 @@ class Consumer(kombu.mixins.ConsumerMixin): per_message = 0 if self.total_processed: per_message = idiff / self.total_processed - LOG.debug("%20s %6dk/%6dk ram, " + LOG.debug("%20s %20s %6dk/%6dk ram, " "%3d/%4d msgs @ %6dk/msg" % - (self.name, diff, idiff, self.processed, + (self.name, self.exchange, diff, idiff, self.processed, self.total_processed, per_message)) self.last_vsz = self.pmi.vsz self.processed = 0