From ab755812976c7352c06e0183b6b494058094e595 Mon Sep 17 00:00:00 2001 From: adriant Date: Tue, 1 Apr 2014 17:24:14 +1300 Subject: [PATCH] Changes to the config system. This patch addresses issues with the need for a global config. Now any module needing config access can simply import the config module. To facilitate this there were a lot of minor changes to config use throughout artifice, as well as changes to classes that were passed a config on creation. Also some changes to constants so that the new config system can work with them easier. Change-Id: I6e9b4cbf0ff30683dc13e37f13334f7ed7ee7add --- artifice/api/web.py | 27 +++++++++++---------------- artifice/config.py | 20 ++++++++++++++++++++ artifice/constants.py | 24 ++++++++++++------------ artifice/helpers.py | 11 +++++++---- artifice/interface.py | 14 +++++++------- artifice/transformers.py | 10 ++++++---- examples/conf.yaml | 26 ++++++++++++++++++-------- tests/__init__.py | 11 +++++++---- tests/constants.py | 34 +++++++++++++++++++++++----------- tests/helpers.py | 14 +++++++------- tests/test_transformers.py | 33 +++++++++++++++++---------------- 11 files changed, 135 insertions(+), 89 deletions(-) create mode 100644 artifice/config.py diff --git a/artifice/api/web.py b/artifice/api/web.py index c334f67..dc38124 100644 --- a/artifice/api/web.py +++ b/artifice/api/web.py @@ -1,6 +1,6 @@ import flask from flask import Flask, Blueprint -from artifice import interface, database +from artifice import interface, database, config from artifice.sales_order import RatesFile from artifice.models import UsageEntry, SalesOrder, Tenant, billing import sqlalchemy @@ -25,36 +25,31 @@ Session = None app = Blueprint("main", __name__) -config = None - invoicer = None DEFAULT_TIMEZONE = "Pacific/Auckland" -current_region = "Wellington" # FIXME - def get_app(conf): actual_app = Flask(__name__) actual_app.register_blueprint(app, url_prefix="/") - global engine - engine = create_engine(conf["main"]["database_uri"], poolclass=NullPool) + config.setup_config(conf) - global config - config = conf + global engine + engine = create_engine(config.main["database_uri"], poolclass=NullPool) global Session Session = scoped_session(lambda: create_session(bind=engine)) global invoicer - module, kls = config["main"]["export_provider"].split(":") + module, kls = config.main["export_provider"].split(":") # TODO: Try/except block invoicer = getattr(importlib.import_module(module), kls) - if config["main"].get("timezone"): + if config.main.get("timezone"): global DEFAULT_TIMEZONE - DEFAULT_TIMEZONE = config["main"]["timezone"] + DEFAULT_TIMEZONE = config.main["timezone"] return actual_app @@ -153,8 +148,8 @@ def run_usage_collection(): """ session = Session() - - artifice = interface.Artifice(config) + + artifice = interface.Artifice() db = database.Database(session) tenants = artifice.tenants @@ -197,7 +192,7 @@ def generate_sales_order(tenant, session, end, rates): # and will probably result in the CSV exporter being changed. billable = billing.build_billable(usage, session) session.close() - exporter = invoicer(start, end, config["export_config"], rates) + exporter = invoicer(start, end, config.export_config, rates) exporter.bill(billable) exporter.close() return {"id": tenant.id, @@ -263,7 +258,7 @@ def run_sales_order_generation(): # Handled like this for a later move to Celery distributed workers resp = {"tenants": []} - rates = RatesFile(config['export_config']) + rates = RatesFile(config.export_config) for tenant in tenant_query: resp['tenants'].append(generate_sales_order(tenant, session, end, rates)) diff --git a/artifice/config.py b/artifice/config.py new file mode 100644 index 0000000..b7d4385 --- /dev/null +++ b/artifice/config.py @@ -0,0 +1,20 @@ + +# This is simply a namespace for global config storage +main = None +export_config = None +auth = None +ceilometer = None +transformers = None + + +def setup_config(conf): + global main + main = conf['main'] + global export_config + export_config = conf['export_config'] + global auth + auth = conf['auth'] + global ceilometer + ceilometer = conf['ceilometer'] + global transformers + transformers = conf['transformers'] diff --git a/artifice/constants.py b/artifice/constants.py index 6e43511..9688e54 100644 --- a/artifice/constants.py +++ b/artifice/constants.py @@ -12,15 +12,15 @@ date_format = "%Y-%m-%dT%H:%M:%S" other_date_format = "%Y-%m-%dT%H:%M:%S.%f" # VM states: -active = 1 -building = 2 -paused = 3 -suspended = 4 -stopped = 5 -rescued = 6 -resized = 7 -soft_deleted = 8 -deleted = 9 -error = 10 -shelved = 11 -shelved_offloaded = 12 +states = {'active': 1, + 'building': 2, + 'paused': 3, + 'suspended': 4, + 'stopped': 5, + 'rescued': 6, + 'resized': 7, + 'soft_deleted': 8, + 'deleted': 9, + 'error': 10, + 'shelved': 11, + 'shelved_offloaded': 12} diff --git a/artifice/helpers.py b/artifice/helpers.py index ec3975b..61bf2a2 100644 --- a/artifice/helpers.py +++ b/artifice/helpers.py @@ -1,9 +1,12 @@ from novaclient.v1_1 import client +import config def flavor_name(f_id): - # TODO get from config: - nova = client.Client("admin", "openstack", "demo", - "http://localhost:5000/v2.0", - service_type="compute") + nova = client.Client( + config.auth['username'], + config.auth['password'], + config.auth['default_tenant'], + config.auth['end_point'], + service_type="compute") return nova.flavors.get(f_id).name diff --git a/artifice/interface.py b/artifice/interface.py index 82cf346..853db39 100644 --- a/artifice/interface.py +++ b/artifice/interface.py @@ -4,6 +4,7 @@ import auth from ceilometerclient.v2.client import Client as ceilometer from artifice.models import resources from constants import date_format +import config def add_dates(start, end): @@ -23,21 +24,20 @@ def add_dates(start, end): class Artifice(object): """Produces billable artifacts""" - def __init__(self, config): + def __init__(self): super(Artifice, self).__init__() - self.config = config # This is the Keystone client connection, which provides our # OpenStack authentication self.auth = auth.Keystone( - username=config["openstack"]["username"], - password=config["openstack"]["password"], - tenant_name=config["openstack"]["default_tenant"], - auth_url=config["openstack"]["authentication_url"] + username=config.auth["username"], + password=config.auth["password"], + tenant_name=config.auth["default_tenant"], + auth_url=config.auth["end_point"] ) self.ceilometer = ceilometer( - self.config["ceilometer"]["host"], + config.ceilometer["host"], # Uses a lambda as ceilometer apparently wants # to use it as a callable? token=lambda: self.auth.auth_token diff --git a/artifice/transformers.py b/artifice/transformers.py index de372a8..757a4a4 100644 --- a/artifice/transformers.py +++ b/artifice/transformers.py @@ -1,6 +1,7 @@ import datetime import constants import helpers +import config class TransformerValidationError(Exception): @@ -42,10 +43,11 @@ class Uptime(Transformer): required_meters = ['state', 'flavor'] def _transform_usage(self, meters, start, end): - # this NEEDS to be moved to a config file - tracked_states = [constants.active, constants.building, - constants.paused, constants.rescued, - constants.resized] + # get tracked states from config + tracked = config.transformers['uptime']['tracked_states'] + + tracked_states = {constants.states[i] for i in tracked} + usage_dict = {} state = meters['state'] diff --git a/examples/conf.yaml b/examples/conf.yaml index cbd8952..7ae5836 100644 --- a/examples/conf.yaml +++ b/examples/conf.yaml @@ -1,17 +1,27 @@ --- -ceilometer: - host: http://localhost:8777/ -invoice_object: +main: + region: Wellington + timezone: Pacific/Auckland + export_provider: billing.csv_invoice:Csv + database_uri: postgres://artifice:123456@localhost:5432/artifice +export_config: delimiter: ',' output_file: '%(tenant)s-%(start)s-%(end)s.csv' output_path: ./ rates: file: /etc/artifice/csv_rates.csv -main: - export_provider: billing.csv_invoice:Csv - database_uri: postgres://artifice:123456@localhost:5432/artifice -openstack: - authentication_url: http://localhost:35357/v2.0 +auth: + end_point: http://localhost:35357/v2.0 default_tenant: demo username: admin password: openstack +ceilometer: + host: http://localhost:8777/ +transformers: + uptime: + tracked_states: + - active + - building + - paused + - rescued + - resized diff --git a/tests/__init__.py b/tests/__init__.py index 5fd3672..ef5cec8 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -4,11 +4,10 @@ from sqlalchemy.orm import sessionmaker, scoped_session,create_session from sqlalchemy.pool import NullPool from artifice.models import Resource, Tenant, UsageEntry, SalesOrder, Base +from artifice import config +from .constants import DATABASE_NAME, PG_DATABASE_URI, MY_DATABASE_URI +from .constants import config as test_config -DATABASE_NAME = "test_artifice" - -PG_DATABASE_URI = "postgresql://aurynn:postgres@localhost/%s" % DATABASE_NAME -MY_DATABASE_URI = "mysql://root:password@localhost/%s" % DATABASE_NAME def setUp(): subprocess.call(["/usr/bin/createdb","%s" % DATABASE_NAME]) @@ -21,6 +20,10 @@ def setUp(): mysql_engine.dispose() pg_engine.dispose() + # setup test config: + config.setup_config(test_config) + + def tearDown(): mysql_engine = create_engine(MY_DATABASE_URI, poolclass=NullPool) diff --git a/tests/constants.py b/tests/constants.py index 0493062..b7c0ba5 100644 --- a/tests/constants.py +++ b/tests/constants.py @@ -1,19 +1,17 @@ -from . import PG_DATABASE_URI + +DATABASE_NAME = "test_artifice" + +PG_DATABASE_URI = "postgresql://aurynn:postgres@localhost/%s" % DATABASE_NAME +MY_DATABASE_URI = "mysql://root:password@localhost/%s" % DATABASE_NAME + config = { - "ceilometer": { - "host": "http://localhost:8777/" - }, "main": { + "region": "Wellington", + "timezone": "Pacific/Auckland", "export_provider": "tests.mock_exporter:MockExporter", "database_uri": PG_DATABASE_URI }, - "openstack": { - "username": "admin", - "authentication_url": "http://localhost:35357/v2.0", - "password": "openstack", - "default_tenant": "demo" - }, "export_config": { "output_path": "./", "delimiter": ",", @@ -22,7 +20,21 @@ config = { "file": "examples/test_rates.csv" } }, - "artifice": {} + "auth": { + "end_point": "http://localhost:35357/v2.0", + "username": "admin", + "password": "openstack", + "default_tenant": "demo" + }, + "ceilometer": { + "host": "http://localhost:8777/" + }, + "transformers": { + "uptime": { + "tracked_states": ["active", "building", + "paused", "rescued", "resized"] + } + } } # from test data: diff --git a/tests/helpers.py b/tests/helpers.py index 3801198..fe59018 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -1,8 +1,8 @@ import mock -from artifice import interface, models +from artifice import interface, models, config from artifice.models import billing from .data_samples import RESOURCES, MAPPINGS -from .constants import config, TENANTS, AUTH_TOKEN +from .constants import TENANTS, AUTH_TOKEN from datetime import timedelta import json @@ -21,14 +21,14 @@ def get_usage(sqlmock, Keystone): return data interface.Meter.get_meter = get_meter - artifice = interface.Artifice(config) + artifice = interface.Artifice() artifice.auth.tenants.list.return_value = TENANTS Keystone.assert_called_with( - username=config["openstack"]["username"], - password=config["openstack"]["password"], - tenant_name=config["openstack"]["default_tenant"], - auth_url=config["openstack"]["authentication_url"] + username=config.auth["username"], + password=config.auth["password"], + tenant_name=config.auth["default_tenant"], + auth_url=config.auth["end_point"] ) tenants = None tenants = artifice.tenants diff --git a/tests/test_transformers.py b/tests/test_transformers.py index cc11844..ad5dfa5 100644 --- a/tests/test_transformers.py +++ b/tests/test_transformers.py @@ -1,6 +1,7 @@ import artifice.transformers from artifice.transformers import TransformerValidationError -import artifice.constants as constants +from artifice import constants +from artifice.constants import states import unittest import mock import datetime @@ -78,8 +79,8 @@ class UptimeTransformerTests(unittest.TestCase): {'timestamp': testdata.t1, 'counter_volume': testdata.flavor}, ]), 'state': TestMeter([ - {'timestamp': testdata.t0, 'counter_volume': constants.active}, - {'timestamp': testdata.t1, 'counter_volume': constants.active} + {'timestamp': testdata.t0, 'counter_volume': states['active']}, + {'timestamp': testdata.t1, 'counter_volume': states['active']} ]), } @@ -99,8 +100,8 @@ class UptimeTransformerTests(unittest.TestCase): {'timestamp': testdata.t1, 'counter_volume': testdata.flavor}, ]), 'state': TestMeter([ - {'timestamp': testdata.t0, 'counter_volume': constants.stopped}, - {'timestamp': testdata.t1, 'counter_volume': constants.stopped} + {'timestamp': testdata.t0, 'counter_volume': states['stopped']}, + {'timestamp': testdata.t1, 'counter_volume': states['stopped']} ]), } @@ -119,9 +120,9 @@ class UptimeTransformerTests(unittest.TestCase): {'timestamp': testdata.t1, 'counter_volume': testdata.flavor}, ]), 'state': TestMeter([ - {'timestamp': testdata.t0, 'counter_volume': constants.active}, - {'timestamp': testdata.t0_30, 'counter_volume': constants.stopped}, - {'timestamp': testdata.t1, 'counter_volume': constants.stopped} + {'timestamp': testdata.t0, 'counter_volume': states['active']}, + {'timestamp': testdata.t0_30, 'counter_volume': states['stopped']}, + {'timestamp': testdata.t1, 'counter_volume': states['stopped']} ]), } @@ -141,9 +142,9 @@ class UptimeTransformerTests(unittest.TestCase): {'timestamp': testdata.t1, 'counter_volume': testdata.flavor2}, ]), 'state': TestMeter([ - {'timestamp': testdata.t0, 'counter_volume': constants.active}, - {'timestamp': testdata.t0_30, 'counter_volume': constants.active}, - {'timestamp': testdata.t1, 'counter_volume': constants.active} + {'timestamp': testdata.t0, 'counter_volume': states['active']}, + {'timestamp': testdata.t0_30, 'counter_volume': states['active']}, + {'timestamp': testdata.t1, 'counter_volume': states['active']} ]), } @@ -162,8 +163,8 @@ class UptimeTransformerTests(unittest.TestCase): {'timestamp': testdata.t1, 'counter_volume': testdata.flavor}, ]), 'state': TestMeter([ - {'timestamp': testdata.t0_10, 'counter_volume': constants.active}, - {'timestamp': testdata.t1, 'counter_volume': constants.active}, + {'timestamp': testdata.t0_10, 'counter_volume': states['active']}, + {'timestamp': testdata.t1, 'counter_volume': states['active']}, ]), } @@ -186,9 +187,9 @@ class UptimeTransformerTests(unittest.TestCase): {'timestamp': testdata.t1, 'counter_volume': testdata.flavor}, ]), 'state': TestMeter([ - {'timestamp': testdata.tpre, 'counter_volume': constants.active}, - {'timestamp': testdata.t0_10, 'counter_volume': constants.active}, - {'timestamp': testdata.t1, 'counter_volume': constants.active}, + {'timestamp': testdata.tpre, 'counter_volume': states['active']}, + {'timestamp': testdata.t0_10, 'counter_volume': states['active']}, + {'timestamp': testdata.t1, 'counter_volume': states['active']}, ]), }