From 9024534306c2edff2938229730acd1b24b477d90 Mon Sep 17 00:00:00 2001 From: Fei Long Wang Date: Tue, 10 May 2016 15:23:11 +1200 Subject: [PATCH] Implement /costs on new service Change-Id: I12469ec39fcaa79d6588d396367f14107cf4299f --- distil/api/auth.py | 17 --- distil/api/v2.py | 9 +- distil/{cli => cmd}/__init__.py | 0 distil/{cli => cmd}/distil_api.py | 0 distil/{cli => cmd}/distil_collector.py | 0 distil/common/constants.py | 47 +++++++ distil/config.py | 22 +++- distil/db/api.py | 9 +- .../versions/001_init_tables.py | 4 +- distil/db/sqlalchemy/api.py | 68 ++++++++-- distil/rater/__init__.py | 19 ++- distil/rater/odoo.py | 17 +-- distil/rater/rate_file.py | 13 +- distil/service/api/v2/costs.py | 121 +++++++++++++++++- distil/service/api/v2/health.py | 4 +- distil/utils/constants.py | 2 +- distil/utils/odoo.py | 12 +- etc/distil.conf.sample | 6 +- requirements.txt | 11 +- setup.cfg | 4 +- 20 files changed, 309 insertions(+), 76 deletions(-) rename distil/{cli => cmd}/__init__.py (100%) rename distil/{cli => cmd}/distil_api.py (100%) rename distil/{cli => cmd}/distil_collector.py (100%) create mode 100644 distil/common/constants.py diff --git a/distil/api/auth.py b/distil/api/auth.py index 0ced283..330efa1 100644 --- a/distil/api/auth.py +++ b/distil/api/auth.py @@ -14,7 +14,6 @@ # limitations under the License. from keystonemiddleware import auth_token -from keystonemiddleware import opts from oslo_config import cfg from oslo_log import log as logging import re @@ -23,22 +22,6 @@ from distil import exceptions CONF = cfg.CONF AUTH_GROUP_NAME = 'keystone_authtoken' - -def _register_opts(): - options = [] - keystone_opts = opts.list_auth_token_opts() - for n in keystone_opts: - if (n[0] == AUTH_GROUP_NAME): - options = n[1] - break - - CONF.register_opts(options, group=AUTH_GROUP_NAME) - auth_token.CONF = CONF - - -_register_opts() - - LOG = logging.getLogger(__name__) diff --git a/distil/api/v2.py b/distil/api/v2.py index 2508a42..bb0e68c 100644 --- a/distil/api/v2.py +++ b/distil/api/v2.py @@ -34,7 +34,14 @@ def prices_get(): @rest.get('/costs') def costs_get(): - return api.render(costs=costs.get_costs()) + # NOTE(flwang): Get 'tenant' first for backward compatibility. + tenant_id = api.get_request_args().get('tenant', None) + project_id = api.get_request_args().get('project_id', tenant_id) + start = api.get_request_args().get('start', None) + end = api.get_request_args().get('end', None) + # NOTE(flwang): Here using 'usage' instead of 'costs' for backward + # compatibility. + return api.render(usage=costs.get_costs(project_id, start, end)) @rest.get('/usages') diff --git a/distil/cli/__init__.py b/distil/cmd/__init__.py similarity index 100% rename from distil/cli/__init__.py rename to distil/cmd/__init__.py diff --git a/distil/cli/distil_api.py b/distil/cmd/distil_api.py similarity index 100% rename from distil/cli/distil_api.py rename to distil/cmd/distil_api.py diff --git a/distil/cli/distil_collector.py b/distil/cmd/distil_collector.py similarity index 100% rename from distil/cli/distil_collector.py rename to distil/cmd/distil_collector.py diff --git a/distil/common/constants.py b/distil/common/constants.py new file mode 100644 index 0000000..b374b38 --- /dev/null +++ b/distil/common/constants.py @@ -0,0 +1,47 @@ +# Copyright (C) 2014 Catalyst IT Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from datetime import datetime + +# Date format Ceilometer uses +# 2013-07-03T13:34:17 +# which is, as an strftime: +# timestamp = datetime.strptime(res["timestamp"], "%Y-%m-%dT%H:%M:%S.%f") +# or +# timestamp = datetime.strptime(res["timestamp"], "%Y-%m-%dT%H:%M:%S") + +# Most of the time we use date_format +date_format = "%Y-%m-%dT%H:%M:%S" +# Sometimes things also have milliseconds, so we look for that too. +# Because why not be annoying in all the ways? +other_date_format = "%Y-%m-%dT%H:%M:%S.%f" + +# Some useful constants +iso_time = "%Y-%m-%dT%H:%M:%S" +iso_date = "%Y-%m-%d" +dawn_of_time = datetime(2014, 4, 1) + +# VM states (SOON TO BE REMOVED): +states = {'active': 1, + 'building': 2, + 'paused': 3, + 'suspended': 4, + 'stopped': 5, + 'rescued': 6, + 'resized': 7, + 'soft_deleted': 8, + 'deleted': 9, + 'error': 10, + 'shelved': 11, + 'shelved_offloaded': 12} diff --git a/distil/config.py b/distil/config.py index f308711..9b12338 100644 --- a/distil/config.py +++ b/distil/config.py @@ -33,9 +33,13 @@ DEFAULT_OPTIONS = ( default=['/', '/v2/prices', '/v2/health'], help='The list of public API routes', ), + cfg.ListOpt('ignore_tenants', + default=[], + help=('The tenant name list which will be ignored when ' + 'collecting metrics from Ceilometer.')), ) -COLLECTOR_OPTIONS = [ +COLLECTOR_OPTS = [ cfg.IntOpt('periodic_interval', default=3600, help=('Interval of usage collection.')), cfg.StrOpt('collector_backend', default='ceilometer', @@ -69,17 +73,29 @@ ODOO_OPTS = [ help='Name of the Odoo database.'), cfg.StrOpt('user', help='Name of Odoo account to login.'), - cfg.StrOpt('password', + cfg.StrOpt('password', secret=True, help='Password of Odoo account to login.'), + cfg.StrOpt('last_update', + help='Last time when the products/prices are updated.') +] + +RATER_OPTS = [ + cfg.StrOpt('rater_type', default='odoo', + help='Rater type, by default it is odoo.'), + cfg.StrOpt('rate_file_path', default='/etc/distil/rates.csv', + help='Rate file path, it will be used when the rater_type ' + 'is "file".'), ] AUTH_GROUP = 'keystone_authtoken' ODOO_GROUP = 'odoo' COLLECTOR_GROUP = 'collector' +RATER_GROUP = 'rater' CONF.register_opts(DEFAULT_OPTIONS) CONF.register_opts(ODOO_OPTS, group=ODOO_GROUP) -CONF.register_opts(COLLECTOR_OPTIONS, group=COLLECTOR_GROUP) +CONF.register_opts(COLLECTOR_OPTS, group=COLLECTOR_GROUP) +CONF.register_opts(RATER_OPTS, group=RATER_GROUP) def _register_keystoneauth_opts(conf): diff --git a/distil/db/api.py b/distil/db/api.py index b0e3ea8..c1170e9 100644 --- a/distil/db/api.py +++ b/distil/db/api.py @@ -80,7 +80,6 @@ def to_dict(func): return decorator -@to_dict def usage_get(project_id, start_at, end_at): """Get usage for specific tenant based on time range. @@ -110,5 +109,13 @@ def project_add(values): return IMPL.project_add(values) +def resource_get_by_ids(project_id, resource_ids): + return IMPL.resource_get_by_ids(project_id, resource_ids) + + +def project_get(project_id): + return IMPL.project_get(project_id) + + def project_get_all(): return IMPL.project_get_all() diff --git a/distil/db/migration/alembic_migrations/versions/001_init_tables.py b/distil/db/migration/alembic_migrations/versions/001_init_tables.py index 4a0239d..363af20 100644 --- a/distil/db/migration/alembic_migrations/versions/001_init_tables.py +++ b/distil/db/migration/alembic_migrations/versions/001_init_tables.py @@ -68,9 +68,9 @@ def upgrade(): primary_key=True, nullable=False), sa.Column('resource_id', sa.String(length=100), primary_key=True, nullable=False), - sa.Column('start_at', sa.DateTime(), primary_key=True, + sa.Column('start', sa.DateTime(), primary_key=True, nullable=True), - sa.Column('end_at', sa.DateTime(), primary_key=True, + sa.Column('end', sa.DateTime(), primary_key=True, nullable=True), sa.Column('created', sa.DateTime(), nullable=True), sa.ForeignKeyConstraint(['tenant_id'], ['tenants.id'], ), diff --git a/distil/db/sqlalchemy/api.py b/distil/db/sqlalchemy/api.py index 68f7a15..cd0a57e 100644 --- a/distil/db/sqlalchemy/api.py +++ b/distil/db/sqlalchemy/api.py @@ -20,11 +20,14 @@ import json import six import sys +import sqlalchemy as sa +from sqlalchemy import func +import threading + from oslo_config import cfg from oslo_db import exception as db_exception from oslo_db.sqlalchemy import session as db_session from oslo_log import log as logging -import sqlalchemy as sa from distil.db.sqlalchemy import models as m from distil.db.sqlalchemy.models import Resource @@ -37,18 +40,17 @@ LOG = logging.getLogger(__name__) CONF = cfg.CONF _FACADE = None +_LOCK = threading.Lock() def _create_facade_lazily(): - global _FACADE + global _LOCK, _FACADE if _FACADE is None: - params = dict(CONF.database.iteritems()) - params["sqlite_fk"] = True - _FACADE = db_session.EngineFacade( - CONF.database.connection, - **params - ) + with _LOCK: + if _FACADE is None: + _FACADE = db_session.EngineFacade.from_config(CONF, + sqlite_fk=True) return _FACADE @@ -141,15 +143,43 @@ def project_get_all(): return query.all() -def usage_get(project_id, start_at, end_at): +def project_get(project_id): session = get_session() - query = session.query(UsageEntry) + query = session.query(Tenant) + query = query.filter(Tenant.id == project_id) + try: + return query.one() + except Exception: + raise exceptions.NotFoundException(project_id) - query = (query.filter(UsageEntry.start_at >= start_at, - UsageEntry.end_at <= end_at). - filter(UsageEntry.project_id == project_id)) - return query.all() +def usage_get(project_id, start, end): + session = get_session() + query = session.query(UsageEntry.tenant_id, + UsageEntry.resource_id, + UsageEntry.service, + UsageEntry.unit, + func.sum(UsageEntry.volume).label("volume")) + + query = (query.filter(UsageEntry.start >= start, + UsageEntry.end <= end). + filter(UsageEntry.tenant_id == project_id). + group_by(UsageEntry.tenant_id, UsageEntry.resource_id, + UsageEntry.service, UsageEntry.unit)) + result = [] + # NOTE(flwang): With group_by and func.sum, the query result is a list of + # array, which is hard to be consumed. So Here we're using a named tuple + # so that it can be easier to use. + for entry in query.all(): + ue = UsageEntry() + ue.tenant_id = entry[0] + ue.resource_id = entry[1] + ue.service = entry[2] + ue.unit = entry[3] + ue.volume = entry[4] + result.append(ue) + + return result def usage_add(project_id, resource_id, samples, unit, @@ -243,6 +273,16 @@ def resource_add(project_id, resource_id, resource_type, raw, metadata): raise e +def resource_get_by_ids(project_id, resource_ids): + session = get_session() + query = session.query(Resource) + + query = (query.filter(Resource.id.in_(resource_ids)). + filter(Resource.tenant_id == project_id)) + + return query.all() + + def _merge_resource_metadata(md_dict, entry, md_def): """Strips metadata from the entry as defined in the config, and merges it with the given metadata dict. diff --git a/distil/rater/__init__.py b/distil/rater/__init__.py index b3fb211..a21d71a 100644 --- a/distil/rater/__init__.py +++ b/distil/rater/__init__.py @@ -13,11 +13,28 @@ # See the License for the specific language governing permissions and # limitations under the License. +from oslo_config import cfg + +from stevedore import driver + +CONF = cfg.CONF +RATER = None + class BaseRater(object): - def __init__(self, conf): + def __init__(self, conf=None): self.conf = conf def rate(self, name, region=None): raise NotImplementedError("Not implemented in base class") + + +def get_rater(): + global RATER + if RATER is None: + RATER = driver.DriverManager('distil.rater', + CONF.rater.rater_type, + invoke_on_load=True, + invoke_kwds={}).driver + return RATER \ No newline at end of file diff --git a/distil/rater/odoo.py b/distil/rater/odoo.py index 00751f8..f948ab5 100644 --- a/distil/rater/odoo.py +++ b/distil/rater/odoo.py @@ -20,19 +20,20 @@ from distil.utils import odoo class OdooRater(rater.BaseRater): - def __init__(self, conf): + def __init__(self): self.prices = odoo.Odoo().get_prices() def rate(self, name, region=None): if not self.prices: return rate_file.FileRater().rate(name, region) + region_prices = (self.prices[region] if region else + self.prices.values[0]) - for region in self.prices: - for category in self.prices[region]: - for product in self.prices[region][category]: - if product == name: - return {'rate': self.prices[name]['price'], - 'unit': self.prices[name]['unit'] - } + for category in region_prices: + for product in region_prices[category]: + if product['resource'] == name: + return {'rate': product['price'], + 'unit': product['unit'] + } return rate_file.FileRater().rate(name, region) diff --git a/distil/rater/rate_file.py b/distil/rater/rate_file.py index 9c8cd98..d913b7b 100644 --- a/distil/rater/rate_file.py +++ b/distil/rater/rate_file.py @@ -16,17 +16,20 @@ import csv from decimal import Decimal -import logging as log +from oslo_config import cfg +import oslo_log as log from distil import rater +from distil import exceptions + +CONF = cfg.CONF class FileRater(rater.BaseRater): - def __init__(self, conf): - super(FileRater, self).__init__(conf) + def __init__(self): try: - with open(self.config['file']) as fh: + with open(CONF.rater.rate_file_path) as fh: # Makes no opinions on the file structure reader = csv.reader(fh, delimiter="|") self.__rates = { @@ -38,7 +41,7 @@ class FileRater(rater.BaseRater): } except Exception as e: log.critical('Failed to load rates file: `%s`' % e) - raise + exceptions.InvalidConfig(e) def rate(self, name, region=None): return { diff --git a/distil/service/api/v2/costs.py b/distil/service/api/v2/costs.py index 4389cd5..0a91222 100644 --- a/distil/service/api/v2/costs.py +++ b/distil/service/api/v2/costs.py @@ -13,12 +13,129 @@ # See the License for the specific language governing permissions and # limitations under the License. +import json +from datetime import datetime +from decimal import Decimal + from oslo_config import cfg from oslo_log import log as logging +from distil import rater +from distil.common import constants +from distil.db import api as db_api +from distil.utils import general + LOG = logging.getLogger(__name__) CONF = cfg.CONF -def get_costs(): - return {'id': 1} \ No newline at end of file +def get_costs(project_id, start, end): + try: + if start is not None: + try: + start = datetime.strptime(start, constants.iso_date) + except ValueError: + start = datetime.strptime(start, constants.iso_time) + else: + return 400, {"missing parameter": {"start": "start date" + + " in format: y-m-d"}} + if not end: + end = datetime.utcnow() + else: + try: + end = datetime.strptime(end, constants.iso_date) + except ValueError: + end = datetime.strptime(end, constants.iso_time) + except ValueError: + return 400, { + "errors": ["'end' date given needs to be in format: " + + "y-m-d, or y-m-dTH:M:S"]} + + if end <= start: + return 400, {"errors": ["end date must be greater than start."]} + + project = db_api.project_get(project_id) + + LOG.debug("Calculating rated data for %s in range: %s - %s" % + (project.id, start, end)) + + costs = _calculate_cost(project, start, end) + + return costs + + +def _calculate_cost(project, start, end): + """Calculate a rated data dict from the given range.""" + + usage = db_api.usage_get(project.id, start, end) + + # Transform the query result into a billable dict. + project_dict = _build_project_dict(project, usage) + project_dict = _add_costs_for_project(project_dict) + + # add sales order range: + project_dict['start'] = str(start) + project_dict['end'] = str(end) + + return project_dict + + +def _build_project_dict(project, usage): + """Builds a dict structure for a given project.""" + + project_dict = {'name': project.name, 'tenant_id': project.id} + + all_resource_ids = [entry.get('resource_id') for entry in usage] + res_list = db_api.resource_get_by_ids(project.id, all_resource_ids) + project_dict['resources'] = {row.id: json.loads(row.info) + for row in res_list} + + for entry in usage: + service = {'name': entry.get('service'), + 'volume': entry.get('volume'), + 'unit': entry.get('unit')} + + resource = project_dict['resources'][entry.get('resource_id')] + service_list = resource.setdefault('services', []) + service_list.append(service) + + return project_dict + + +def _add_costs_for_project(project): + """Adds cost values to services using the given rates manager.""" + + current_rater = rater.get_rater() + + project_total = 0 + for resource in project['resources'].values(): + resource_total = 0 + for service in resource['services']: + try: + rate = current_rater.rate(service['name']) + except KeyError: + # no rate exists for this service + service['cost'] = "0" + service['volume'] = "unknown unit conversion" + service['unit'] = "unknown" + service['rate'] = "missing rate" + continue + + volume = general.convert_to(service['volume'], + service['unit'], + rate['unit']) + + # round to 2dp so in dollars. + cost = round(volume * Decimal(rate['rate']), 2) + + service['cost'] = str(cost) + service['volume'] = str(volume) + service['unit'] = rate['unit'] + service['rate'] = str(rate['rate']) + + resource_total += cost + resource['total_cost'] = str(resource_total) + project_total += resource_total + project['total_cost'] = str(project_total) + + return project diff --git a/distil/service/api/v2/health.py b/distil/service/api/v2/health.py index d7d8766..da9bd62 100644 --- a/distil/service/api/v2/health.py +++ b/distil/service/api/v2/health.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import datetime +from datetime import datetime as dt from oslo_config import cfg from oslo_log import log as logging @@ -36,7 +36,7 @@ def get_health(): # tenant is still active in Keystone, we believe it should be investigated. failed_collected_count = 0 for p in projects: - delta = (datetime.now() - p.last_collected).total_seconds() // 3600 + delta = (dt.now() - p.last_collected).total_seconds() // 3600 if delta >= 24 and p.id in project_id_list_keystone: failed_collected_count += 1 diff --git a/distil/utils/constants.py b/distil/utils/constants.py index db1f0f5..ed8c6cf 100644 --- a/distil/utils/constants.py +++ b/distil/utils/constants.py @@ -32,7 +32,7 @@ date_format_f = "%Y-%m-%dT%H:%M:%S.%f" # Some useful constants iso_time = "%Y-%m-%dT%H:%M:%S" iso_date = "%Y-%m-%d" -dawn_of_time = datetime(2014, 4, 1) +dawn_of_time = datetime(2016, 5, 10) # VM states: states = {'active': 1, diff --git a/distil/utils/odoo.py b/distil/utils/odoo.py index 13e5cae..ba32e42 100644 --- a/distil/utils/odoo.py +++ b/distil/utils/odoo.py @@ -54,7 +54,7 @@ class Odoo(object): for r in regions: prices[r] = {} for category in PRODUCT_CATEGORY: - prices[r][category.lower()] = {} + prices[r][category.lower()] = [] c = self.category.search([('name', '=', category), ('display_name', 'ilike', r)]) product_ids = self.product.search([('categ_id', '=', c[0]), @@ -65,15 +65,17 @@ class Odoo(object): product_ids) for p in products: name = p['name_template'][len(r) + 1:] + if 'pre-prod' in name: + continue price = round(p['lst_price'], 5) # NOTE(flwang): default_code is Internal Reference on Odoo # GUI unit = p['default_code'] desc = p['description'] - prices[r][category.lower()][name] = {'price': price, - 'unit': unit, - 'description': desc - } + prices[r][category.lower()].append({'resource': name, + 'price': price, + 'unit': unit, + 'description': desc}) return prices diff --git a/etc/distil.conf.sample b/etc/distil.conf.sample index 945e354..8b8cf75 100644 --- a/etc/distil.conf.sample +++ b/etc/distil.conf.sample @@ -11,8 +11,8 @@ dawn_of_time = 2016-05-18 01:00:00 meter_mappings_file = /etc/distil/meter_mappings.yml [rater] -type = file -rates_file = /etc/distil/rates.csv +rater_type = odoo +rate_file_path = /etc/distil/rates.csv [odoo] version=8.0 @@ -30,7 +30,7 @@ backend = sqlalchemy [keystone_authtoken] memcache_servers = 127.0.0.1:11211 signing_dir = /var/cache/distil -cafile = /opt/stack/data/ca-bundle.pem +cafile = /opt/stack/data/ca-bundle.pem auth_uri = http://127.0.0.1:5000 project_domain_id = default project_name = service diff --git a/requirements.txt b/requirements.txt index 4338381..a7aea9d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,24 +13,17 @@ keystonemiddleware!=4.1.0,>=4.0.0 # Apache-2.0 keystoneauth1>=2.1.0 # Apache-2.0 python-cinderclient>=1.6.0 # Apache-2.0 -python-keystoneclient>=1.7.0,!=1.8.0,!=2.1.0 # Apache-2.0 -python-manilaclient>=1.3.0 # Apache-2.0 +python-keystoneclient!=1.8.0,!=2.1.0,>=1.6.0 # Apache-2.0 python-novaclient!=2.33.0,>=2.29.0 # Apache-2.0 -python-swiftclient>=2.2.0 # Apache-2.0 python-neutronclient>=4.2.0 # Apache-2.0 -python-heatclient>=0.6.0 # Apache-2.0 python-ceilometerclient>=2.2.1 # Apache-2.0 +python-glanceclient>=2.0.0 # Apache-2.0 oslo.config>=3.9.0 # Apache-2.0 -oslo.concurrency>=3.5.0 # Apache-2.0 oslo.context>=2.2.0 # Apache-2.0 oslo.db>=4.1.0 # Apache-2.0 oslo.i18n>=2.1.0 # Apache-2.0 oslo.log>=1.14.0 # Apache-2.0 -oslo.messaging>=4.5.0 # Apache-2.0 -oslo.middleware>=3.0.0 # Apache-2.0 -oslo.policy>=0.5.0 # Apache-2.0 -oslo.rootwrap>=2.0.0 # Apache-2.0 oslo.serialization>=1.10.0 # Apache-2.0 oslo.service>=1.0.0 # Apache-2.0 oslo.utils>=3.5.0 # Apache-2.0 diff --git a/setup.cfg b/setup.cfg index 42cae03..52b37a1 100644 --- a/setup.cfg +++ b/setup.cfg @@ -27,8 +27,8 @@ data_files = [entry_points] console_scripts = - distil-api = distil.cli.distil_api:main - distil-collector = distil.cli.distil_collector:main + distil-api = distil.cmd.distil_api:main + distil-collector = distil.cmd.distil_collector:main distil-db-manage = distil.db.migration.cli:main distil.collector =