Implement /costs on new service

Change-Id: I12469ec39fcaa79d6588d396367f14107cf4299f

distil/api/auth.py

@@ -14,7 +14,6 @@
 # limitations under the License.
 
 from keystonemiddleware import auth_token
-from keystonemiddleware import opts
 from oslo_config import cfg
 from oslo_log import log as logging
 import re
@@ -23,22 +22,6 @@ from distil import exceptions
 CONF = cfg.CONF
 AUTH_GROUP_NAME = 'keystone_authtoken'
 
-
-def _register_opts():
-    options = []
-    keystone_opts = opts.list_auth_token_opts()
-    for n in keystone_opts:
-        if (n[0] == AUTH_GROUP_NAME):
-            options = n[1]
-            break
-
-        CONF.register_opts(options, group=AUTH_GROUP_NAME)
-        auth_token.CONF = CONF
-
-
-_register_opts()
-
-
 LOG = logging.getLogger(__name__)
 
 

distil/api/v2.py

@@ -34,7 +34,14 @@ def prices_get():
 
 @rest.get('/costs')
 def costs_get():
-    return api.render(costs=costs.get_costs())
+    # NOTE(flwang): Get 'tenant' first for backward compatibility.
+    tenant_id = api.get_request_args().get('tenant', None)
+    project_id = api.get_request_args().get('project_id', tenant_id)
+    start = api.get_request_args().get('start', None)
+    end = api.get_request_args().get('end', None)
+    # NOTE(flwang): Here using 'usage' instead of 'costs' for backward
+    # compatibility.
+    return api.render(usage=costs.get_costs(project_id, start, end))
 
 
 @rest.get('/usages')

distil/common/constants.py

@@ -0,0 +1,47 @@
+# Copyright (C) 2014 Catalyst IT Ltd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from datetime import datetime
+
+# Date format Ceilometer uses
+# 2013-07-03T13:34:17
+# which is, as an strftime:
+# timestamp = datetime.strptime(res["timestamp"], "%Y-%m-%dT%H:%M:%S.%f")
+# or
+# timestamp = datetime.strptime(res["timestamp"], "%Y-%m-%dT%H:%M:%S")
+
+# Most of the time we use date_format
+date_format = "%Y-%m-%dT%H:%M:%S"
+# Sometimes things also have milliseconds, so we look for that too.
+# Because why not be annoying in all the ways?
+other_date_format = "%Y-%m-%dT%H:%M:%S.%f"
+
+# Some useful constants
+iso_time = "%Y-%m-%dT%H:%M:%S"
+iso_date = "%Y-%m-%d"
+dawn_of_time = datetime(2014, 4, 1)
+
+# VM states (SOON TO BE REMOVED):
+states = {'active': 1,
+          'building': 2,
+          'paused': 3,
+          'suspended': 4,
+          'stopped': 5,
+          'rescued': 6,
+          'resized': 7,
+          'soft_deleted': 8,
+          'deleted': 9,
+          'error': 10,
+          'shelved': 11,
+          'shelved_offloaded': 12}

distil/config.py

@@ -33,9 +33,13 @@ DEFAULT_OPTIONS = (
                 default=['/', '/v2/prices', '/v2/health'],
                 help='The list of public API routes',
                 ),
+    cfg.ListOpt('ignore_tenants',
+                default=[],
+                help=('The tenant name list which will be ignored when '
+                      'collecting metrics from Ceilometer.')),
 )
 
-COLLECTOR_OPTIONS = [
+COLLECTOR_OPTS = [
     cfg.IntOpt('periodic_interval', default=3600,
                help=('Interval of usage collection.')),
     cfg.StrOpt('collector_backend', default='ceilometer',
@@ -69,17 +73,29 @@ ODOO_OPTS = [
                help='Name of the Odoo database.'),
     cfg.StrOpt('user',
                help='Name of Odoo account to login.'),
-    cfg.StrOpt('password',
+    cfg.StrOpt('password', secret=True,
                help='Password of Odoo account to login.'),
+    cfg.StrOpt('last_update',
+               help='Last time when the products/prices are updated.')
+]
+
+RATER_OPTS = [
+    cfg.StrOpt('rater_type', default='odoo',
+               help='Rater type, by default it is odoo.'),
+    cfg.StrOpt('rate_file_path', default='/etc/distil/rates.csv',
+               help='Rate file path, it will be used when the rater_type '
+               'is "file".'),
 ]
 
 AUTH_GROUP = 'keystone_authtoken'
 ODOO_GROUP = 'odoo'
 COLLECTOR_GROUP = 'collector'
+RATER_GROUP = 'rater'
 
 CONF.register_opts(DEFAULT_OPTIONS)
 CONF.register_opts(ODOO_OPTS, group=ODOO_GROUP)
-CONF.register_opts(COLLECTOR_OPTIONS, group=COLLECTOR_GROUP)
+CONF.register_opts(COLLECTOR_OPTS, group=COLLECTOR_GROUP)
+CONF.register_opts(RATER_OPTS, group=RATER_GROUP)
 
 
 def _register_keystoneauth_opts(conf):

distil/db/api.py

@@ -80,7 +80,6 @@ def to_dict(func):
     return decorator
 
 
-@to_dict
 def usage_get(project_id, start_at, end_at):
     """Get usage for specific tenant based on time range.
 
@@ -110,5 +109,13 @@ def project_add(values):
     return IMPL.project_add(values)
 
 
+def resource_get_by_ids(project_id, resource_ids):
+    return IMPL.resource_get_by_ids(project_id, resource_ids)
+
+
+def project_get(project_id):
+    return IMPL.project_get(project_id)
+
+
 def project_get_all():
     return IMPL.project_get_all()

distil/db/migration/alembic_migrations/versions/001_init_tables.py

@@ -68,9 +68,9 @@ def upgrade():
                               primary_key=True, nullable=False),
                     sa.Column('resource_id', sa.String(length=100),
                               primary_key=True, nullable=False),
-                    sa.Column('start_at', sa.DateTime(), primary_key=True,
+                    sa.Column('start', sa.DateTime(), primary_key=True,
                               nullable=True),
-                    sa.Column('end_at', sa.DateTime(), primary_key=True,
+                    sa.Column('end', sa.DateTime(), primary_key=True,
                               nullable=True),
                     sa.Column('created', sa.DateTime(), nullable=True),
                     sa.ForeignKeyConstraint(['tenant_id'], ['tenants.id'], ),

distil/db/sqlalchemy/api.py

@@ -20,11 +20,14 @@ import json
 import six
 import sys
 
+import sqlalchemy as sa
+from sqlalchemy import func
+import threading
+
 from oslo_config import cfg
 from oslo_db import exception as db_exception
 from oslo_db.sqlalchemy import session as db_session
 from oslo_log import log as logging
-import sqlalchemy as sa
 
 from distil.db.sqlalchemy import models as m
 from distil.db.sqlalchemy.models import Resource
@@ -37,18 +40,17 @@ LOG = logging.getLogger(__name__)
 CONF = cfg.CONF
 
 _FACADE = None
+_LOCK = threading.Lock()
 
 
 def _create_facade_lazily():
-    global _FACADE
+    global _LOCK, _FACADE
 
     if _FACADE is None:
-        params = dict(CONF.database.iteritems())
+        with _LOCK:
-        params["sqlite_fk"] = True
+            if _FACADE is None:
-        _FACADE = db_session.EngineFacade(
+                _FACADE = db_session.EngineFacade.from_config(CONF,
-            CONF.database.connection,
+                                                              sqlite_fk=True)
-            **params
-        )
     return _FACADE
 
 
@@ -141,15 +143,43 @@ def project_get_all():
     return query.all()
 
 
-def usage_get(project_id, start_at, end_at):
+def project_get(project_id):
     session = get_session()
-    query = session.query(UsageEntry)
+    query = session.query(Tenant)
+    query = query.filter(Tenant.id == project_id)
+    try:
+        return query.one()
+    except Exception:
+        raise exceptions.NotFoundException(project_id)
 
-    query = (query.filter(UsageEntry.start_at >= start_at,
-                          UsageEntry.end_at <= end_at).
-             filter(UsageEntry.project_id == project_id))
 
-    return query.all()
+def usage_get(project_id, start, end):
+    session = get_session()
+    query = session.query(UsageEntry.tenant_id,
+                          UsageEntry.resource_id,
+                          UsageEntry.service,
+                          UsageEntry.unit,
+                          func.sum(UsageEntry.volume).label("volume"))
+
+    query = (query.filter(UsageEntry.start >= start,
+                          UsageEntry.end <= end).
+             filter(UsageEntry.tenant_id == project_id).
+             group_by(UsageEntry.tenant_id, UsageEntry.resource_id,
+                      UsageEntry.service, UsageEntry.unit))
+    result = []
+    # NOTE(flwang): With group_by and func.sum, the query result is a list of
+    # array, which is hard to be consumed. So Here we're using a named tuple
+    # so that it can be easier to use.
+    for entry in query.all():
+        ue = UsageEntry()
+        ue.tenant_id = entry[0]
+        ue.resource_id = entry[1]
+        ue.service = entry[2]
+        ue.unit = entry[3]
+        ue.volume = entry[4]
+        result.append(ue)
+
+    return result
 
 
 def usage_add(project_id, resource_id, samples, unit,
@@ -243,6 +273,16 @@ def resource_add(project_id, resource_id, resource_type, raw, metadata):
         raise e
 
 
+def resource_get_by_ids(project_id, resource_ids):
+    session = get_session()
+    query = session.query(Resource)
+
+    query = (query.filter(Resource.id.in_(resource_ids)).
+             filter(Resource.tenant_id == project_id))
+
+    return query.all()
+
+
 def _merge_resource_metadata(md_dict, entry, md_def):
     """Strips metadata from the entry as defined in the config,
        and merges it with the given metadata dict.

distil/rater/__init__.py

@@ -13,11 +13,28 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from oslo_config import cfg
+
+from stevedore import driver
+
+CONF = cfg.CONF
+RATER = None
+
 
 class BaseRater(object):
 
-    def __init__(self, conf):
+    def __init__(self, conf=None):
         self.conf = conf
 
     def rate(self, name, region=None):
         raise NotImplementedError("Not implemented in base class")
+
+
+def get_rater():
+    global RATER
+    if RATER is None:
+        RATER = driver.DriverManager('distil.rater',
+                                     CONF.rater.rater_type,
+                                     invoke_on_load=True,
+                                     invoke_kwds={}).driver
+    return RATER

distil/rater/odoo.py

@@ -20,19 +20,20 @@ from distil.utils import odoo
 
 class OdooRater(rater.BaseRater):
 
-    def __init__(self, conf):
+    def __init__(self):
         self.prices = odoo.Odoo().get_prices()
 
     def rate(self, name, region=None):
         if not self.prices:
             return rate_file.FileRater().rate(name, region)
+        region_prices = (self.prices[region] if region else
+                         self.prices.values[0])
 
-        for region in self.prices:
+        for category in region_prices:
-            for category in self.prices[region]:
+            for product in region_prices[category]:
-                for product in self.prices[region][category]:
+                if product['resource'] == name:
-                    if product == name:
+                    return {'rate': product['price'],
-                        return {'rate': self.prices[name]['price'],
+                            'unit': product['unit']
-                                'unit': self.prices[name]['unit']
                             }
 
         return rate_file.FileRater().rate(name, region)

distil/rater/rate_file.py

@@ -16,17 +16,20 @@
 import csv
 from decimal import Decimal
 
-import logging as log
+from oslo_config import cfg
+import oslo_log as log
 
 from distil import rater
+from distil import exceptions
+
+CONF = cfg.CONF
 
 
 class FileRater(rater.BaseRater):
-    def __init__(self, conf):
+    def __init__(self):
-        super(FileRater, self).__init__(conf)
 
         try:
-            with open(self.config['file']) as fh:
+            with open(CONF.rater.rate_file_path) as fh:
                 # Makes no opinions on the file structure
                 reader = csv.reader(fh, delimiter="|")
                 self.__rates = {
@@ -38,7 +41,7 @@ class FileRater(rater.BaseRater):
                 }
         except Exception as e:
             log.critical('Failed to load rates file: `%s`' % e)
-            raise
+            exceptions.InvalidConfig(e)
 
     def rate(self, name, region=None):
         return {

distil/service/api/v2/costs.py

@@ -13,12 +13,129 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import json
+from datetime import datetime
+from decimal import Decimal
+
 from oslo_config import cfg
 from oslo_log import log as logging
 
+from distil import rater
+from distil.common import constants
+from distil.db import api as db_api
+from distil.utils import general
+
 LOG = logging.getLogger(__name__)
 CONF = cfg.CONF
 
 
-def get_costs():
+def get_costs(project_id, start, end):
-    return {'id': 1}
+    try:
+        if start is not None:
+            try:
+                start = datetime.strptime(start, constants.iso_date)
+            except ValueError:
+                start = datetime.strptime(start, constants.iso_time)
+        else:
+            return 400, {"missing parameter": {"start": "start date" +
+                                               " in format: y-m-d"}}
+        if not end:
+            end = datetime.utcnow()
+        else:
+            try:
+                end = datetime.strptime(end, constants.iso_date)
+            except ValueError:
+                end = datetime.strptime(end, constants.iso_time)
+    except ValueError:
+            return 400, {
+                "errors": ["'end' date given needs to be in format: " +
+                           "y-m-d, or y-m-dTH:M:S"]}
+
+    if end <= start:
+        return 400, {"errors": ["end date must be greater than start."]}
+
+    project = db_api.project_get(project_id)
+
+    LOG.debug("Calculating rated data for %s in range: %s - %s" %
+              (project.id, start, end))
+
+    costs = _calculate_cost(project, start, end)
+
+    return costs
+
+
+def _calculate_cost(project, start, end):
+    """Calculate a rated data dict from the given range."""
+
+    usage = db_api.usage_get(project.id, start, end)
+
+    # Transform the query result into a billable dict.
+    project_dict = _build_project_dict(project, usage)
+    project_dict = _add_costs_for_project(project_dict)
+
+    # add sales order range:
+    project_dict['start'] = str(start)
+    project_dict['end'] = str(end)
+
+    return project_dict
+
+
+def _build_project_dict(project, usage):
+    """Builds a dict structure for a given project."""
+
+    project_dict = {'name': project.name, 'tenant_id': project.id}
+
+    all_resource_ids = [entry.get('resource_id') for entry in usage]
+    res_list = db_api.resource_get_by_ids(project.id, all_resource_ids)
+    project_dict['resources'] = {row.id: json.loads(row.info)
+                                 for row in res_list}
+
+    for entry in usage:
+        service = {'name': entry.get('service'),
+                   'volume': entry.get('volume'),
+                   'unit': entry.get('unit')}
+
+        resource = project_dict['resources'][entry.get('resource_id')]
+        service_list = resource.setdefault('services', [])
+        service_list.append(service)
+
+    return project_dict
+
+
+def _add_costs_for_project(project):
+    """Adds cost values to services using the given rates manager."""
+
+    current_rater = rater.get_rater()
+
+    project_total = 0
+    for resource in project['resources'].values():
+        resource_total = 0
+        for service in resource['services']:
+            try:
+                rate = current_rater.rate(service['name'])
+            except KeyError:
+                # no rate exists for this service
+                service['cost'] = "0"
+                service['volume'] = "unknown unit conversion"
+                service['unit'] = "unknown"
+                service['rate'] = "missing rate"
+                continue
+
+            volume = general.convert_to(service['volume'],
+                                        service['unit'],
+                                        rate['unit'])
+
+            # round to 2dp so in dollars.
+            cost = round(volume * Decimal(rate['rate']), 2)
+
+            service['cost'] = str(cost)
+            service['volume'] = str(volume)
+            service['unit'] = rate['unit']
+            service['rate'] = str(rate['rate'])
+
+            resource_total += cost
+        resource['total_cost'] = str(resource_total)
+        project_total += resource_total
+    project['total_cost'] = str(project_total)
+
+    return project

distil/service/api/v2/health.py

@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import datetime
+from datetime import datetime as dt
 
 from oslo_config import cfg
 from oslo_log import log as logging
@@ -36,7 +36,7 @@ def get_health():
     # tenant is still active in Keystone, we believe it should be investigated.
     failed_collected_count = 0
     for p in projects:
-        delta = (datetime.now() - p.last_collected).total_seconds() // 3600
+        delta = (dt.now() - p.last_collected).total_seconds() // 3600
         if delta >= 24 and p.id in project_id_list_keystone:
             failed_collected_count += 1
 

distil/utils/constants.py

@@ -32,7 +32,7 @@ date_format_f = "%Y-%m-%dT%H:%M:%S.%f"
 # Some useful constants
 iso_time = "%Y-%m-%dT%H:%M:%S"
 iso_date = "%Y-%m-%d"
-dawn_of_time = datetime(2014, 4, 1)
+dawn_of_time = datetime(2016, 5, 10)
 
 # VM states:
 states = {'active': 1,

distil/utils/odoo.py

@@ -54,7 +54,7 @@ class Odoo(object):
         for r in regions:
             prices[r] = {}
             for category in PRODUCT_CATEGORY:
-                prices[r][category.lower()] = {}
+                prices[r][category.lower()] = []
                 c = self.category.search([('name', '=', category),
                                           ('display_name', 'ilike', r)])
                 product_ids = self.product.search([('categ_id', '=', c[0]),
@@ -65,15 +65,17 @@ class Odoo(object):
                                              product_ids)
                 for p in products:
                     name = p['name_template'][len(r) + 1:]
+                    if 'pre-prod' in name:
+                        continue
                     price = round(p['lst_price'], 5)
                     # NOTE(flwang): default_code is Internal Reference on Odoo
                     # GUI
                     unit = p['default_code']
                     desc = p['description']
-                    prices[r][category.lower()][name] = {'price': price,
+                    prices[r][category.lower()].append({'resource': name,
+                                                        'price': price,
                                                         'unit': unit,
-                                                         'description': desc
+                                                        'description': desc})
-                                                         }
 
         return prices
 

etc/distil.conf.sample

@@ -11,8 +11,8 @@ dawn_of_time = 2016-05-18 01:00:00
 meter_mappings_file = /etc/distil/meter_mappings.yml
 
 [rater]
-type = file
+rater_type = odoo
-rates_file = /etc/distil/rates.csv
+rate_file_path = /etc/distil/rates.csv
 
 [odoo]
 version=8.0

requirements.txt

@@ -13,24 +13,17 @@ keystonemiddleware!=4.1.0,>=4.0.0 # Apache-2.0
 keystoneauth1>=2.1.0  # Apache-2.0
 
 python-cinderclient>=1.6.0 # Apache-2.0
-python-keystoneclient>=1.7.0,!=1.8.0,!=2.1.0  # Apache-2.0
+python-keystoneclient!=1.8.0,!=2.1.0,>=1.6.0 # Apache-2.0
-python-manilaclient>=1.3.0 # Apache-2.0
 python-novaclient!=2.33.0,>=2.29.0 # Apache-2.0
-python-swiftclient>=2.2.0 # Apache-2.0
 python-neutronclient>=4.2.0 # Apache-2.0
-python-heatclient>=0.6.0 # Apache-2.0
 python-ceilometerclient>=2.2.1  # Apache-2.0
+python-glanceclient>=2.0.0  # Apache-2.0
 
 oslo.config>=3.9.0 # Apache-2.0
-oslo.concurrency>=3.5.0 # Apache-2.0
 oslo.context>=2.2.0 # Apache-2.0
 oslo.db>=4.1.0 # Apache-2.0
 oslo.i18n>=2.1.0 # Apache-2.0
 oslo.log>=1.14.0 # Apache-2.0
-oslo.messaging>=4.5.0 # Apache-2.0
-oslo.middleware>=3.0.0 # Apache-2.0
-oslo.policy>=0.5.0 # Apache-2.0
-oslo.rootwrap>=2.0.0 # Apache-2.0
 oslo.serialization>=1.10.0 # Apache-2.0
 oslo.service>=1.0.0 # Apache-2.0
 oslo.utils>=3.5.0 # Apache-2.0

setup.cfg

@@ -27,8 +27,8 @@ data_files =
 
 [entry_points]
 console_scripts =
-    distil-api = distil.cli.distil_api:main
+    distil-api = distil.cmd.distil_api:main
-    distil-collector = distil.cli.distil_collector:main
+    distil-collector = distil.cmd.distil_collector:main
     distil-db-manage = distil.db.migration.cli:main
 
 distil.collector =