x/distil
Code Issues Proposed changes

Implement /costs on new service

Browse Source 
Change-Id: I12469ec39fcaa79d6588d396367f14107cf4299f
This commit is contained in:
Fei Long Wang 2016-05-10 15:23:11 +12:00 committed by Lingxian Kong
parent 6c9962b582
commit 9024534306
20 changed files with 309 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
distil/api/auth.py
View File

@ -14,7 +14,6 @@
# limitations under the License.
from keystonemiddleware import auth_token
from keystonemiddleware import opts
from oslo_config import cfg
from oslo_log import log as logging
import re
@ -23,22 +22,6 @@ from distil import exceptions
CONF = cfg.CONF
AUTH_GROUP_NAME = 'keystone_authtoken'
def _register_opts():
options = []
keystone_opts = opts.list_auth_token_opts()
for n in keystone_opts:
if (n[0] == AUTH_GROUP_NAME):
options = n[1]
break
CONF.register_opts(options, group=AUTH_GROUP_NAME)
auth_token.CONF = CONF
_register_opts()
LOG = logging.getLogger(__name__)

9
distil/api/v2.py
View File

@ -34,7 +34,14 @@ def prices_get():
@rest.get('/costs')
def costs_get():
return api.render(costs=costs.get_costs())
# NOTE(flwang): Get 'tenant' first for backward compatibility.
tenant_id = api.get_request_args().get('tenant', None)
project_id = api.get_request_args().get('project_id', tenant_id)
start = api.get_request_args().get('start', None)
end = api.get_request_args().get('end', None)
# NOTE(flwang): Here using 'usage' instead of 'costs' for backward
# compatibility.
return api.render(usage=costs.get_costs(project_id, start, end))
@rest.get('/usages')

0
distil/cli/__init__.py → distil/cmd/__init__.py
View File

0
distil/cli/distil_api.py → distil/cmd/distil_api.py
View File

0
distil/cli/distil_collector.py → distil/cmd/distil_collector.py
View File

47
distil/common/constants.py Normal file
View File

@ -0,0 +1,47 @@
# Copyright (C) 2014 Catalyst IT Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from datetime import datetime
# Date format Ceilometer uses
# 2013-07-03T13:34:17
# which is, as an strftime:
# timestamp = datetime.strptime(res["timestamp"], "%Y-%m-%dT%H:%M:%S.%f")
# or
# timestamp = datetime.strptime(res["timestamp"], "%Y-%m-%dT%H:%M:%S")
# Most of the time we use date_format
date_format = "%Y-%m-%dT%H:%M:%S"
# Sometimes things also have milliseconds, so we look for that too.
# Because why not be annoying in all the ways?
other_date_format = "%Y-%m-%dT%H:%M:%S.%f"
# Some useful constants
iso_time = "%Y-%m-%dT%H:%M:%S"
iso_date = "%Y-%m-%d"
dawn_of_time = datetime(2014, 4, 1)
# VM states (SOON TO BE REMOVED):
states = {'active': 1,
'building': 2,
'paused': 3,
'suspended': 4,
'stopped': 5,
'rescued': 6,
'resized': 7,
'soft_deleted': 8,
'deleted': 9,
'error': 10,
'shelved': 11,
'shelved_offloaded': 12}

22
distil/config.py
View File

@ -33,9 +33,13 @@ DEFAULT_OPTIONS = (
default=['/', '/v2/prices', '/v2/health'],
help='The list of public API routes',
),
cfg.ListOpt('ignore_tenants',
default=[],
help=('The tenant name list which will be ignored when '
'collecting metrics from Ceilometer.')),
)
COLLECTOR_OPTIONS = [
COLLECTOR_OPTS = [
cfg.IntOpt('periodic_interval', default=3600,
help=('Interval of usage collection.')),
cfg.StrOpt('collector_backend', default='ceilometer',
@ -69,17 +73,29 @@ ODOO_OPTS = [
help='Name of the Odoo database.'),
cfg.StrOpt('user',
help='Name of Odoo account to login.'),
cfg.StrOpt('password',
cfg.StrOpt('password', secret=True,
help='Password of Odoo account to login.'),
cfg.StrOpt('last_update',
help='Last time when the products/prices are updated.')
]
RATER_OPTS = [
cfg.StrOpt('rater_type', default='odoo',
help='Rater type, by default it is odoo.'),
cfg.StrOpt('rate_file_path', default='/etc/distil/rates.csv',
help='Rate file path, it will be used when the rater_type '
'is "file".'),
]
AUTH_GROUP = 'keystone_authtoken'
ODOO_GROUP = 'odoo'
COLLECTOR_GROUP = 'collector'
RATER_GROUP = 'rater'
CONF.register_opts(DEFAULT_OPTIONS)
CONF.register_opts(ODOO_OPTS, group=ODOO_GROUP)
CONF.register_opts(COLLECTOR_OPTIONS, group=COLLECTOR_GROUP)
CONF.register_opts(COLLECTOR_OPTS, group=COLLECTOR_GROUP)
CONF.register_opts(RATER_OPTS, group=RATER_GROUP)
def _register_keystoneauth_opts(conf):

9
distil/db/api.py
View File

@ -80,7 +80,6 @@ def to_dict(func):
return decorator
@to_dict
def usage_get(project_id, start_at, end_at):
"""Get usage for specific tenant based on time range.
@ -110,5 +109,13 @@ def project_add(values):
return IMPL.project_add(values)
def resource_get_by_ids(project_id, resource_ids):
return IMPL.resource_get_by_ids(project_id, resource_ids)
def project_get(project_id):
return IMPL.project_get(project_id)
def project_get_all():
return IMPL.project_get_all()

4
distil/db/migration/alembic_migrations/versions/001_init_tables.py
View File

@ -68,9 +68,9 @@ def upgrade():
primary_key=True, nullable=False),
sa.Column('resource_id', sa.String(length=100),
primary_key=True, nullable=False),
sa.Column('start_at', sa.DateTime(), primary_key=True,
sa.Column('start', sa.DateTime(), primary_key=True,
nullable=True),
sa.Column('end_at', sa.DateTime(), primary_key=True,
sa.Column('end', sa.DateTime(), primary_key=True,
nullable=True),
sa.Column('created', sa.DateTime(), nullable=True),
sa.ForeignKeyConstraint(['tenant_id'], ['tenants.id'], ),

68
distil/db/sqlalchemy/api.py
View File

@ -20,11 +20,14 @@ import json
import six
import sys
import sqlalchemy as sa
from sqlalchemy import func
import threading
from oslo_config import cfg
from oslo_db import exception as db_exception
from oslo_db.sqlalchemy import session as db_session
from oslo_log import log as logging
import sqlalchemy as sa
from distil.db.sqlalchemy import models as m
from distil.db.sqlalchemy.models import Resource
@ -37,18 +40,17 @@ LOG = logging.getLogger(__name__)
CONF = cfg.CONF
_FACADE = None
_LOCK = threading.Lock()
def _create_facade_lazily():
global _FACADE
global _LOCK, _FACADE
if _FACADE is None:
params = dict(CONF.database.iteritems())
params["sqlite_fk"] = True
_FACADE = db_session.EngineFacade(
CONF.database.connection,
**params
)
with _LOCK:
if _FACADE is None:
_FACADE = db_session.EngineFacade.from_config(CONF,
sqlite_fk=True)
return _FACADE
@ -141,15 +143,43 @@ def project_get_all():
return query.all()
def usage_get(project_id, start_at, end_at):
def project_get(project_id):
session = get_session()
query = session.query(UsageEntry)
query = session.query(Tenant)
query = query.filter(Tenant.id == project_id)
try:
return query.one()
except Exception:
raise exceptions.NotFoundException(project_id)
query = (query.filter(UsageEntry.start_at >= start_at,
UsageEntry.end_at <= end_at).
filter(UsageEntry.project_id == project_id))
return query.all()
def usage_get(project_id, start, end):
session = get_session()
query = session.query(UsageEntry.tenant_id,
UsageEntry.resource_id,
UsageEntry.service,
UsageEntry.unit,
func.sum(UsageEntry.volume).label("volume"))
query = (query.filter(UsageEntry.start >= start,
UsageEntry.end <= end).
filter(UsageEntry.tenant_id == project_id).
group_by(UsageEntry.tenant_id, UsageEntry.resource_id,
UsageEntry.service, UsageEntry.unit))
result = []
# NOTE(flwang): With group_by and func.sum, the query result is a list of
# array, which is hard to be consumed. So Here we're using a named tuple
# so that it can be easier to use.
for entry in query.all():
ue = UsageEntry()
ue.tenant_id = entry[0]
ue.resource_id = entry[1]
ue.service = entry[2]
ue.unit = entry[3]
ue.volume = entry[4]
result.append(ue)
return result
def usage_add(project_id, resource_id, samples, unit,
@ -243,6 +273,16 @@ def resource_add(project_id, resource_id, resource_type, raw, metadata):
raise e
def resource_get_by_ids(project_id, resource_ids):
session = get_session()
query = session.query(Resource)
query = (query.filter(Resource.id.in_(resource_ids)).
filter(Resource.tenant_id == project_id))
return query.all()
def _merge_resource_metadata(md_dict, entry, md_def):
"""Strips metadata from the entry as defined in the config,
and merges it with the given metadata dict.

19
distil/rater/__init__.py
View File

@ -13,11 +13,28 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from oslo_config import cfg
from stevedore import driver
CONF = cfg.CONF
RATER = None
class BaseRater(object):
def __init__(self, conf):
def __init__(self, conf=None):
self.conf = conf
def rate(self, name, region=None):
raise NotImplementedError("Not implemented in base class")
def get_rater():
global RATER
if RATER is None:
RATER = driver.DriverManager('distil.rater',
CONF.rater.rater_type,
invoke_on_load=True,
invoke_kwds={}).driver
return RATER

17
distil/rater/odoo.py
View File

@ -20,19 +20,20 @@ from distil.utils import odoo
class OdooRater(rater.BaseRater):
def __init__(self, conf):
def __init__(self):
self.prices = odoo.Odoo().get_prices()
def rate(self, name, region=None):
if not self.prices:
return rate_file.FileRater().rate(name, region)
region_prices = (self.prices[region] if region else
self.prices.values[0])
for region in self.prices:
for category in self.prices[region]:
for product in self.prices[region][category]:
if product == name:
return {'rate': self.prices[name]['price'],
'unit': self.prices[name]['unit']
}
for category in region_prices:
for product in region_prices[category]:
if product['resource'] == name:
return {'rate': product['price'],
'unit': product['unit']
}
return rate_file.FileRater().rate(name, region)

13
distil/rater/rate_file.py
View File

@ -16,17 +16,20 @@
import csv
from decimal import Decimal
import logging as log
from oslo_config import cfg
import oslo_log as log
from distil import rater
from distil import exceptions
CONF = cfg.CONF
class FileRater(rater.BaseRater):
def __init__(self, conf):
super(FileRater, self).__init__(conf)
def __init__(self):
try:
with open(self.config['file']) as fh:
with open(CONF.rater.rate_file_path) as fh:
# Makes no opinions on the file structure
reader = csv.reader(fh, delimiter="|")
self.__rates = {
@ -38,7 +41,7 @@ class FileRater(rater.BaseRater):
}
except Exception as e:
log.critical('Failed to load rates file: `%s`' % e)
raise
exceptions.InvalidConfig(e)
def rate(self, name, region=None):
return {

121
distil/service/api/v2/costs.py
View File

@ -13,12 +13,129 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import json
from datetime import datetime
from decimal import Decimal
from oslo_config import cfg
from oslo_log import log as logging
from distil import rater
from distil.common import constants
from distil.db import api as db_api
from distil.utils import general
LOG = logging.getLogger(__name__)
CONF = cfg.CONF
def get_costs():
return {'id': 1}
def get_costs(project_id, start, end):
try:
if start is not None:
try:
start = datetime.strptime(start, constants.iso_date)
except ValueError:
start = datetime.strptime(start, constants.iso_time)
else:
return 400, {"missing parameter": {"start": "start date" +
" in format: y-m-d"}}
if not end:
end = datetime.utcnow()
else:
try:
end = datetime.strptime(end, constants.iso_date)
except ValueError:
end = datetime.strptime(end, constants.iso_time)
except ValueError:
return 400, {
"errors": ["'end' date given needs to be in format: " +
"y-m-d, or y-m-dTH:M:S"]}
if end <= start:
return 400, {"errors": ["end date must be greater than start."]}
project = db_api.project_get(project_id)
LOG.debug("Calculating rated data for %s in range: %s - %s" %
(project.id, start, end))
costs = _calculate_cost(project, start, end)
return costs
def _calculate_cost(project, start, end):
"""Calculate a rated data dict from the given range."""
usage = db_api.usage_get(project.id, start, end)
# Transform the query result into a billable dict.
project_dict = _build_project_dict(project, usage)
project_dict = _add_costs_for_project(project_dict)
# add sales order range:
project_dict['start'] = str(start)
project_dict['end'] = str(end)
return project_dict
def _build_project_dict(project, usage):
"""Builds a dict structure for a given project."""
project_dict = {'name': project.name, 'tenant_id': project.id}
all_resource_ids = [entry.get('resource_id') for entry in usage]
res_list = db_api.resource_get_by_ids(project.id, all_resource_ids)
project_dict['resources'] = {row.id: json.loads(row.info)
for row in res_list}
for entry in usage:
service = {'name': entry.get('service'),
'volume': entry.get('volume'),
'unit': entry.get('unit')}
resource = project_dict['resources'][entry.get('resource_id')]
service_list = resource.setdefault('services', [])
service_list.append(service)
return project_dict
def _add_costs_for_project(project):
"""Adds cost values to services using the given rates manager."""
current_rater = rater.get_rater()
project_total = 0
for resource in project['resources'].values():
resource_total = 0
for service in resource['services']:
try:
rate = current_rater.rate(service['name'])
except KeyError:
# no rate exists for this service
service['cost'] = "0"
service['volume'] = "unknown unit conversion"
service['unit'] = "unknown"
service['rate'] = "missing rate"
continue
volume = general.convert_to(service['volume'],
service['unit'],
rate['unit'])
# round to 2dp so in dollars.
cost = round(volume * Decimal(rate['rate']), 2)
service['cost'] = str(cost)
service['volume'] = str(volume)
service['unit'] = rate['unit']
service['rate'] = str(rate['rate'])
resource_total += cost
resource['total_cost'] = str(resource_total)
project_total += resource_total
project['total_cost'] = str(project_total)
return project

4
distil/service/api/v2/health.py
View File

@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import datetime
from datetime import datetime as dt
from oslo_config import cfg
from oslo_log import log as logging
@ -36,7 +36,7 @@ def get_health():
# tenant is still active in Keystone, we believe it should be investigated.
failed_collected_count = 0
for p in projects:
delta = (datetime.now() - p.last_collected).total_seconds() // 3600
delta = (dt.now() - p.last_collected).total_seconds() // 3600
if delta >= 24 and p.id in project_id_list_keystone:
failed_collected_count += 1

2
distil/utils/constants.py
View File

@ -32,7 +32,7 @@ date_format_f = "%Y-%m-%dT%H:%M:%S.%f"
# Some useful constants
iso_time = "%Y-%m-%dT%H:%M:%S"
iso_date = "%Y-%m-%d"
dawn_of_time = datetime(2014, 4, 1)
dawn_of_time = datetime(2016, 5, 10)
# VM states:
states = {'active': 1,

12
distil/utils/odoo.py
View File

@ -54,7 +54,7 @@ class Odoo(object):
for r in regions:
prices[r] = {}
for category in PRODUCT_CATEGORY:
prices[r][category.lower()] = {}
prices[r][category.lower()] = []
c = self.category.search([('name', '=', category),
('display_name', 'ilike', r)])
product_ids = self.product.search([('categ_id', '=', c[0]),
@ -65,15 +65,17 @@ class Odoo(object):
product_ids)
for p in products:
name = p['name_template'][len(r) + 1:]
if 'pre-prod' in name:
continue
price = round(p['lst_price'], 5)
# NOTE(flwang): default_code is Internal Reference on Odoo
# GUI
unit = p['default_code']
desc = p['description']
prices[r][category.lower()][name] = {'price': price,
'unit': unit,
'description': desc
}
prices[r][category.lower()].append({'resource': name,
'price': price,
'unit': unit,
'description': desc})
return prices

6
etc/distil.conf.sample
View File

@ -11,8 +11,8 @@ dawn_of_time = 2016-05-18 01:00:00
meter_mappings_file = /etc/distil/meter_mappings.yml
[rater]
type = file
rates_file = /etc/distil/rates.csv
rater_type = odoo
rate_file_path = /etc/distil/rates.csv
[odoo]
version=8.0
@ -30,7 +30,7 @@ backend = sqlalchemy
[keystone_authtoken]
memcache_servers = 127.0.0.1:11211
signing_dir = /var/cache/distil
cafile = /opt/stack/data/ca-bundle.pem
cafile = /opt/stack/data/ca-bundle.pem
auth_uri = http://127.0.0.1:5000
project_domain_id = default
project_name = service

11
requirements.txt
View File

@ -13,24 +13,17 @@ keystonemiddleware!=4.1.0,>=4.0.0 # Apache-2.0
keystoneauth1>=2.1.0 # Apache-2.0
python-cinderclient>=1.6.0 # Apache-2.0
python-keystoneclient>=1.7.0,!=1.8.0,!=2.1.0 # Apache-2.0
python-manilaclient>=1.3.0 # Apache-2.0
python-keystoneclient!=1.8.0,!=2.1.0,>=1.6.0 # Apache-2.0
python-novaclient!=2.33.0,>=2.29.0 # Apache-2.0
python-swiftclient>=2.2.0 # Apache-2.0
python-neutronclient>=4.2.0 # Apache-2.0
python-heatclient>=0.6.0 # Apache-2.0
python-ceilometerclient>=2.2.1 # Apache-2.0
python-glanceclient>=2.0.0 # Apache-2.0
oslo.config>=3.9.0 # Apache-2.0
oslo.concurrency>=3.5.0 # Apache-2.0
oslo.context>=2.2.0 # Apache-2.0
oslo.db>=4.1.0 # Apache-2.0
oslo.i18n>=2.1.0 # Apache-2.0
oslo.log>=1.14.0 # Apache-2.0
oslo.messaging>=4.5.0 # Apache-2.0
oslo.middleware>=3.0.0 # Apache-2.0
oslo.policy>=0.5.0 # Apache-2.0
oslo.rootwrap>=2.0.0 # Apache-2.0
oslo.serialization>=1.10.0 # Apache-2.0
oslo.service>=1.0.0 # Apache-2.0
oslo.utils>=3.5.0 # Apache-2.0

4
setup.cfg
View File

@ -27,8 +27,8 @@ data_files =
[entry_points]
console_scripts =
distil-api = distil.cli.distil_api:main
distil-collector = distil.cli.distil_collector:main
distil-api = distil.cmd.distil_api:main
distil-collector = distil.cmd.distil_collector:main
distil-db-manage = distil.db.migration.cli:main
distil.collector =