x/distil
Code Issues Proposed changes

Merge "Support JSON file as an ERP driver"

Browse Source
This commit is contained in:
Zuul 2019-03-11 22:54:15 +00:00 committed by Gerrit Code Review
commit aafa2c0038
6 changed files with 752 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
distil/config.py
View File

@ -115,6 +115,12 @@ ODOO_OPTS = [
'c1.c1r2-windows or c1.c2r4-sql-server-standard-windows')
]
JSONFILE_OPTS = [
cfg.StrOpt('products_file_path',
default='/etc/distil/products.json',
help='Json file to contain the products and prices.'),
]
CLI_OPTS = [
cfg.StrOpt(
@ -128,10 +134,12 @@ CLI_OPTS = [
AUTH_GROUP = 'keystone_authtoken'
ODOO_GROUP = 'odoo'
COLLECTOR_GROUP = 'collector'
JSONFILE_GROUP = 'jsonfile'
CONF.register_opts(DEFAULT_OPTIONS)
CONF.register_opts(ODOO_OPTS, group=ODOO_GROUP)
CONF.register_opts(JSONFILE_OPTS, group=JSONFILE_GROUP)
CONF.register_opts(COLLECTOR_OPTS, group=COLLECTOR_GROUP)
CONF.register_cli_opts(CLI_OPTS)

323
distil/erp/drivers/jsonfile.py Normal file
View File

@ -0,0 +1,323 @@
# Copyright 2019 Catalyst IT Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import collections
import copy
from datetime import datetime
from datetime import timedelta
import decimal
import os
import json
import itertools
from oslo_config import cfg
from oslo_log import log
from distil.common import cache
from distil.common import constants
from distil import exceptions
from distil.common import general
from distil.db import api as db_api
from distil.erp import driver
LOG = log.getLogger(__name__)
CONF = cfg.CONF
class JsonFileDriver(driver.BaseDriver):
"""Json driver
"""
conf = None
def __init__(self, conf):
self.conf = conf
def _load_products(self):
try:
with open(self.conf['jsonfile']['products_file_path']) as fh:
products = json.load(fh)
return products
except Exception as e:
LOG.critical('Failed to load rates json file: `%s`' % e)
raise e
def is_healthy(self):
"""Check if the ERP back end is healthy or not
:returns True if the ERP is healthy, otherwise False
"""
try:
p = self._load_products()
return p is not None
except Exception as e:
LOG.error(e)
return False
def get_products(self, regions=[]):
"""List products based on given regions
:param regions: List of regions to get projects
:returns Dict of products based on the given regions
"""
# TODO(flwang): Apply regions
products = self._load_products()
if regions:
region_products = {}
for r in regions:
if r in products:
region_products[r] = products[r]
return region_products
return products
def create_product(self, product):
"""Create product in ERP backend.
:param product: info used to create product
"""
raise NotImplementedError()
def get_credits(self, project_id, expiry_date):
"""Get project credits
:param project_id: Project ID
:param expiry_date: Any credit which expires after this date can be
listed
:returns list of credits current project can get
"""
raise NotImplementedError()
def create_credit(self, project, credit):
"""Create credit for a given project
:param project: project
"""
raise NotImplementedError()
def _get_invoice_time_ranges(self, start, end):
previous_months = []
indicator = end
while start <= indicator - timedelta(seconds=1):
end_inv = indicator.replace(
day=1, hour=0, minute=0, second=0, microsecond=0)
last_month = indicator.month - 1
if last_month:
indicator = indicator.replace(
month=last_month, day=1, hour=0, minute=0, second=0,
microsecond=0)
else:
last_year = indicator.year - 1
indicator = indicator.replace(
year=last_year, month=12, day=1, hour=0, minute=0,
second=0, microsecond=0)
bill_date = end_inv - timedelta(seconds=1)
previous_months.append((
indicator,
end_inv,
bill_date.strftime('%Y-%m-%d')
))
return previous_months
def get_invoices(self, start, end, project_id, detailed=False):
"""Get history invoices from ERP service given a time range.
This will only get the history for the current given region.
:param start: Start time, a datetime object.
:param end: End time, a datetime object.
:param project_id: project ID.
:param detailed: If get detailed information or not.
:return: The history invoices information for each month.
"""
region_name = CONF.keystone_authtoken.region_name
previous_months = self._get_invoice_time_ranges(start, end)
invoices = {}
for start_inv, end_inv, bill_date_str in previous_months:
usage = db_api.usage_get(project_id, start_inv, end_inv)
all_resource_ids = set(
[entry.get('resource_id') for entry in usage])
res_list = db_api.resource_get_by_ids(project_id, all_resource_ids)
quotation = self.get_quotations(
region_name,
project_id,
measurements=usage,
resources=res_list,
detailed=detailed
)
quotation['status'] = 'paid'
invoices[bill_date_str] = quotation
return invoices
@cache.memoize
def _get_service_mapping(self, products):
"""Gets mapping from service name to service type.
:param products: Product dict in a region returned from odoo.
"""
srv_mapping = {}
for category, p_list in products.items():
for p in p_list:
srv_mapping[p['name']] = category.title()
return srv_mapping
@cache.memoize
def _get_service_price(self, service_name, service_type, products):
"""Get service price information from price definitions."""
price = {'service_name': service_name}
if service_type in products:
for s in products[service_type]:
if s['name'] == service_name:
price.update({'rate': s['rate'], 'unit': s['unit']})
break
else:
found = False
for category, services in products.items():
for s in services:
if s['name'] == service_name:
price.update({'rate': s['rate'], 'unit': s['unit']})
found = True
break
if not found:
raise exceptions.NotFoundException(
'Price not found, service name: %s, service type: %s' %
(service_name, service_type)
)
return price
def _get_entry_info(self, entry, resources_info, service_mapping):
service_name = entry.get('service')
volume = entry.get('volume')
unit = entry.get('unit')
res_id = entry.get('resource_id')
resource = resources_info.get(res_id, {})
# resource_type is the type defined in meter_mappings.yml.
resource_type = resource.get('type')
service_type = service_mapping.get(service_name, resource_type)
return (service_name, service_type, volume, unit, resource,
resource_type)
def get_quotations(self, region, project_id, measurements=[], resources=[],
detailed=False):
"""Get current month quotation.
Return value is in the following format:
{
'<current_date>': {
'total_cost': 100,
'details': {
'Compute': {
'total_cost': xxx,
'breakdown': {}
}
}
}
}
:param region: Region name.
:param project_id: Project ID.
:param measurements: Current month usage collection.
:param resources: List of resources.
:param detailed: If get detailed information or not.
:return: Current month quotation.
"""
total_cost = 0
price_mapping = {}
cost_details = {}
resources_info = {}
for row in resources:
info = json.loads(row.info)
info.update({'id': row.id})
resources_info[row.id] = info
# NOTE(flwang): For most of the cases of Distil API, the request comes
# from billing panel. Billing panel sends 1 API call for /invoices and
# several API calls for /quotations against different regions. So it's
# not efficient to specify the region for get_products method because
# it won't help cache the products based on the parameters.
products = self.get_products()[region]
service_mapping = self._get_service_mapping(products)
for entry in measurements:
(service_name, service_type, volume, unit, resource,
resource_type) = self._get_entry_info(entry, resources_info,
service_mapping)
res_id = resource['id']
if service_type not in cost_details:
cost_details[service_type] = {
'total_cost': 0,
'breakdown': collections.defaultdict(list)
}
if service_name not in price_mapping:
price_spec = self._get_service_price(
service_name, service_type, products
)
price_mapping[service_name] = price_spec
price_spec = price_mapping[service_name]
# Convert volume according to unit in price definition.
volume = float(
general.convert_to(volume, unit, price_spec['unit'])
)
cost = (round(volume * price_spec['rate'], constants.PRICE_DIGITS)
if price_spec['rate'] else 0)
total_cost += cost
if detailed:
erp_service_name = "%s.%s" % (region, service_name)
cost_details[service_type]['total_cost'] = round(
(cost_details[service_type]['total_cost'] + cost),
constants.PRICE_DIGITS
)
cost_details[service_type]['breakdown'][
erp_service_name
].append(
{
"resource_name": resource.get('name', ''),
"resource_id": res_id,
"cost": cost,
"quantity": round(volume, 3),
"rate": round(price_spec['rate'],
constants.RATE_DIGITS),
"unit": price_spec['unit'],
}
)
result = {
'total_cost': round(float(total_cost), constants.PRICE_DIGITS)
}
if detailed:
result.update({'details': cost_details})
return result

1
distil/service/api/v2/quotations.py
View File

@ -52,7 +52,6 @@ def get_quotations(project_id, detailed=False):
usage = db_api.usage_get(project_id, start, end)
all_resource_ids = set([entry.get('resource_id') for entry in usage])
res_list = db_api.resource_get_by_ids(project_id, all_resource_ids)
erp_driver = erp_utils.load_erp_driver(CONF)
quotations = erp_driver.get_quotations(
region_name,

316
distil/tests/unit/erp/drivers/test_jsonfile.py Normal file
View File

@ -0,0 +1,316 @@
# Copyright (c) 2019 Catalyst Cloud Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from collections import namedtuple
from datetime import datetime
from decimal import Decimal
import mock
from distil.erp.drivers import jsonfile
from distil.tests.unit import base
REGION = namedtuple('Region', ['id'])
PRODUCTS = [
{
'id': 1,
'categ_id': [1, 'All products (.NET) / nz-1 / Compute'],
'name_template': 'nz-1.c1.c1r1',
'lst_price': 0.00015,
'default_code': 'hour',
'description': '1 CPU, 1GB RAM'
},
{
'id': 2,
'categ_id': [2, 'All products (.NET) / nz-1 / Network'],
'name_template': 'nz-1.n1.router',
'lst_price': 0.00025,
'default_code': 'hour',
'description': 'Router'
},
{
'id': 3,
'categ_id': [1, 'All products (.NET) / nz-1 / Block Storage'],
'name_template': 'nz-1.b1.volume',
'lst_price': 0.00035,
'default_code': 'hour',
'description': 'Block storage'
}
]
class TestJsonFileDriver(base.DistilTestCase):
config_file = 'distil.conf'
def test_get_products(self):
pass
@mock.patch('distil.erp.drivers.jsonfile.db_api')
@mock.patch('distil.erp.drivers.jsonfile.JsonFileDriver.get_quotations')
def test_get_invoices_without_details(self, mock_get_quotations,
mock_db_api):
start = datetime(2017, 4, 1)
end = datetime(2018, 3, 1)
fake_project = '123'
mock_get_quotations.return_value = {'total_cost': 20}
mock_db_api.usage_get.return_value = []
mock_db_api.resource_get_by_ids.return_value = []
jsondriver = jsonfile.JsonFileDriver(self.conf)
invoices = jsondriver.get_invoices(start, end, fake_project)
self.assertEqual(
{
'2017-04-30': {'status': 'paid', 'total_cost': 20},
'2017-05-31': {'status': 'paid', 'total_cost': 20},
'2017-06-30': {'status': 'paid', 'total_cost': 20},
'2017-07-31': {'status': 'paid', 'total_cost': 20},
'2017-08-31': {'status': 'paid', 'total_cost': 20},
'2017-09-30': {'status': 'paid', 'total_cost': 20},
'2017-10-31': {'status': 'paid', 'total_cost': 20},
'2017-11-30': {'status': 'paid', 'total_cost': 20},
'2017-12-31': {'status': 'paid', 'total_cost': 20},
'2018-01-31': {'status': 'paid', 'total_cost': 20},
'2018-02-28': {'status': 'paid', 'total_cost': 20}
},
invoices
)
@mock.patch('distil.erp.drivers.jsonfile.db_api')
@mock.patch('distil.erp.drivers.jsonfile.JsonFileDriver.get_products')
@mock.patch('distil.erp.drivers.jsonfile.CONF')
def test_get_invoices_with_details(self, mock_conf, mock_get_products,
mock_db_api):
start = datetime(2018, 2, 1)
end = datetime(2018, 3, 1)
fake_project = '123'
mock_conf.keystone_authtoken.region_name = "nz-1"
mock_get_products.return_value = {
'nz-1': {
'Compute': [
{
'name': 'c1.c1r1', 'description': 'c1.c1r1',
'rate': 0.01, 'unit': 'hour'
}
]
}
}
# mock_get_quotations.return_value = {'total_cost': 20}
mock_db_api.usage_get.return_value = [
{'service': 'c1.c1r1', 'volume': 500, 'unit': 'hour',
'resource_id': '47aa'}
]
mock_db_api.resource_get_by_ids.return_value = [
mock.Mock(
id='47aa',
info='{"name": "test-1", "type": "Virtual Machine"}')
]
jsondriver = jsonfile.JsonFileDriver(self.conf)
invoices = jsondriver.get_invoices(
start, end, fake_project, detailed=True)
self.assertEqual(
{
'2018-02-28': {
'details': {
'Compute': {
'breakdown': {
'nz-1.c1.c1r1': [
{
'rate': 0.01,
'resource_id': '47aa',
'cost': 5.0,
'unit': 'hour',
'quantity': 500.0,
'resource_name': 'test-1'
}
]
},
'total_cost': 5.0
}
},
'total_cost': 5.0,
'status': 'paid',
}
},
invoices
)
@mock.patch('distil.erp.drivers.jsonfile.JsonFileDriver.get_products')
def test_get_quotations_without_details(self, mock_get_products):
mock_get_products.return_value = {
'nz-1': {
'Compute': [
{
'name': 'c1.c2r16', 'description': 'c1.c2r16',
'rate': 0.01, 'unit': 'hour'
}
],
'Block Storage': [
{
'name': 'b1.standard', 'description': 'b1.standard',
'rate': 0.02, 'unit': 'gigabyte'
}
]
}
}
class Resource(object):
def __init__(self, id, info):
self.id = id
self.info = info
resources = [
Resource(1, '{"name": "", "type": "Volume"}'),
Resource(2, '{"name": "", "type": "Virtual Machine"}')
]
usage = [
{
'service': 'b1.standard',
'resource_id': 1,
'volume': 1024 * 1024 * 1024,
'unit': 'byte',
},
{
'service': 'c1.c2r16',
'resource_id': 2,
'volume': 3600,
'unit': 'second',
}
]
jf = jsonfile.JsonFileDriver(self.conf)
quotations = jf.get_quotations(
'nz-1', 'fake_id', measurements=usage, resources=resources
)
self.assertEqual(
{'total_cost': 0.03},
quotations
)
@mock.patch('distil.erp.drivers.jsonfile.JsonFileDriver.get_products')
def test_get_quotations_with_details(self, mock_get_products):
mock_get_products.return_value = {
'nz-1': {
'Compute': [
{
'name': 'c1.c2r16', 'description': 'c1.c2r16',
'rate': 0.01, 'unit': 'hour'
}
],
'Block Storage': [
{
'name': 'b1.standard', 'description': 'b1.standard',
'rate': 0.02, 'unit': 'gigabyte'
}
]
}
}
class Resource(object):
def __init__(self, id, info):
self.id = id
self.info = info
resources = [
Resource(1, '{"name": "volume1", "type": "Volume"}'),
Resource(2, '{"name": "instance2", "type": "Virtual Machine"}')
]
usage = [
{
'service': 'b1.standard',
'resource_id': 1,
'volume': 1024 * 1024 * 1024,
'unit': 'byte',
},
{
'service': 'c1.c2r16',
'resource_id': 2,
'volume': 3600,
'unit': 'second',
}
]
jf = jsonfile.JsonFileDriver(self.conf)
quotations = jf.get_quotations(
'nz-1', 'fake_id', measurements=usage, resources=resources,
detailed=True
)
self.assertDictEqual(
{
'total_cost': 0.03,
'details': {
'Compute': {
'total_cost': 0.01,
'breakdown': {
'nz-1.c1.c2r16': [
{
"resource_name": "instance2",
"resource_id": 2,
"cost": 0.01,
"quantity": 1.0,
"rate": 0.01,
"unit": "hour",
}
],
}
},
'Block Storage': {
'total_cost': 0.02,
'breakdown': {
'nz-1.b1.standard': [
{
"resource_name": "volume1",
"resource_id": 1,
"cost": 0.02,
"quantity": 1.0,
"rate": 0.02,
"unit": "gigabyte",
}
]
}
}
}
},
quotations
)
def test_is_healthy(self):
def _fake_load_products():
return {'a': 1}
jf = jsonfile.JsonFileDriver(self.conf)
jf._load_products = _fake_load_products
self.assertTrue(jf.is_healthy())
def test_is_healthy_false(self):
def _fake_load_products():
raise Exception()
jf = jsonfile.JsonFileDriver(self.conf)
jf._load_products = _fake_load_products
self.assertFalse(jf.is_healthy())

104
etc/products.json Normal file
View File

@ -0,0 +1,104 @@
{
"example-region-1":{
"compute":[
{
"name":"c1.c1r05",
"unit":"hour",
"rate":"0.019",
"description":"1 vCPU, 0.5 GB RAM",
},
{
"name":"c1.c1r1",
"unit":"hour",
"rate":"0.039",
"description":"1 vCPU, 1 GB RAM",
},
{
"name":"c1.c1r2",
"unit":"hour",
"rate":"0.062",
"description":"1 vCPU, 2 GB RAM",
},
{
"name":"c1.c1r4",
"unit":"hour",
"rate":"0.098",
"description":"1 vCPU, 4 GB RAM",
}
],
"block_storage":[
{
"name":"b1.standard",
"unit":"gigabyte",
"rate":"0.000416",
"description":"b1.standard"
}
],
"object_storage":[
{
"name":"o1.standard",
"unit":"gigabyte",
"rate":"0.000275",
"description":"Multi-region 3 replicas"
}
],
"network":[
{
"name":"n1.ipv4",
"unit":"hour",
"rate":"0.006",
"description":"Public IPv4"
},
{
"name":"n1.network",
"unit":"hour",
"rate":"0.016400",
"description":"Network",
},
{
"name":"n1.router",
"unit":"hour",
"rate":"0.017",
"description":"Router",
},
{
"name":"n1.lb",
"unit":"hour",
"rate":"0.034",
"description":"Load balancer"
},
{
"name":"n1.vpn",
"unit":"hour",
"rate":"0.017",
"description":"VPN"
}
],
"data transfer":[
{
"name":"n1.local",
"unit":"gigabyte",
"rate":"0",
"description":"Between cloud services in the same region"
},
{
"name":"n1.inter-region",
"unit":"gigabyte",
"rate":"0.12",
"description":"Between cloud services in different regions"
},
{
"name":"n1.national",
"unit":"gigabyte",
"rate":"0.12",
"description":"National traffic over the internet"
},
{
"name":"n1.international",
"unit":"gigabyte",
"rate":"0.3",
"description":"International traffic over the internet"
}
]
}
}

1
setup.cfg
View File

@ -47,6 +47,7 @@ distil.transformer =
distil.erp =
odoo = distil.erp.drivers.odoo:OdooDriver
jsonfile = distil.erp.drivers.jsonfile:JsonFileDriver
[build_sphinx]
all_files = 1