diff --git a/distil/config.py b/distil/config.py index 545c78a..75e2ed6 100644 --- a/distil/config.py +++ b/distil/config.py @@ -115,6 +115,12 @@ ODOO_OPTS = [ 'c1.c1r2-windows or c1.c2r4-sql-server-standard-windows') ] +JSONFILE_OPTS = [ + cfg.StrOpt('products_file_path', + default='/etc/distil/products.json', + help='Json file to contain the products and prices.'), +] + CLI_OPTS = [ cfg.StrOpt( @@ -128,10 +134,12 @@ CLI_OPTS = [ AUTH_GROUP = 'keystone_authtoken' ODOO_GROUP = 'odoo' COLLECTOR_GROUP = 'collector' +JSONFILE_GROUP = 'jsonfile' CONF.register_opts(DEFAULT_OPTIONS) CONF.register_opts(ODOO_OPTS, group=ODOO_GROUP) +CONF.register_opts(JSONFILE_OPTS, group=JSONFILE_GROUP) CONF.register_opts(COLLECTOR_OPTS, group=COLLECTOR_GROUP) CONF.register_cli_opts(CLI_OPTS) diff --git a/distil/erp/drivers/jsonfile.py b/distil/erp/drivers/jsonfile.py new file mode 100644 index 0000000..10e91ca --- /dev/null +++ b/distil/erp/drivers/jsonfile.py @@ -0,0 +1,323 @@ +# Copyright 2019 Catalyst IT Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import collections +import copy +from datetime import datetime +from datetime import timedelta +import decimal +import os +import json +import itertools + +from oslo_config import cfg +from oslo_log import log + +from distil.common import cache +from distil.common import constants +from distil import exceptions +from distil.common import general +from distil.db import api as db_api +from distil.erp import driver + +LOG = log.getLogger(__name__) +CONF = cfg.CONF + + +class JsonFileDriver(driver.BaseDriver): + """Json driver + """ + conf = None + + def __init__(self, conf): + self.conf = conf + + def _load_products(self): + try: + with open(self.conf['jsonfile']['products_file_path']) as fh: + products = json.load(fh) + return products + except Exception as e: + LOG.critical('Failed to load rates json file: `%s`' % e) + raise e + + def is_healthy(self): + """Check if the ERP back end is healthy or not + + :returns True if the ERP is healthy, otherwise False + """ + try: + p = self._load_products() + return p is not None + except Exception as e: + LOG.error(e) + return False + + def get_products(self, regions=[]): + """List products based on given regions + + :param regions: List of regions to get projects + :returns Dict of products based on the given regions + """ + # TODO(flwang): Apply regions + products = self._load_products() + if regions: + region_products = {} + for r in regions: + if r in products: + region_products[r] = products[r] + return region_products + return products + + def create_product(self, product): + """Create product in ERP backend. + + :param product: info used to create product + """ + raise NotImplementedError() + + def get_credits(self, project_id, expiry_date): + """Get project credits + + :param project_id: Project ID + :param expiry_date: Any credit which expires after this date can be + listed + :returns list of credits current project can get + """ + raise NotImplementedError() + + def create_credit(self, project, credit): + """Create credit for a given project + + :param project: project + """ + raise NotImplementedError() + + def _get_invoice_time_ranges(self, start, end): + previous_months = [] + + indicator = end + while start <= indicator - timedelta(seconds=1): + end_inv = indicator.replace( + day=1, hour=0, minute=0, second=0, microsecond=0) + + last_month = indicator.month - 1 + if last_month: + indicator = indicator.replace( + month=last_month, day=1, hour=0, minute=0, second=0, + microsecond=0) + else: + last_year = indicator.year - 1 + indicator = indicator.replace( + year=last_year, month=12, day=1, hour=0, minute=0, + second=0, microsecond=0) + + bill_date = end_inv - timedelta(seconds=1) + + previous_months.append(( + indicator, + end_inv, + bill_date.strftime('%Y-%m-%d') + )) + + return previous_months + + def get_invoices(self, start, end, project_id, detailed=False): + """Get history invoices from ERP service given a time range. + + This will only get the history for the current given region. + + :param start: Start time, a datetime object. + :param end: End time, a datetime object. + :param project_id: project ID. + :param detailed: If get detailed information or not. + :return: The history invoices information for each month. + """ + region_name = CONF.keystone_authtoken.region_name + + previous_months = self._get_invoice_time_ranges(start, end) + + invoices = {} + for start_inv, end_inv, bill_date_str in previous_months: + + usage = db_api.usage_get(project_id, start_inv, end_inv) + all_resource_ids = set( + [entry.get('resource_id') for entry in usage]) + res_list = db_api.resource_get_by_ids(project_id, all_resource_ids) + quotation = self.get_quotations( + region_name, + project_id, + measurements=usage, + resources=res_list, + detailed=detailed + ) + quotation['status'] = 'paid' + invoices[bill_date_str] = quotation + return invoices + + @cache.memoize + def _get_service_mapping(self, products): + """Gets mapping from service name to service type. + + :param products: Product dict in a region returned from odoo. + """ + srv_mapping = {} + + for category, p_list in products.items(): + for p in p_list: + srv_mapping[p['name']] = category.title() + + return srv_mapping + + @cache.memoize + def _get_service_price(self, service_name, service_type, products): + """Get service price information from price definitions.""" + price = {'service_name': service_name} + + if service_type in products: + for s in products[service_type]: + if s['name'] == service_name: + price.update({'rate': s['rate'], 'unit': s['unit']}) + break + else: + found = False + for category, services in products.items(): + for s in services: + if s['name'] == service_name: + price.update({'rate': s['rate'], 'unit': s['unit']}) + found = True + break + + if not found: + raise exceptions.NotFoundException( + 'Price not found, service name: %s, service type: %s' % + (service_name, service_type) + ) + + return price + + def _get_entry_info(self, entry, resources_info, service_mapping): + service_name = entry.get('service') + volume = entry.get('volume') + unit = entry.get('unit') + res_id = entry.get('resource_id') + resource = resources_info.get(res_id, {}) + # resource_type is the type defined in meter_mappings.yml. + resource_type = resource.get('type') + service_type = service_mapping.get(service_name, resource_type) + + return (service_name, service_type, volume, unit, resource, + resource_type) + + def get_quotations(self, region, project_id, measurements=[], resources=[], + detailed=False): + """Get current month quotation. + + Return value is in the following format: + { + '': { + 'total_cost': 100, + 'details': { + 'Compute': { + 'total_cost': xxx, + 'breakdown': {} + } + } + } + } + + :param region: Region name. + :param project_id: Project ID. + :param measurements: Current month usage collection. + :param resources: List of resources. + :param detailed: If get detailed information or not. + :return: Current month quotation. + """ + total_cost = 0 + price_mapping = {} + cost_details = {} + + resources_info = {} + for row in resources: + info = json.loads(row.info) + info.update({'id': row.id}) + resources_info[row.id] = info + + # NOTE(flwang): For most of the cases of Distil API, the request comes + # from billing panel. Billing panel sends 1 API call for /invoices and + # several API calls for /quotations against different regions. So it's + # not efficient to specify the region for get_products method because + # it won't help cache the products based on the parameters. + products = self.get_products()[region] + service_mapping = self._get_service_mapping(products) + + for entry in measurements: + (service_name, service_type, volume, unit, resource, + resource_type) = self._get_entry_info(entry, resources_info, + service_mapping) + res_id = resource['id'] + + if service_type not in cost_details: + cost_details[service_type] = { + 'total_cost': 0, + 'breakdown': collections.defaultdict(list) + } + + if service_name not in price_mapping: + price_spec = self._get_service_price( + service_name, service_type, products + ) + price_mapping[service_name] = price_spec + + price_spec = price_mapping[service_name] + + # Convert volume according to unit in price definition. + volume = float( + general.convert_to(volume, unit, price_spec['unit']) + ) + cost = (round(volume * price_spec['rate'], constants.PRICE_DIGITS) + if price_spec['rate'] else 0) + + total_cost += cost + + if detailed: + erp_service_name = "%s.%s" % (region, service_name) + + cost_details[service_type]['total_cost'] = round( + (cost_details[service_type]['total_cost'] + cost), + constants.PRICE_DIGITS + ) + cost_details[service_type]['breakdown'][ + erp_service_name + ].append( + { + "resource_name": resource.get('name', ''), + "resource_id": res_id, + "cost": cost, + "quantity": round(volume, 3), + "rate": round(price_spec['rate'], + constants.RATE_DIGITS), + "unit": price_spec['unit'], + } + ) + + result = { + 'total_cost': round(float(total_cost), constants.PRICE_DIGITS) + } + + if detailed: + result.update({'details': cost_details}) + + return result diff --git a/distil/service/api/v2/quotations.py b/distil/service/api/v2/quotations.py index b7d56bf..e3b8597 100644 --- a/distil/service/api/v2/quotations.py +++ b/distil/service/api/v2/quotations.py @@ -52,7 +52,6 @@ def get_quotations(project_id, detailed=False): usage = db_api.usage_get(project_id, start, end) all_resource_ids = set([entry.get('resource_id') for entry in usage]) res_list = db_api.resource_get_by_ids(project_id, all_resource_ids) - erp_driver = erp_utils.load_erp_driver(CONF) quotations = erp_driver.get_quotations( region_name, diff --git a/distil/tests/unit/erp/drivers/test_jsonfile.py b/distil/tests/unit/erp/drivers/test_jsonfile.py new file mode 100644 index 0000000..e82de2e --- /dev/null +++ b/distil/tests/unit/erp/drivers/test_jsonfile.py @@ -0,0 +1,316 @@ +# Copyright (c) 2019 Catalyst Cloud Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from collections import namedtuple +from datetime import datetime +from decimal import Decimal +import mock + +from distil.erp.drivers import jsonfile +from distil.tests.unit import base + +REGION = namedtuple('Region', ['id']) + +PRODUCTS = [ + { + 'id': 1, + 'categ_id': [1, 'All products (.NET) / nz-1 / Compute'], + 'name_template': 'nz-1.c1.c1r1', + 'lst_price': 0.00015, + 'default_code': 'hour', + 'description': '1 CPU, 1GB RAM' + }, + { + 'id': 2, + 'categ_id': [2, 'All products (.NET) / nz-1 / Network'], + 'name_template': 'nz-1.n1.router', + 'lst_price': 0.00025, + 'default_code': 'hour', + 'description': 'Router' + }, + { + 'id': 3, + 'categ_id': [1, 'All products (.NET) / nz-1 / Block Storage'], + 'name_template': 'nz-1.b1.volume', + 'lst_price': 0.00035, + 'default_code': 'hour', + 'description': 'Block storage' + } +] + + +class TestJsonFileDriver(base.DistilTestCase): + config_file = 'distil.conf' + + def test_get_products(self): + pass + + @mock.patch('distil.erp.drivers.jsonfile.db_api') + @mock.patch('distil.erp.drivers.jsonfile.JsonFileDriver.get_quotations') + def test_get_invoices_without_details(self, mock_get_quotations, + mock_db_api): + start = datetime(2017, 4, 1) + end = datetime(2018, 3, 1) + fake_project = '123' + + + mock_get_quotations.return_value = {'total_cost': 20} + mock_db_api.usage_get.return_value = [] + mock_db_api.resource_get_by_ids.return_value = [] + + jsondriver = jsonfile.JsonFileDriver(self.conf) + + invoices = jsondriver.get_invoices(start, end, fake_project) + + self.assertEqual( + { + '2017-04-30': {'status': 'paid', 'total_cost': 20}, + '2017-05-31': {'status': 'paid', 'total_cost': 20}, + '2017-06-30': {'status': 'paid', 'total_cost': 20}, + '2017-07-31': {'status': 'paid', 'total_cost': 20}, + '2017-08-31': {'status': 'paid', 'total_cost': 20}, + '2017-09-30': {'status': 'paid', 'total_cost': 20}, + '2017-10-31': {'status': 'paid', 'total_cost': 20}, + '2017-11-30': {'status': 'paid', 'total_cost': 20}, + '2017-12-31': {'status': 'paid', 'total_cost': 20}, + '2018-01-31': {'status': 'paid', 'total_cost': 20}, + '2018-02-28': {'status': 'paid', 'total_cost': 20} + }, + invoices + ) + + @mock.patch('distil.erp.drivers.jsonfile.db_api') + @mock.patch('distil.erp.drivers.jsonfile.JsonFileDriver.get_products') + @mock.patch('distil.erp.drivers.jsonfile.CONF') + def test_get_invoices_with_details(self, mock_conf, mock_get_products, + mock_db_api): + start = datetime(2018, 2, 1) + end = datetime(2018, 3, 1) + fake_project = '123' + + mock_conf.keystone_authtoken.region_name = "nz-1" + + mock_get_products.return_value = { + 'nz-1': { + 'Compute': [ + { + 'name': 'c1.c1r1', 'description': 'c1.c1r1', + 'rate': 0.01, 'unit': 'hour' + } + ] + } + } + + + # mock_get_quotations.return_value = {'total_cost': 20} + mock_db_api.usage_get.return_value = [ + {'service': 'c1.c1r1', 'volume': 500, 'unit': 'hour', + 'resource_id': '47aa'} + ] + mock_db_api.resource_get_by_ids.return_value = [ + mock.Mock( + id='47aa', + info='{"name": "test-1", "type": "Virtual Machine"}') + ] + + jsondriver = jsonfile.JsonFileDriver(self.conf) + + invoices = jsondriver.get_invoices( + start, end, fake_project, detailed=True) + + self.assertEqual( + { + '2018-02-28': { + 'details': { + 'Compute': { + 'breakdown': { + 'nz-1.c1.c1r1': [ + { + 'rate': 0.01, + 'resource_id': '47aa', + 'cost': 5.0, + 'unit': 'hour', + 'quantity': 500.0, + 'resource_name': 'test-1' + } + ] + }, + 'total_cost': 5.0 + } + }, + 'total_cost': 5.0, + 'status': 'paid', + } + }, + invoices + ) + + @mock.patch('distil.erp.drivers.jsonfile.JsonFileDriver.get_products') + def test_get_quotations_without_details(self, mock_get_products): + mock_get_products.return_value = { + 'nz-1': { + 'Compute': [ + { + 'name': 'c1.c2r16', 'description': 'c1.c2r16', + 'rate': 0.01, 'unit': 'hour' + } + ], + 'Block Storage': [ + { + 'name': 'b1.standard', 'description': 'b1.standard', + 'rate': 0.02, 'unit': 'gigabyte' + } + ] + } + } + + class Resource(object): + def __init__(self, id, info): + self.id = id + self.info = info + + resources = [ + Resource(1, '{"name": "", "type": "Volume"}'), + Resource(2, '{"name": "", "type": "Virtual Machine"}') + ] + + usage = [ + { + 'service': 'b1.standard', + 'resource_id': 1, + 'volume': 1024 * 1024 * 1024, + 'unit': 'byte', + }, + { + 'service': 'c1.c2r16', + 'resource_id': 2, + 'volume': 3600, + 'unit': 'second', + } + ] + + jf = jsonfile.JsonFileDriver(self.conf) + quotations = jf.get_quotations( + 'nz-1', 'fake_id', measurements=usage, resources=resources + ) + + self.assertEqual( + {'total_cost': 0.03}, + quotations + ) + + @mock.patch('distil.erp.drivers.jsonfile.JsonFileDriver.get_products') + def test_get_quotations_with_details(self, mock_get_products): + mock_get_products.return_value = { + 'nz-1': { + 'Compute': [ + { + 'name': 'c1.c2r16', 'description': 'c1.c2r16', + 'rate': 0.01, 'unit': 'hour' + } + ], + 'Block Storage': [ + { + 'name': 'b1.standard', 'description': 'b1.standard', + 'rate': 0.02, 'unit': 'gigabyte' + } + ] + } + } + + class Resource(object): + def __init__(self, id, info): + self.id = id + self.info = info + + resources = [ + Resource(1, '{"name": "volume1", "type": "Volume"}'), + Resource(2, '{"name": "instance2", "type": "Virtual Machine"}') + ] + + usage = [ + { + 'service': 'b1.standard', + 'resource_id': 1, + 'volume': 1024 * 1024 * 1024, + 'unit': 'byte', + }, + { + 'service': 'c1.c2r16', + 'resource_id': 2, + 'volume': 3600, + 'unit': 'second', + } + ] + + jf = jsonfile.JsonFileDriver(self.conf) + quotations = jf.get_quotations( + 'nz-1', 'fake_id', measurements=usage, resources=resources, + detailed=True + ) + + self.assertDictEqual( + { + 'total_cost': 0.03, + 'details': { + 'Compute': { + 'total_cost': 0.01, + 'breakdown': { + 'nz-1.c1.c2r16': [ + { + "resource_name": "instance2", + "resource_id": 2, + "cost": 0.01, + "quantity": 1.0, + "rate": 0.01, + "unit": "hour", + } + ], + } + }, + 'Block Storage': { + 'total_cost': 0.02, + 'breakdown': { + 'nz-1.b1.standard': [ + { + "resource_name": "volume1", + "resource_id": 1, + "cost": 0.02, + "quantity": 1.0, + "rate": 0.02, + "unit": "gigabyte", + } + ] + } + } + } + }, + quotations + ) + + def test_is_healthy(self): + def _fake_load_products(): + return {'a': 1} + jf = jsonfile.JsonFileDriver(self.conf) + jf._load_products = _fake_load_products + self.assertTrue(jf.is_healthy()) + + def test_is_healthy_false(self): + def _fake_load_products(): + raise Exception() + + jf = jsonfile.JsonFileDriver(self.conf) + jf._load_products = _fake_load_products + self.assertFalse(jf.is_healthy()) diff --git a/etc/products.json b/etc/products.json new file mode 100644 index 0000000..544d31d --- /dev/null +++ b/etc/products.json @@ -0,0 +1,104 @@ +{ + "example-region-1":{ + "compute":[ + { + "name":"c1.c1r05", + "unit":"hour", + "rate":"0.019", + "description":"1 vCPU, 0.5 GB RAM", + }, + { + "name":"c1.c1r1", + "unit":"hour", + "rate":"0.039", + "description":"1 vCPU, 1 GB RAM", + }, + { + "name":"c1.c1r2", + "unit":"hour", + "rate":"0.062", + "description":"1 vCPU, 2 GB RAM", + }, + { + "name":"c1.c1r4", + "unit":"hour", + "rate":"0.098", + "description":"1 vCPU, 4 GB RAM", + } + ], + "block_storage":[ + { + "name":"b1.standard", + "unit":"gigabyte", + "rate":"0.000416", + "description":"b1.standard" + } + ], + "object_storage":[ + { + "name":"o1.standard", + "unit":"gigabyte", + "rate":"0.000275", + "description":"Multi-region 3 replicas" + } + ], + "network":[ + { + "name":"n1.ipv4", + "unit":"hour", + "rate":"0.006", + "description":"Public IPv4" + }, + { + "name":"n1.network", + "unit":"hour", + "rate":"0.016400", + "description":"Network", + }, + { + "name":"n1.router", + "unit":"hour", + "rate":"0.017", + "description":"Router", + }, + { + "name":"n1.lb", + "unit":"hour", + "rate":"0.034", + "description":"Load balancer" + }, + { + "name":"n1.vpn", + "unit":"hour", + "rate":"0.017", + "description":"VPN" + } + ], + "data transfer":[ + { + "name":"n1.local", + "unit":"gigabyte", + "rate":"0", + "description":"Between cloud services in the same region" + }, + { + "name":"n1.inter-region", + "unit":"gigabyte", + "rate":"0.12", + "description":"Between cloud services in different regions" + }, + { + "name":"n1.national", + "unit":"gigabyte", + "rate":"0.12", + "description":"National traffic over the internet" + }, + { + "name":"n1.international", + "unit":"gigabyte", + "rate":"0.3", + "description":"International traffic over the internet" + } + ] + } + } diff --git a/setup.cfg b/setup.cfg index 87ff223..5ed374f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -47,6 +47,7 @@ distil.transformer = distil.erp = odoo = distil.erp.drivers.odoo:OdooDriver + jsonfile = distil.erp.drivers.jsonfile:JsonFileDriver [build_sphinx] all_files = 1