From d1fa0f62b778aa67758d99eecd62055154b302ce Mon Sep 17 00:00:00 2001 From: adriant Date: Thu, 30 Jan 2014 11:09:30 +1300 Subject: [PATCH 1/2] general hacks to get artifice to bill for a few more things. --- artifice/billing/csv_invoice.py | 126 ++++++++++++++++++++++---------- artifice/interface.py | 49 +++++++++++-- artifice/models/resources.py | 58 +++++++++------ bin/bill.py | 3 +- bin/usage.py | 9 ++- examples/conf.yaml | 38 ++++++++-- 6 files changed, 204 insertions(+), 79 deletions(-) diff --git a/artifice/billing/csv_invoice.py b/artifice/billing/csv_invoice.py index 4447c16..d4ec39c 100644 --- a/artifice/billing/csv_invoice.py +++ b/artifice/billing/csv_invoice.py @@ -32,44 +32,95 @@ class Csv(invoice.RatesFileMixin, invoice.NamesFileMixin, invoice.Invoice): # Usage is one of VMs, Storage, or Volumes. for element in usage: appendee = [] - for key in self.config["row_layout"]: - if key == "cost": - # Ignore costs for now. - appendee.append(None) - continue - # What do we expect element to be? - if key == "rate": - appendee.append(self.rate(self.pretty_name(element.type))) - if key == "type": - # Fetch the 'pretty' name from the mappings, if any - # The default is that this returns the internal name - appendee.append(self.pretty_name(element.get(key))) - continue + for key in self.config["row_layout"][element.type]: + if element.type is "vm": + if key == "flavor": + appendee.append(element.get(key)) + continue + if key == "cost": + cost = (element.uptime.volume() * + self.rate(element.flavor)) + appendee.append(cost) + print ("flavor: " + element.get("flavor")) + print " - name : " + str(element.get("name")) + print " - usage: " + str(element.uptime.volume()) + print (" - rate: " + + str(self.rate(element.flavor))) + print " - cost: " + str(cost) + continue + if key == "rate": + appendee.append(self.rate(element.flavor)) + continue + + if element.type is "ip": + if key == "cost": + cost = element.duration * self.rate("ip.floating") + appendee.append(cost) + print "id: " + print " - usage: " + str(element.duration) + print (" - rate: " + + str(self.rate("ip.floating"))) + print " - cost: " + str(cost) + continue + if key == "rate": + appendee.append(self.rate("ip.floating")) + continue + + if element.type is "object": + if key == "cost": + cost = element.size * self.rate("storage.objects.size") + appendee.append(cost) + print "id:" + print " - usage: " + str(element.size) + print (" - rate: " + + str(self.rate("storage.objects.size"))) + print " - cost: " + str(cost) + continue + if key == "rate": + appendee.append(self.rate("storage.objects.size")) + continue + + if element.type is "volume": + if key == "cost": + cost = element.size * self.rate("volume.size") + appendee.append(cost) + print "id:" + print " - usage: " + str(element.size) + print (" - rate: " + + str(self.rate("volume.size"))) + print " - cost: " + str(cost) + continue + if key == "rate": + appendee.append(self.rate("volume.size")) + continue + + if element.type is "network": + if key == "cost": + cost_in = (element.incoming * + self.rate("network.outgoing.bytes")) + cost_out = (element.outgoing * + self.rate("network.outgoing.bytes")) + print "id:" + print " - incoming: " + str(element.incoming) + print (" - rate: " + + str(self.rate("network.incoming.bytes"))) + print " - outgoing: " + str(element.outgoing) + print (" - rate: " + + str(self.rate("network.outgoing.bytes"))) + print " - cost: " + str(cost_in + cost_out) + appendee.append(cost_in + cost_out) + continue + if key == "incoming_rate": + appendee.append(self.rate("network.incoming.bytes")) + continue + if key == "outgoing_rate": + appendee.append(self.rate("network.outgoing.bytes")) + continue + try: appendee.append(element.get(key)) except AttributeError: appendee.append("") - try: - x = self.config["row_layout"].index("cost") - appendee[x] = (element.amount.volume() * - self.rate(self.pretty_name(element.type))) - # print (str(appendee[1]) + " - From: " + str(appendee[2]) + - # ", Until: " + str(appendee[3])) - print ("type: " + str(self.pretty_name(element.get("type")))) - try: - print " - name : " + str(element.get("name")) - except: - # Just means it isn't a VM - pass - print " - usage: " + str(element.amount.volume()) - print (" - rate: " + - str(self.rate(self.pretty_name(element.type)))) - print " - cost: " + str(appendee[x]) - - except ValueError: - # Not in this array. Well okay. - # We're not storing cost info, apparently. - raise RuntimeError("No costing information in CSV layout.") # print appendee self.add_line(appendee) @@ -103,7 +154,7 @@ class Csv(invoice.RatesFileMixin, invoice.NamesFileMixin, invoice.Invoice): csvwriter.writerow(["usage range: ", str(self.start), str(self.end)]) csvwriter.writerow([]) - csvwriter.writerow(self.config["row_layout"]) + # csvwriter.writerow(self.config["row_layout"]) for line in self.lines: # Line is expected to be an iterable row csvwriter.writerow(line) @@ -120,11 +171,8 @@ class Csv(invoice.RatesFileMixin, invoice.NamesFileMixin, invoice.Invoice): def total(self): total = Decimal(0.0) for line in self.lines: - # Cheatery - # Creates a dict on the fly from the row layout and the line value - v = dict([(k, v) for k, v in zip(self.config["row_layout"], line)]) try: - total += Decimal(v["cost"]) + total += line[len(line) - 1] except (TypeError, ValueError): total += 0 return total diff --git a/artifice/interface.py b/artifice/interface.py index 8ba2b47..9fcec6e 100644 --- a/artifice/interface.py +++ b/artifice/interface.py @@ -257,32 +257,35 @@ class Tenant(object): vms = [] networks = [] + ips = [] storage = [] volumes = [] # Object storage is mapped by project_id for resource in self.resources(start, end): - # print dir(resource) rels = [link["rel"] for link in resource.links if link["rel"] != 'self'] if "storage.objects" in rels: # Unknown how this data layout happens yet. storage.append(Resource(resource, self.conn)) pass - elif "network" in rels: + elif "network.incoming.bytes" in rels: # Have we seen the VM that owns this yet? networks.append(Resource(resource, self.conn)) - elif "volumne" in rels: + elif "volume" in rels: volumes.append(Resource(resource, self.conn)) elif 'instance' in rels: vms.append(Resource(resource, self.conn)) + elif 'ip.floating' in rels: + ips.append(Resource(resource, self.conn)) datacenters = {} region_tmpl = { "vms": vms, - "network": networks, + "networks": networks, "objects": storage, - "volumes": volumes + "volumes": volumes, + "ips": ips } return Usage(region_tmpl, start, end, self.conn) @@ -303,6 +306,8 @@ class Usage(object): self._vms = [] self._objects = [] self._volumes = [] + self._networks = [] + self._ips = [] # Replaces all the internal references with better references to # actual metered values. @@ -331,6 +336,26 @@ class Usage(object): self._objs = objs return self._objs + @property + def networks(self): + if not self._networks: + networks = [] + for obj in self.contents["networks"]: + obj = resources.Network(obj, self.start, self.end) + networks.append(obj) + self._networks = networks + return self._networks + + @property + def ips(self): + if not self._ips: + ips = [] + for obj in self.contents["ips"]: + obj = resources.FloatingIP(obj, self.start, self.end) + ips.append(obj) + self._ips = ips + return self._ips + @property def volumes(self): if not self._volumes: @@ -551,7 +576,7 @@ class Cumulative(Artifact): def volume(self): measurements = self.usage measurements = sorted(measurements, key=lambda x: x["timestamp"]) - count = 1 + count = 0 usage = 0 last_measure = None for measure in measurements: @@ -563,13 +588,15 @@ class Cumulative(Artifact): usage = usage + measurements[-1]["counter_volume"] - total_usage = usage - measurements[0]["counter_volume"] + if count > 1: + total_usage = usage - measurements[0]["counter_volume"] return total_usage # Gauge and Delta have very little to do: They are expected only to # exist as "not a cumulative" sort of artifact. class Gauge(Artifact): + def volume(self): """ Default billable number for this volume @@ -607,6 +634,12 @@ class Gauge(Artifact): else: curr.append(val) + # this adds the last remaining values as a block of their own on exit + # might mean people are billed twice for an hour at times... + # but solves the issue of not billing if there isn't enough data for + # full hour. + blocks.append(curr) + # We are now sorted into 1-hour blocks totals = [] for block in blocks: @@ -615,7 +648,6 @@ class Gauge(Artifact): # totals = [max(x, key=lambda val: val["counter_volume"] ) for x in blocks] # totals is now an array of max values per hour for a given month. - # print totals return sum(totals) def uptime(self, tracked): @@ -660,6 +692,7 @@ class Gauge(Artifact): # the timedelta should be the ceilometer interval. # do nothing if different greater than twice interval? # or just add interval length to uptime. + # FLAGS! logs these events so sys ops can doulbe check them pass else: # otherwise just add difference. diff --git a/artifice/models/resources.py b/artifice/models/resources.py index a180e24..cfdabf4 100644 --- a/artifice/models/resources.py +++ b/artifice/models/resources.py @@ -69,8 +69,9 @@ class VM(BaseModelConstruct): # The only relevant meters of interest are the type of the interest # and the amount of network we care about. # Oh, and floating IPs. - relevant_meters = ["state", "instance", "cpu", "instance:", - "network.incoming.bytes", "network.outgoing.bytes"] + relevant_meters = ["state"] + + type = "vm" def _fetch_meter_name(self, name): if name == "instance:": @@ -79,10 +80,6 @@ class VM(BaseModelConstruct): @property def uptime(self): - return self.amount - - @property - def amount(self): # this NEEDS to be moved to a config file or # possibly be queried from Clerk? @@ -105,7 +102,7 @@ class VM(BaseModelConstruct): return Amount() @property - def type(self): + def flavor(self): # TODO FIgure out what the hell is going on with ceilometer here, # and why flavor.name isn't always there, and why # sometimes instance_type is needed instead.... @@ -132,21 +129,25 @@ class VM(BaseModelConstruct): def state(self): return self._raw["metadata"]["state"] - @property - def bandwidth(self): - # This is a metered value - return 0 - - @property - def ips(self): - """floating IPs; this is a metered value""" - return 0 - @property def name(self): return self._raw["metadata"]["display_name"] +class FloatingIP(BaseModelConstruct): + + relevant_meters = ["ip.floating"] + + type = "ip" # object storage + + @property + def duration(self): + # How much use this had. + return Decimal(self.usage()["ip.floating"].volume()) + # Size is a gauge measured every 10 minutes. + # So that needs to be compressed to 60-minute intervals + + class Object(BaseModelConstruct): relevant_meters = ["storage.objects.size"] @@ -156,8 +157,7 @@ class Object(BaseModelConstruct): @property def size(self): # How much use this had. - return self._raw.meter("storage.objects.size", - self.start, self.end).volume() + return Decimal(self.usage()["storage.objects.size"].volume()) # Size is a gauge measured every 10 minutes. # So that needs to be compressed to 60-minute intervals @@ -166,15 +166,25 @@ class Volume(BaseModelConstruct): relevant_meters = ["volume.size"] - @property - def location(self): - return self._location + type = "volume" @property def size(self): # Size of the thing over time. - return self._raw.meter("volume.size", self.start, self.end).volume() + return Decimal(self.usage()["volume.size"].volume()) class Network(BaseModelConstruct): - relevant_meters = ["ip.floating"] + relevant_meters = ["network.outgoing.bytes", "network.incoming.bytes"] + + type = "network" + + @property + def outgoing(self): + # Size of the thing over time. + return Decimal(self.usage()["network.outgoing.bytes"].volume()) + + @property + def incoming(self): + # Size of the thing over time. + return Decimal(self.usage()["network.incoming.bytes"].volume()) diff --git a/bin/bill.py b/bin/bill.py index c561b1f..aa09f28 100644 --- a/bin/bill.py +++ b/bin/bill.py @@ -98,7 +98,8 @@ if __name__ == '__main__': usage.save() invoice.bill(usage.vms) invoice.bill(usage.volumes) - # invoice.bill(usage.objects) + invoice.bill(usage.objects) + invoice.bill(usage.networks) invoice.close() print invoice.total() diff --git a/bin/usage.py b/bin/usage.py index aae704a..04bd9fa 100644 --- a/bin/usage.py +++ b/bin/usage.py @@ -115,9 +115,16 @@ if __name__ == '__main__': # A Usage set is the entirety of time for this Tenant. # It's not time-limited at all. + print "# Virtual Machines #" invoice.bill(usage.vms) + print "# Volumes #" invoice.bill(usage.volumes) - # invoice.bill(usage.objects) + print "# Objects #" + invoice.bill(usage.objects) + print "# Networks #" + invoice.bill(usage.networks) + print "# Floating IPs #" + invoice.bill(usage.ips) print "Total invoice value: %s" % invoice.total() print format_title("End of Tenant: %s" % tenant.name, max_len) diff --git a/examples/conf.yaml b/examples/conf.yaml index 82d644e..45fbe29 100644 --- a/examples/conf.yaml +++ b/examples/conf.yaml @@ -12,12 +12,38 @@ invoice_object: output_file: '%(tenant)s-%(start)s-%(end)s.csv' output_path: ./ row_layout: - - location - - type - - start - - end - - amount - - cost + vm: + - name + - id + - location + - flavor + - start + - end + - uptime + - rate + - cost + object: + - id + - size + - rate + - cost + volume: + - id + - size + - rate + - cost + network: + - id + - incoming + - outgoing + - incoming_rate + - outgoing_rate + - cost + ip: + - id + - duration + - rate + - cost rates: file: /etc/artifice/csv_rates.csv names: /etc/artifice/csv_names.csv From ddd1815abc97fbd75a8e73e988323cc4289e87f6 Mon Sep 17 00:00:00 2001 From: adriant Date: Thu, 30 Jan 2014 11:18:26 +1300 Subject: [PATCH 2/2] adding api stuff --- api/artifice_api.py | 76 +++++++++++++++++++++++++++++++++++++++++++++ api/keystone_api.py | 32 +++++++++++++++++++ 2 files changed, 108 insertions(+) create mode 100644 api/artifice_api.py create mode 100644 api/keystone_api.py diff --git a/api/artifice_api.py b/api/artifice_api.py new file mode 100644 index 0000000..f77a8cd --- /dev/null +++ b/api/artifice_api.py @@ -0,0 +1,76 @@ +from flask import Flask, jsonify, abort, make_response +from flask.ext.restful import Api, Resource, reqparse, fields, marshal +from keystone_api import keystone_auth_decorator + + +app = Flask(__name__, static_url_path="") +api = Api(app) + + +service_list = {"serv1": "Service 1", "serv2": "Service 2", + "serv3": "Service 3", "serv4": "Service 4", + "serv5": "Service 5", "serv6": "Service 6"} + + +billed_months = {"bills": [{"date": "Oct, 2013", + "charges": + {"Service 1": "56", "Service 2": "32", + "Service 3": "45", "Service 4": "28", + "Service 5": "42"} + }, + {"date": "Nov, 2013", + "charges": + {"Service 2": "16", + "Service 3": "45.4", "Service 4": "28", + "Service 5": "42.25"} + }, + {"date": "Dec, 2013", + "charges": + {"Service 1": "23", "Service 2": "89", + "Service 3": "62", "Service 4": "89", + "Service 5": "99", "Service 6": "66"} + }, + {"date": "Jan, 2014", + "charges": + {"Service 1": "25", "Service 2": "12", + "Service 3": "43.2", "Service 4": "60", + "Service 5": "86", "Service 6": "32"} + }] + } + +unbilled_month = {"date": "Feb, 2014", + "charges": + {"Service 1": "12", "Service 2": "11.2", + "Service 3": "12", "Service 4": "35", + "Service 5": "55.7", "Service 6": "23"} + } + + + +class BillAPI(Resource): + decorators = [keystone_auth_decorator] + + def get(self, id): + return billed_months + + +class UnbilledAPI(Resource): + decorators = [keystone_auth_decorator] + + def get(self, id): + return unbilled_month + + +class ServiceList(Resource): + + def get(self): + return service_list + + +api.add_resource(BillAPI, '/artifice/bills/', endpoint='bills') +api.add_resource(UnbilledAPI, '/artifice/bills//unbilled', + endpoint='unbilled') +api.add_resource(ServiceList, '/artifice/service_list', endpoint='services') + +if __name__ == '__main__': + app.run(debug=True, host='0.0.0.0', port=2000) diff --git a/api/keystone_api.py b/api/keystone_api.py new file mode 100644 index 0000000..ea22b14 --- /dev/null +++ b/api/keystone_api.py @@ -0,0 +1,32 @@ +from keystoneclient.v2_0 import client +from flask import request +import flask_restful + + +def validate_user_in_tenancy(tenant_id): + headers = request.headers + if 'user_id' in headers: + user_id = headers['user_id'] + endpoint = "http://0.0.0.0:35357/v2.0" # MAJOR TODO + admin_token = "bob" # MAJOR TODO + keystone = client.Client(token=admin_token, endpoint=endpoint) + tenant = keystone.tenants.get(tenant_id) + for user in tenant.list_users(): + if user.id == user_id: + return True + return False + else: + flask_restful.abort(403, message=("'user_id' and 'tenant_id' are" + + "required values.")) + + +def keystone_auth_decorator(func): + + def wrapper(*args, **kwargs): + if validate_user_in_tenancy(kwargs['id']): + return func(*args, **kwargs) + else: + flask_restful.abort(403, message=("User does not have access" + + "to this tenant.")) + + return wrapper