x/distil
Code Issues Proposed changes

Merge branch 'master' of git+ssh://git.catalyst.net.nz/git/private/openstack-artifice

Browse Source
This commit is contained in:
Aurynn Shaw 2014-01-30 11:23:37 +13:00
commit 29ee5cbeac
8 changed files with 312 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

76
api/artifice_api.py Normal file
View File

@ -0,0 +1,76 @@
from flask import Flask, jsonify, abort, make_response
from flask.ext.restful import Api, Resource, reqparse, fields, marshal
from keystone_api import keystone_auth_decorator
app = Flask(__name__, static_url_path="")
api = Api(app)
service_list = {"serv1": "Service 1", "serv2": "Service 2",
"serv3": "Service 3", "serv4": "Service 4",
"serv5": "Service 5", "serv6": "Service 6"}
billed_months = {"bills": [{"date": "Oct, 2013",
"charges":
{"Service 1": "56", "Service 2": "32",
"Service 3": "45", "Service 4": "28",
"Service 5": "42"}
},
{"date": "Nov, 2013",
"charges":
{"Service 2": "16",
"Service 3": "45.4", "Service 4": "28",
"Service 5": "42.25"}
},
{"date": "Dec, 2013",
"charges":
{"Service 1": "23", "Service 2": "89",
"Service 3": "62", "Service 4": "89",
"Service 5": "99", "Service 6": "66"}
},
{"date": "Jan, 2014",
"charges":
{"Service 1": "25", "Service 2": "12",
"Service 3": "43.2", "Service 4": "60",
"Service 5": "86", "Service 6": "32"}
}]
}
unbilled_month = {"date": "Feb, 2014",
"charges":
{"Service 1": "12", "Service 2": "11.2",
"Service 3": "12", "Service 4": "35",
"Service 5": "55.7", "Service 6": "23"}
}
class BillAPI(Resource):
decorators = [keystone_auth_decorator]
def get(self, id):
return billed_months
class UnbilledAPI(Resource):
decorators = [keystone_auth_decorator]
def get(self, id):
return unbilled_month
class ServiceList(Resource):
def get(self):
return service_list
api.add_resource(BillAPI, '/artifice/bills/<string:id>', endpoint='bills')
api.add_resource(UnbilledAPI, '/artifice/bills/<string:id>/unbilled',
endpoint='unbilled')
api.add_resource(ServiceList, '/artifice/service_list', endpoint='services')
if __name__ == '__main__':
app.run(debug=True, host='0.0.0.0', port=2000)

32
api/keystone_api.py Normal file
View File

@ -0,0 +1,32 @@
from keystoneclient.v2_0 import client
from flask import request
import flask_restful
def validate_user_in_tenancy(tenant_id):
headers = request.headers
if 'user_id' in headers:
user_id = headers['user_id']
endpoint = "http://0.0.0.0:35357/v2.0" # MAJOR TODO
admin_token = "bob" # MAJOR TODO
keystone = client.Client(token=admin_token, endpoint=endpoint)
tenant = keystone.tenants.get(tenant_id)
for user in tenant.list_users():
if user.id == user_id:
return True
return False
else:
flask_restful.abort(403, message=("'user_id' and 'tenant_id' are" +
"required values."))
def keystone_auth_decorator(func):
def wrapper(*args, **kwargs):
if validate_user_in_tenancy(kwargs['id']):
return func(*args, **kwargs)
else:
flask_restful.abort(403, message=("User does not have access" +
"to this tenant."))
return wrapper

118
artifice/billing/csv_invoice.py
View File

@ -32,44 +32,95 @@ class Csv(invoice.RatesFileMixin, invoice.NamesFileMixin, invoice.Invoice):
# Usage is one of VMs, Storage, or Volumes.
for element in usage:
appendee = []
for key in self.config["row_layout"]:
for key in self.config["row_layout"][element.type]:
if element.type is "vm":
if key == "flavor":
appendee.append(element.get(key))
continue
if key == "cost":
# Ignore costs for now.
appendee.append(None)
cost = (element.uptime.volume() *
self.rate(element.flavor))
appendee.append(cost)
print ("flavor: " + element.get("flavor"))
print " - name : " + str(element.get("name"))
print " - usage: " + str(element.uptime.volume())
print (" - rate: " +
str(self.rate(element.flavor)))
print " - cost: " + str(cost)
continue
# What do we expect element to be?
if key == "rate":
appendee.append(self.rate(self.pretty_name(element.type)))
if key == "type":
# Fetch the 'pretty' name from the mappings, if any
# The default is that this returns the internal name
appendee.append(self.pretty_name(element.get(key)))
appendee.append(self.rate(element.flavor))
continue
if element.type is "ip":
if key == "cost":
cost = element.duration * self.rate("ip.floating")
appendee.append(cost)
print "id: "
print " - usage: " + str(element.duration)
print (" - rate: " +
str(self.rate("ip.floating")))
print " - cost: " + str(cost)
continue
if key == "rate":
appendee.append(self.rate("ip.floating"))
continue
if element.type is "object":
if key == "cost":
cost = element.size * self.rate("storage.objects.size")
appendee.append(cost)
print "id:"
print " - usage: " + str(element.size)
print (" - rate: " +
str(self.rate("storage.objects.size")))
print " - cost: " + str(cost)
continue
if key == "rate":
appendee.append(self.rate("storage.objects.size"))
continue
if element.type is "volume":
if key == "cost":
cost = element.size * self.rate("volume.size")
appendee.append(cost)
print "id:"
print " - usage: " + str(element.size)
print (" - rate: " +
str(self.rate("volume.size")))
print " - cost: " + str(cost)
continue
if key == "rate":
appendee.append(self.rate("volume.size"))
continue
if element.type is "network":
if key == "cost":
cost_in = (element.incoming *
self.rate("network.outgoing.bytes"))
cost_out = (element.outgoing *
self.rate("network.outgoing.bytes"))
print "id:"
print " - incoming: " + str(element.incoming)
print (" - rate: " +
str(self.rate("network.incoming.bytes")))
print " - outgoing: " + str(element.outgoing)
print (" - rate: " +
str(self.rate("network.outgoing.bytes")))
print " - cost: " + str(cost_in + cost_out)
appendee.append(cost_in + cost_out)
continue
if key == "incoming_rate":
appendee.append(self.rate("network.incoming.bytes"))
continue
if key == "outgoing_rate":
appendee.append(self.rate("network.outgoing.bytes"))
continue
try:
appendee.append(element.get(key))
except AttributeError:
appendee.append("")
try:
x = self.config["row_layout"].index("cost")
appendee[x] = (element.amount.volume() *
self.rate(self.pretty_name(element.type)))
# print (str(appendee[1]) + " - From: " + str(appendee[2]) +
# ", Until: " + str(appendee[3]))
print ("type: " + str(self.pretty_name(element.get("type"))))
try:
print " - name : " + str(element.get("name"))
except:
# Just means it isn't a VM
pass
print " - usage: " + str(element.amount.volume())
print (" - rate: " +
str(self.rate(self.pretty_name(element.type))))
print " - cost: " + str(appendee[x])
except ValueError:
# Not in this array. Well okay.
# We're not storing cost info, apparently.
raise RuntimeError("No costing information in CSV layout.")
# print appendee
self.add_line(appendee)
@ -103,7 +154,7 @@ class Csv(invoice.RatesFileMixin, invoice.NamesFileMixin, invoice.Invoice):
csvwriter.writerow(["usage range: ", str(self.start), str(self.end)])
csvwriter.writerow([])
csvwriter.writerow(self.config["row_layout"])
# csvwriter.writerow(self.config["row_layout"])
for line in self.lines:
# Line is expected to be an iterable row
csvwriter.writerow(line)
@ -120,11 +171,8 @@ class Csv(invoice.RatesFileMixin, invoice.NamesFileMixin, invoice.Invoice):
def total(self):
total = Decimal(0.0)
for line in self.lines:
# Cheatery
# Creates a dict on the fly from the row layout and the line value
v = dict([(k, v) for k, v in zip(self.config["row_layout"], line)])
try:
total += Decimal(v["cost"])
total += line[len(line) - 1]
except (TypeError, ValueError):
total += 0
return total

47
artifice/interface.py
View File

@ -257,32 +257,35 @@ class Tenant(object):
vms = []
networks = []
ips = []
storage = []
volumes = []
# Object storage is mapped by project_id
for resource in self.resources(start, end):
# print dir(resource)
rels = [link["rel"] for link in resource.links if link["rel"] != 'self']
if "storage.objects" in rels:
# Unknown how this data layout happens yet.
storage.append(Resource(resource, self.conn))
pass
elif "network" in rels:
elif "network.incoming.bytes" in rels:
# Have we seen the VM that owns this yet?
networks.append(Resource(resource, self.conn))
elif "volumne" in rels:
elif "volume" in rels:
volumes.append(Resource(resource, self.conn))
elif 'instance' in rels:
vms.append(Resource(resource, self.conn))
elif 'ip.floating' in rels:
ips.append(Resource(resource, self.conn))
datacenters = {}
region_tmpl = {
"vms": vms,
"network": networks,
"networks": networks,
"objects": storage,
"volumes": volumes
"volumes": volumes,
"ips": ips
}
return Usage(region_tmpl, start, end, self.conn)
@ -303,6 +306,8 @@ class Usage(object):
self._vms = []
self._objects = []
self._volumes = []
self._networks = []
self._ips = []
# Replaces all the internal references with better references to
# actual metered values.
@ -331,6 +336,26 @@ class Usage(object):
self._objs = objs
return self._objs
@property
def networks(self):
if not self._networks:
networks = []
for obj in self.contents["networks"]:
obj = resources.Network(obj, self.start, self.end)
networks.append(obj)
self._networks = networks
return self._networks
@property
def ips(self):
if not self._ips:
ips = []
for obj in self.contents["ips"]:
obj = resources.FloatingIP(obj, self.start, self.end)
ips.append(obj)
self._ips = ips
return self._ips
@property
def volumes(self):
if not self._volumes:
@ -551,7 +576,7 @@ class Cumulative(Artifact):
def volume(self):
measurements = self.usage
measurements = sorted(measurements, key=lambda x: x["timestamp"])
count = 1
count = 0
usage = 0
last_measure = None
for measure in measurements:
@ -563,6 +588,7 @@ class Cumulative(Artifact):
usage = usage + measurements[-1]["counter_volume"]
if count > 1:
total_usage = usage - measurements[0]["counter_volume"]
return total_usage
@ -570,6 +596,7 @@ class Cumulative(Artifact):
# Gauge and Delta have very little to do: They are expected only to
# exist as "not a cumulative" sort of artifact.
class Gauge(Artifact):
def volume(self):
"""
Default billable number for this volume
@ -607,6 +634,12 @@ class Gauge(Artifact):
else:
curr.append(val)
# this adds the last remaining values as a block of their own on exit
# might mean people are billed twice for an hour at times...
# but solves the issue of not billing if there isn't enough data for
# full hour.
blocks.append(curr)
# We are now sorted into 1-hour blocks
totals = []
for block in blocks:
@ -615,7 +648,6 @@ class Gauge(Artifact):
# totals = [max(x, key=lambda val: val["counter_volume"] ) for x in blocks]
# totals is now an array of max values per hour for a given month.
# print totals
return sum(totals)
def uptime(self, tracked):
@ -660,6 +692,7 @@ class Gauge(Artifact):
# the timedelta should be the ceilometer interval.
# do nothing if different greater than twice interval?
# or just add interval length to uptime.
# FLAGS! logs these events so sys ops can doulbe check them
pass
else:
# otherwise just add difference.

58
artifice/models/resources.py
View File

@ -69,8 +69,9 @@ class VM(BaseModelConstruct):
# The only relevant meters of interest are the type of the interest
# and the amount of network we care about.
# Oh, and floating IPs.
relevant_meters = ["state", "instance", "cpu", "instance:<type>",
"network.incoming.bytes", "network.outgoing.bytes"]
relevant_meters = ["state"]
type = "vm"
def _fetch_meter_name(self, name):
if name == "instance:<type>":
@ -79,10 +80,6 @@ class VM(BaseModelConstruct):
@property
def uptime(self):
return self.amount
@property
def amount(self):
# this NEEDS to be moved to a config file or
# possibly be queried from Clerk?
@ -105,7 +102,7 @@ class VM(BaseModelConstruct):
return Amount()
@property
def type(self):
def flavor(self):
# TODO FIgure out what the hell is going on with ceilometer here,
# and why flavor.name isn't always there, and why
# sometimes instance_type is needed instead....
@ -132,21 +129,25 @@ class VM(BaseModelConstruct):
def state(self):
return self._raw["metadata"]["state"]
@property
def bandwidth(self):
# This is a metered value
return 0
@property
def ips(self):
"""floating IPs; this is a metered value"""
return 0
@property
def name(self):
return self._raw["metadata"]["display_name"]
class FloatingIP(BaseModelConstruct):
relevant_meters = ["ip.floating"]
type = "ip" # object storage
@property
def duration(self):
# How much use this had.
return Decimal(self.usage()["ip.floating"].volume())
# Size is a gauge measured every 10 minutes.
# So that needs to be compressed to 60-minute intervals
class Object(BaseModelConstruct):
relevant_meters = ["storage.objects.size"]
@ -156,8 +157,7 @@ class Object(BaseModelConstruct):
@property
def size(self):
# How much use this had.
return self._raw.meter("storage.objects.size",
self.start, self.end).volume()
return Decimal(self.usage()["storage.objects.size"].volume())
# Size is a gauge measured every 10 minutes.
# So that needs to be compressed to 60-minute intervals
@ -166,15 +166,25 @@ class Volume(BaseModelConstruct):
relevant_meters = ["volume.size"]
@property
def location(self):
return self._location
type = "volume"
@property
def size(self):
# Size of the thing over time.
return self._raw.meter("volume.size", self.start, self.end).volume()
return Decimal(self.usage()["volume.size"].volume())
class Network(BaseModelConstruct):
relevant_meters = ["ip.floating"]
relevant_meters = ["network.outgoing.bytes", "network.incoming.bytes"]
type = "network"
@property
def outgoing(self):
# Size of the thing over time.
return Decimal(self.usage()["network.outgoing.bytes"].volume())
@property
def incoming(self):
# Size of the thing over time.
return Decimal(self.usage()["network.incoming.bytes"].volume())

3
bin/bill.py
View File

@ -98,7 +98,8 @@ if __name__ == '__main__':
usage.save()
invoice.bill(usage.vms)
invoice.bill(usage.volumes)
# invoice.bill(usage.objects)
invoice.bill(usage.objects)
invoice.bill(usage.networks)
invoice.close()
print invoice.total()

9
bin/usage.py
View File

@ -115,9 +115,16 @@ if __name__ == '__main__':
# A Usage set is the entirety of time for this Tenant.
# It's not time-limited at all.
print "# Virtual Machines #"
invoice.bill(usage.vms)
print "# Volumes #"
invoice.bill(usage.volumes)
# invoice.bill(usage.objects)
print "# Objects #"
invoice.bill(usage.objects)
print "# Networks #"
invoice.bill(usage.networks)
print "# Floating IPs #"
invoice.bill(usage.ips)
print "Total invoice value: %s" % invoice.total()
print format_title("End of Tenant: %s" % tenant.name, max_len)

30
examples/conf.yaml
View File

@ -12,11 +12,37 @@ invoice_object:
output_file: '%(tenant)s-%(start)s-%(end)s.csv'
output_path: ./
row_layout:
vm:
- name
- id
- location
- type
- flavor
- start
- end
- amount
- uptime
- rate
- cost
object:
- id
- size
- rate
- cost
volume:
- id
- size
- rate
- cost
network:
- id
- incoming
- outgoing
- incoming_rate
- outgoing_rate
- cost
ip:
- id
- duration
- rate
- cost
rates:
file: /etc/artifice/csv_rates.csv