x/distil
Code Issues Proposed changes

Basic fixes to get artifice working. Most just hacky workarounds to get it running for demo purposes.

Browse Source 
Some few useful bug fixes such as replacement of password without "\n", and issues with "flavor.name" vs "instance_type".
This commit is contained in:
adriant 2014-01-06 14:25:33 +13:00
parent b816b1de49
commit 67aaf5f5ab
9 changed files with 386 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
artifice/billing/csv_invoice.py
View File

@ -6,13 +6,13 @@ from decimal import *
class Csv(invoice.RatesFileMixin, invoice.NamesFileMixin, invoice.Invoice):
def __init__(self, tenant, config):
def __init__(self, tenant, start, end, config):
self.config = config
self.tenant = tenant
self.lines = []
self.closed = False
self.start = None
self.end = None
self.start = start
self.end = end
# This has been moved to the mixin
# try:
# fh = open(config["rates"][ "file" ])
@ -37,6 +37,8 @@ class Csv(invoice.RatesFileMixin, invoice.NamesFileMixin, invoice.Invoice):
appendee.append(None)
continue
# What do we expect element to be?
if key == "rate":
appendee.append(self.rate(self.pretty_name(element.type)))
if key == "type":
# Fetch the 'pretty' name from the mappings, if any
# The default is that this returns the internal name
@ -50,8 +52,17 @@ class Csv(invoice.RatesFileMixin, invoice.NamesFileMixin, invoice.Invoice):
x = self.config["row_layout"].index("cost")
appendee[ x ] = element.amount.volume() * \
self.rate( self.pretty_name(element.type) )
print self.rate(self.pretty_name(element.type))
print appendee
# print (str(appendee[1]) + " - From: " + str(appendee[2]) +
# ", Until: " + str(appendee[3]))
print ("type: " + str(self.pretty_name(element.get("type"))))
try:
print " - name : " + str(element.get("name"))
except:
# Just means it isn't a VM
pass
print " - usage: " + str(element.amount.volume())
print " - rate: " + str(self.rate(self.pretty_name(element.type)))
print " - cost: " + str(appendee[ x ])
except ValueError:
# Not in this array. Well okay.
@ -70,10 +81,11 @@ class Csv(invoice.RatesFileMixin, invoice.NamesFileMixin, invoice.Invoice):
def filename(self):
fn = os.path.join(
self.config["output_path"],
self.config["output_file"] % dict(tenant=self.tenant,
self.config["output_file"] % dict(tenant=self.tenant.tenant['name'],
start=self.start, end=self.end)
)
return fn
def close(self):
try:
read = open( self.filename )
@ -81,12 +93,24 @@ class Csv(invoice.RatesFileMixin, invoice.NamesFileMixin, invoice.Invoice):
except IOError:
pass
fh = open(self.filename, "w")
csvwriter = writer(fh, dialect='excel', delimiter=',')
csvwriter.writerow(["", "from", "until"])
csvwriter.writerow(["usage range: ", str(self.start), str(self.end)])
csvwriter.writerow([])
csvwriter.writerow(self.config["row_layout"])
for line in self.lines:
# Line is expected to be an iterable row
csvwriter.writerow(line)
# write a blank line
csvwriter.writerow([])
# write total
total = ["total: ", self.total()]
csvwriter.writerow(total)
fh.close()
self.closed = True

6
artifice/clerk_mixins.py
View File

@ -1,6 +1,6 @@
import requests
from decimal import Decimal
# from artifice import NotFound
from artifice import NotFound
class ClerkNamesMixin(object):
@ -13,8 +13,8 @@ class ClerkNamesMixin(object):
if response.status_code == 200:
return str(response.json()['pretty_name'])
elif response.status_code == 404:
# raise NotFound
print "not found"
raise NotFound
class ClerkRatesMixin(object):
@ -28,5 +28,5 @@ class ClerkRatesMixin(object):
if response.status_code == 200:
return Decimal(response.json()['rate'])
elif response.status_code == 404:
# raise NotFound
print "not found"
raise NotFound

80
artifice/interface.py
View File

@ -140,7 +140,7 @@ class Artifice(object):
"""
# :raises: AttributeError, KeyError
# How does this get implemented ? Should there be a module injection?
return host # For the moment, passthrough
return "Data Center 1" # For the moment, passthrough
# TODO: FIXME.
def tenant(self, name):
@ -218,22 +218,12 @@ class Tenant(object):
funct = getattr(_package, call)
self.invoice_type = funct
config = self.conn.config["invoice_object"]
invoice = self.invoice_type(self, config)
invoice = self.invoice_type(self, start, end, config)
return invoice
def resources(self, start, end):
if not self._resources:
date_fields = [{
"field": "timestamp",
"op": "ge",
"value": start.strftime(date_format)
},
{
"field": "timestamp",
"op": "lt",
"value": end.strftime(date_format)
},
{ "field": "project_id",
date_fields = [{ "field": "project_id",
"op": "eq",
"value": self.tenant["id"]
},
@ -553,7 +543,18 @@ class Cumulative(Artifact):
def volume(self):
measurements = self.usage
measurements = sorted( measurements, key= lambda x: x["timestamp"] )
total_usage = measurements[-1]["counter_volume"] - measurements[0]["counter_volume"]
count = 1
usage = 0
last_measurement = None
for measurement in measurements:
if measurement["counter_volume"] <= 0 and last_measurement is not None:
usage = usage + last_measurement["counter_volume"]
count = count + 1
last_measurement = measurement
usage = usage + measurements[-1]["counter_volume"]
total_usage = usage - measurements[0]["counter_volume"]
return total_usage
@ -575,7 +576,7 @@ class Gauge(Artifact):
except ValueError:
last["timestamp"] = datetime.datetime.strptime(last["timestamp"], other_date_format)
except TypeError:
pass
pass
for val in usage[1:]:
try:
@ -603,5 +604,54 @@ class Gauge(Artifact):
# print totals
return sum(totals)
def uptime(self):
"""THIS IS AN OVERRIDE METHOD FOR A QUICK AND DIRTY DEMO!
DO NOT ACTAULLY USE THIS IN PROD. DELETE IT QUICKLY."""
usage = sorted(self.usage, key=lambda x: x["timestamp"])
blocks = []
curr = [usage[0]]
last = usage[0]
try:
last["timestamp"] = datetime.datetime.strptime(last["timestamp"], date_format)
except ValueError:
last["timestamp"] = datetime.datetime.strptime(last["timestamp"], other_date_format)
except TypeError:
pass
next_hour = True
for val in usage[1:]:
try:
val["timestamp"] = datetime.datetime.strptime(val["timestamp"], date_format)
except ValueError:
val["timestamp"] = datetime.datetime.strptime(val["timestamp"], other_date_format)
except TypeError:
pass
if (val['timestamp'] - last["timestamp"]) > datetime.timedelta(hours=1):
blocks.append(curr)
curr = [val]
last = val
next_hour = False
else:
curr.append(val)
next_hour = True
# We are now sorted into 1-hour blocks
totals = []
for block in blocks:
usage = max( [v["counter_volume"] for v in block])
totals.append( usage )
# totals = [max(x, key=lambda val: val["counter_volume"] ) for x in blocks]
# totals is now an array of max values per hour for a given month.
# print totals
if next_hour:
return sum(totals) + 1
else:
return sum(totals)
class Delta(Artifact):
pass

4
artifice/invoice.py
View File

@ -56,7 +56,7 @@ class Invoice(object):
# __metaclass__ = requirements
def __init__(self, tenant, config):
def __init__(self, tenant, start, end, config):
self.tenant = tenant
self.config = config
@ -149,7 +149,7 @@ class NamesFileMixin(object):
if not self.__names:
self.__names = {}
try:
fh = open(self.config["rates"][ "name" ])
fh = open(self.config["rates"][ "names" ])
reader = csv.reader(fh, delimiter="|") # Makes no opinions on the file structure
for row in reader:
# The default layout is expected to be:

34
artifice/models/resources.py
View File

@ -34,7 +34,6 @@ class BaseModelConstruct(object):
# Returns a given name value thing?
# Based on patterning, this is expected to be a dict of usage information
# based on a meter, I guess?
return getattr(self, name)
def _fetch_meter_name(self, name):
@ -68,27 +67,52 @@ class VM(BaseModelConstruct):
# The only relevant meters of interest are the type of the interest
# and the amount of network we care about.
# Oh, and floating IPs.
relevant_meters = ["instance:<type>", "network.incoming.bytes", "network.outgoing.bytes"]
relevant_meters = [ "instance", "cpu", "instance:<type>", "network.incoming.bytes", "network.outgoing.bytes"]
def _fetch_meter_name(self, name):
if name == "instance:<type>":
return "instance:%s" % self.type
return name
@property
def uptime(self):
return self.amount
@property
def amount(self):
# cpu_usage = self.usage()['cpu'].volume()
# cpu_usage_in_s = Decimal(cpu_usage / 1000000000)
# class Amount(object):
# def volume(self):
# return Decimal(cpu_usage_in_s)
# def __str__(self):
# return str(cpu_usage_in_s) + " s"
# def __repr__(self):
# return str(self)
uptime = self.usage()['instance'].uptime()
class Amount(object):
def volume(self):
return Decimal(1.0)
return Decimal(uptime)
def __str__(self):
return "1.0"
return str(uptime) + " hr"
def __repr__(self):
return str(self)
return Amount()
@property
def type(self):
return self._raw["metadata"]["nistance_type"]
# TODO FIgure out what the hell is going on with ceilometer here,
# and why flavor.name isn't always there, and why sometimes instance_type
# is needed instead....
try:
# print "\"flavor.name\" was used"
return self._raw["metadata"]["flavor.name"]
except KeyError:
# print "\"instance_type\" was used"
return self._raw["metadata"]["instance_type"]
@property
def size(self):

21
bin/bill.py
View File

@ -37,7 +37,13 @@ if __name__ == '__main__':
# Ranging
# We want to get stuff from, to.
parser.add_argument("--from", dest="start", help="When to start our range, date format %s", type=date_fmt_fnc)
parser.add_argument(
"--from",
dest="start",
help="When to start our range, date format %s",
type=date_fmt_fnc,
default=datetime.datetime.now() - datetime.timedelta(days=31)
)
parser.add_argument("--to", dest="end", help="When to end our date range. Defaults to yesterday.",
type=date_fmt_fnc, default=datetime.datetime.now() - datetime.timedelta(days=1) )
@ -59,13 +65,16 @@ if __name__ == '__main__':
password = fh.read()
fh.close()
# Make ourselves a nice interaction object
conf["database"]["password"] = password
conf["database"]["password"] = password.replace("\n", "")
n = interface.Artifice(conf)
tenants = args.tenants
if not args.tenants:
# only parse this list of tenants
tenants = n.tenants
print "\n # ----------------- bill summary for: ----------------- # "
print "Range: %s -> %s" % (args.start, args.end)
for tenant_name in tenants:
# artifact = n.tenant(tenant_name).section(section).usage(args.start, args.end)
# data should now be an artifact-like construct.
@ -75,9 +84,8 @@ if __name__ == '__main__':
tenant = n.tenant(tenant_name)
# Makes a new invoice up for this tenant.
invoice = tenant.invoice(args.start, args.end)
print "Tenant: %s" % tenant.name
print "Range: %s -> %s" % (args.start, args.end)
print "\n# ------------------------ Tenant: %s ------------------------ #" % tenant.name
# usage = tenant.usage(start, end)
usage = tenant.usage(args.start, args.end)
# A Usage set is the entirety of time for this Tenant.
@ -86,10 +94,11 @@ if __name__ == '__main__':
usage.save()
invoice.bill(usage.vms)
invoice.bill(usage.volumes)
invoice.bill(usage.objects)
# invoice.bill(usage.objects)
invoice.close()
print invoice.total()
print "# --------------------- End of Tenant: %s --------------------- #" % tenant.name
# for datacenter, sections in usage.iteritems():
# # DC is the name of the DC/region. Or the internal code. W/E.

139
bin/bill.py~ Normal file
View File

@ -0,0 +1,139 @@
#!/usr/bin/env python
import sys, os
try:
from artifice import interface
except ImportError:
loc, fn = os.path.split(__file__)
print loc
here = os.path.abspath(os.path.join(loc +"/../"))
sys.path.insert(0, here)
# # Are we potentially in a virtualenv? Add that in.
# if os.path.exists( os.path.join(here, "lib/python2.7" ) ):
# sys.path.insert(1, os.path.join(here, "lib/python2.7"))
from artifice import interface
import datetime
import yaml
date_format = "%Y-%m-%dT%H:%M:%S"
other_date_format = "%Y-%m-%dT%H:%M:%S.%f"
date_fmt_fnc = lambda x: datetime.datetime.strptime(date_format)
if __name__ == '__main__':
import argparse
parser = argparse.ArgumentParser()
# Takes names to display.
# none means display them all.
parser.add_argument("-t", "--tenant", dest="tenants", help='Tenant to display', action="append", default=[])
# Add some sections to show data from.
# Empty is display all
parser.add_argument("-s", "--section", dest="sections", help="Sections to display", action="append")
# Ranging
# We want to get stuff from, to.
parser.add_argument(
"--from",
dest="start",
help="When to start our range, date format %s",
type=date_fmt_fnc,
default=datetime.datetime.now() - datetime.timedelta(days=31)
)
parser.add_argument("--to", dest="end", help="When to end our date range. Defaults to yesterday.",
type=date_fmt_fnc, default=datetime.datetime.now() - datetime.timedelta(days=1) )
parser.add_argument("--config", dest="config", help="Config file", default="/opt/stack/artifice/etc/artifice/conf.yaml")
args = parser.parse_args()
try:
conf = yaml.load(open(args.config).read())
except IOError:
# Whoops
print "couldn't load %s " % args.config
sys.exit(1)
# Make ourselves a nice interaction object
# Password needs to be loaded from /etc/artifice/database
fh = open(conf["database"]["password_path"])
password = fh.read()
fh.close()
# Make ourselves a nice interaction object
conf["database"]["password"] = password
n = interface.Artifice(conf)
tenants = args.tenants
if not args.tenants:
# only parse this list of tenants
tenants = n.tenants
print "\n # ----------------- bill summary for: ----------------- # "
print "Range: %s -> %s" % (args.start, args.end)
for tenant_name in tenants:
# artifact = n.tenant(tenant_name).section(section).usage(args.start, args.end)
# data should now be an artifact-like construct.
# Data also knows about Samples, where as an Artifact doesn't.
# An artifact knows its section
tenant = n.tenant(tenant_name)
# Makes a new invoice up for this tenant.
invoice = tenant.invoice(args.start, args.end)
print "\n# ------------------------ Tenant: %s ------------------------ #" % tenant.name
# usage = tenant.usage(start, end)
usage = tenant.usage(args.start, args.end)
# A Usage set is the entirety of time for this Tenant.
# It's not time-limited at all.
# But the
usage.save()
invoice.bill(usage.vms)
invoice.bill(usage.volumes)
# invoice.bill(usage.objects)
invoice.close()
print invoice.total()
print "# --------------------- End of Tenant: %s --------------------- #" % tenant.name
# for datacenter, sections in usage.iteritems():
# # DC is the name of the DC/region. Or the internal code. W/E.
# print datacenter
# for section_name in args.sections:
# assert section in sections
# # section = sections[ section ]
# print sections[section_name]
# for resources in sections[section_name]:
# for resource in resources:
# print resource
# for meter in resource.meters:
# usage = meter.usage(start, end)
# if usage.has_been_saved():
# continue
# print usage.volume()
# print usage.cost()
# usage.save()
# # Finally, bill it.
# # All of these things need to be converted to the
# # publicly-viewable version now.
# invoice.bill(datacenter, resource, meter, usage)
# # Section is going to be in the set of vm, network, storage, image
# # # or just all of them.
# # # It's not going to be an individual meter name.
# # artifacts = section.usage(args.start, args.end)
# # for artifact in artifacts:
# # if artifact.has_been_saved:
# # # Does this artifact exist in the DB?
# # continue
# # artifact.save() # Save to the Artifact storage
# # # Saves to the invoice.
# # invoice.bill ( artifact )
# # # artifact.bill( invoice.id )
# # print "%s: %s" % (section.name, artifact.volume)

8
bin/usage.py
View File

@ -52,6 +52,7 @@ if __name__ == '__main__':
parser.add_argument("-c", "--config", dest="config", help="Config file", default="/opt/stack/artifice/etc/artifice/conf.yaml")
args = parser.parse_args()
print "\n # ----------------- usage summary for: ----------------- # "
print "Range: %s -> %s" % (args.start, args.end)
try:
conf = yaml.load(open(args.config).read())
@ -64,7 +65,7 @@ if __name__ == '__main__':
password = fh.read()
fh.close()
# Make ourselves a nice interaction object
conf["database"]["password"] = password
conf["database"]["password"] = password.replace("\n", "")
instance = interface.Artifice(conf)
tenants = args.tenants
if not args.tenants:
@ -80,7 +81,7 @@ if __name__ == '__main__':
tenant = instance.tenant(tenant_name)
# Makes a new invoice up for this tenant.
invoice = tenant.invoice(args.start, args.end)
print "Tenant: %s" % tenant.name
print "\n# ------------------------ Tenant: %s ------------------------ #" % tenant.name
# usage = tenant.usage(start, end)
@ -90,6 +91,7 @@ if __name__ == '__main__':
invoice.bill(usage.vms)
invoice.bill(usage.volumes)
invoice.bill(usage.objects)
# invoice.bill(usage.objects)
print "Total invoice value: %s" % invoice.total()
print "# --------------------- End of Tenant: %s --------------------- #" % tenant.name

97
bin/usage.py~ Normal file
View File

@ -0,0 +1,97 @@
#!/usr/bin/env python
import os, sys
try:
from artifice import interface
except ImportError:
loc, fn = os.path.split(__file__)
print loc
here = os.path.abspath(os.path.join(loc +"/../"))
sys.path.insert(0, here)
# # Are we potentially in a virtualenv? Add that in.
# if os.path.exists( os.path.join(here, "lib/python2.7" ) ):
# sys.path.insert(1, os.path.join(here, "lib/python2.7"))
from artifice import interface
import datetime
import yaml
date_format = "%Y-%m-%dT%H:%M:%S"
other_date_format = "%Y-%m-%dT%H:%M:%S.%f"
date_fmt = "%Y-%m-%d"
def date_fmt_fnc(val):
return datetime.datetime.strptime(val, date_fmt)
if __name__ == '__main__':
import argparse
parser = argparse.ArgumentParser()
# Takes names to display.
# none means display them all.
parser.add_argument("-t", "--tenant", dest="tenants", help='Tenant to display', action="append", default=[])
# Add some sections to show data from.
# Empty is display all
parser.add_argument("-s", "--section", dest="sections", help="Sections to display", action="append")
# Ranging
# We want to get stuff from, to.
parser.add_argument(
"--from",
dest="start",
help="When to start our range, date format %s",
type=date_fmt_fnc,
default=datetime.datetime.now() - datetime.timedelta(days=31)
)
parser.add_argument("--to", dest="end", help="When to end our date range. Defaults to yesterday.",
type=date_fmt_fnc, default=datetime.datetime.now() - datetime.timedelta(days=1) )
parser.add_argument("-c", "--config", dest="config", help="Config file", default="/opt/stack/artifice/etc/artifice/conf.yaml")
args = parser.parse_args()
print "\n # ----------------- usage summary for: ----------------- # "
print "Range: %s -> %s" % (args.start, args.end)
try:
conf = yaml.load(open(args.config).read())
except IOError:
# Whoops
print "couldn't load %s " % args.config
sys.exit(1)
fh = open(conf["database"]["password_path"])
password = fh.read()
fh.close()
# Make ourselves a nice interaction object
conf["database"]["password"] = password
instance = interface.Artifice(conf)
tenants = args.tenants
if not args.tenants:
# only parse this list of tenants
tenants = instance.tenants
for tenant_name in tenants:
# artifact = n.tenant(tenant_name).section(section).usage(args.start, args.end)
# data should now be an artifact-like construct.
# Data also knows about Samples, where as an Artifact doesn't.
# An artifact knows its section
tenant = instance.tenant(tenant_name)
# Makes a new invoice up for this tenant.
invoice = tenant.invoice(args.start, args.end)
print "\n# ------------------------ Tenant: %s ------------------------ #" % tenant.name
# usage = tenant.usage(start, end)
usage = tenant.usage(args.start, args.end)
# A Usage set is the entirety of time for this Tenant.
# It's not time-limited at all.
invoice.bill(usage.vms)
invoice.bill(usage.volumes)
# invoice.bill(usage.objects)
print "Total invoice value: %s" % invoice.total()
print "# --------------------- End of Tenant: %s --------------------- #" % tenant.name