x/distil
Code Issues Proposed changes

More pep8 formatting and tidying up.

Browse Source
This commit is contained in:
adriant 2014-01-15 17:47:55 +13:00
parent 757adef5e2
commit 051d2c64ff
7 changed files with 147 additions and 86 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
artifice/csv_mixin.py
View File

@ -5,7 +5,7 @@ class CSV_File_mixin(object):
def write_output(self): def write_output(self):
try: try:
read = open(self.filename) open(self.filename)
raise RuntimeError("Can't write to an existing file!") raise RuntimeError("Can't write to an existing file!")
except IOError: except IOError:
pass pass

74
artifice/interface.py
View File

@ -43,16 +43,17 @@ def get_meter(meter, start, end, auth):
# Meter is a href; in this case, it has a set of fields with it already. # Meter is a href; in this case, it has a set of fields with it already.
# print meter.link # print meter.link
# print dir(meter) # print dir(meter)
date_fields = [{ date_fields = [
"field": "timestamp", {
"op": "ge", "field": "timestamp",
"value": start.strftime(date_format) "op": "ge",
}, "value": start.strftime(date_format)
{ },
"field": "timestamp", {
"op": "lt", "field": "timestamp",
"value": end.strftime(date_format) "op": "lt",
} "value": end.strftime(date_format)
}
] ]
fields = [] fields = []
for field in date_fields: for field in date_fields:
@ -83,7 +84,7 @@ class keystone(KeystoneClient.Client):
url = "%(url)s/tenants?%(query)s" % { url = "%(url)s/tenants?%(query)s" % {
"url": authenticator, "url": authenticator,
"query": urllib.urlencode({"name": name}) "query": urllib.urlencode({"name": name})
} }
r = requests.get(url, headers={ r = requests.get(url, headers={
"X-Auth-Token": self.auth_token, "X-Auth-Token": self.auth_token,
"Content-Type": "application/json" "Content-Type": "application/json"
@ -107,18 +108,22 @@ class Artifice(object):
# This is the Keystone client connection, which provides our # This is the Keystone client connection, which provides our
# OpenStack authentication # OpenStack authentication
self.auth = keystone( self.auth = keystone(
username= config["openstack"]["username"], username=config["openstack"]["username"],
password= config["openstack"]["password"], password=config["openstack"]["password"],
tenant_name= config["openstack"]["default_tenant"], tenant_name=config["openstack"]["default_tenant"],
auth_url= config["openstack"]["authentication_url"] auth_url=config["openstack"]["authentication_url"]
) )
conn_string = 'postgresql://%(username)s:%(password)s@%(host)s:%(port)s/%(database)s' % {
conn_dict = {
"username": config["database"]["username"], "username": config["database"]["username"],
"password": config["database"]["password"], "password": config["database"]["password"],
"host": config["database"]["host"], "host": config["database"]["host"],
"port": config["database"]["port"], "port": config["database"]["port"],
"database": config["database"]["database"] "database": config["database"]["database"]
} }
conn_string = ('postgresql://%(username)s:%(password)s@' +
'%(host)s:%(port)s/%(database)s') % conn_dict
engine = create_engine(conn_string) engine = create_engine(conn_string)
Session.configure(bind=engine) Session.configure(bind=engine)
self.session = Session() self.session = Session()
@ -211,7 +216,8 @@ class Tenant(object):
if self.invoice_type is None: if self.invoice_type is None:
invoice_type = self.conn.config["main"]["invoice:object"] invoice_type = self.conn.config["main"]["invoice:object"]
if ":" not in invoice_type: if ":" not in invoice_type:
raise AttributeError("Invoice configuration incorrect! %s" % invoice_type) raise AttributeError("Invoice configuration incorrect! %s" %
invoice_type)
module, call = invoice_type.split(":") module, call = invoice_type.split(":")
_package = __import__(module, globals(), locals(), [call]) _package = __import__(module, globals(), locals(), [call])
@ -223,10 +229,11 @@ class Tenant(object):
def resources(self, start, end): def resources(self, start, end):
if not self._resources: if not self._resources:
date_fields = [{"field": "project_id", date_fields = [
"op": "eq", {"field": "project_id",
"value": self.tenant["id"] "op": "eq",
}, "value": self.tenant["id"]
},
] ]
# Sets up our resources as Ceilometer objects. # Sets up our resources as Ceilometer objects.
# That's cool, I think. # That's cool, I think.
@ -485,7 +492,8 @@ class Artifact(object):
def save(self): def save(self):
""" """
Persists to our database backend. Opinionatedly this is a sql datastore. Persists to our database backend.
Opinionatedly this is a sql datastore.
""" """
value = self.volume() value = self.volume()
session = self.resource.conn.session session = self.resource.conn.session
@ -509,7 +517,8 @@ class Artifact(object):
session.add(tenant) session.add(tenant)
else: else:
try: try:
res = session.query(resources.Resource).filter(resources.Resource.id == resource_id)[0] matching = resources.Resource.id == resource_id
res = session.query(resources.Resource).filter(matching)[0]
tenant = res.tenant tenant = res.tenant
except IndexError: except IndexError:
res = resources.Resource() res = resources.Resource()
@ -572,21 +581,26 @@ class Gauge(Artifact):
curr = [usage[0]] curr = [usage[0]]
last = usage[0] last = usage[0]
try: try:
last["timestamp"] = datetime.datetime.strptime(last["timestamp"], date_format) last["timestamp"] = datetime.datetime.strptime(last["timestamp"],
date_format)
except ValueError: except ValueError:
last["timestamp"] = datetime.datetime.strptime(last["timestamp"], other_date_format) last["timestamp"] = datetime.datetime.strptime(last["timestamp"],
other_date_format)
except TypeError: except TypeError:
pass pass
for val in usage[1:]: for val in usage[1:]:
try: try:
val["timestamp"] = datetime.datetime.strptime(val["timestamp"], date_format) val["timestamp"] = datetime.datetime.strptime(val["timestamp"],
date_format)
except ValueError: except ValueError:
val["timestamp"] = datetime.datetime.strptime(val["timestamp"], other_date_format) val["timestamp"] = datetime.datetime.strptime(val["timestamp"],
other_date_format)
except TypeError: except TypeError:
pass pass
if (val['timestamp'] - last["timestamp"]) > datetime.timedelta(hours=1): difference = (val['timestamp'] - last["timestamp"])
if difference > datetime.timedelta(hours=1):
blocks.append(curr) blocks.append(curr)
curr = [val] curr = [val]
last = val last = val

60
artifice/invoice.py
View File

@ -12,18 +12,30 @@ an Invoice interface consists of:
from decimal import * from decimal import *
class IntegrityViolation(BaseException): pass import csv
class BillingOverlap(BaseException): pass
class NoSuchType(KeyError): pass class IntegrityViolation(BaseException):
pass
class BillingOverlap(BaseException):
pass
class NoSuchType(KeyError):
pass
class NoSuchLocation(KeyError):
pass
class NoSuchLocation(KeyError): pass
costs = { costs = {
"cpu_util" : { "nova": "1" } "cpu_util": {"nova": "1"}
} }
class Costs(object): class Costs(object):
def cost(self, location, name): def cost(self, location, name):
@ -39,6 +51,7 @@ class Costs(object):
required = ["add_line", "close"] required = ["add_line", "close"]
def requirements(name, parents, attrs): def requirements(name, parents, attrs):
for attr_name in required: for attr_name in required:
try: try:
@ -52,6 +65,7 @@ def requirements(name, parents, attrs):
raise RuntimeError("%s is not callable" % (attr_name)) raise RuntimeError("%s is not callable" % (attr_name))
return type(name, parents, attrs) return type(name, parents, attrs)
class Invoice(object): class Invoice(object):
# __metaclass__ = requirements # __metaclass__ = requirements
@ -77,17 +91,17 @@ class Invoice(object):
# DC is the name of the DC/region. Or the internal code. W/E. # DC is the name of the DC/region. Or the internal code. W/E.
# print datacenter # print datacenter
self.subheading(dc["name"]) self.subheading(dc["name"])
for section in dc["sections"]: # will be vm, network, storage for section in dc["sections"]: # will be vm, network, storage
self.add_section( section ) self.add_section(section)
meters = dc["sections"][section] meters = dc["sections"][section]
for usage in meters: for usage in meters:
cost = self.cost( dc["name"], meter["name"] ) cost = self.cost(dc["name"], meter["name"])
self.add_line( "%s per unit " % cost, usage.volume, cost * usage.volume )
self.commit() # Writes to OpenERP? Closes the invoice? Something.
self.add_line("%s per unit " % (cost, usage.volume,
cost * usage.volume))
self.commit() # Writes to OpenERP? Closes the invoice? Something.
def add_line(self, item): def add_line(self, item):
raise NotImplementedError("Not implemented in base class") raise NotImplementedError("Not implemented in base class")
@ -98,7 +112,7 @@ class Invoice(object):
def total(self): def total(self):
raise NotImplementedError("Not implemented in the base class") raise NotImplementedError("Not implemented in the base class")
import csv
class RatesFileMixin(object): class RatesFileMixin(object):
# Mixin # Mixin
# Adds a rates file loader, expecting various things from the # Adds a rates file loader, expecting various things from the
@ -112,15 +126,16 @@ class RatesFileMixin(object):
if not self.__rates: if not self.__rates:
self.__rates = {} self.__rates = {}
try: try:
fh = open(self.config["rates"][ "file" ]) fh = open(self.config["rates"]["file"])
reader = csv.reader(fh, delimiter = "|") # Makes no opinions on the file structure # Makes no opinions on the file structure
reader = csv.reader(fh, delimiter="|")
for row in reader: for row in reader:
# The default layout is expected to be: # The default layout is expected to be:
# location | rate name | rate measurement | rate value # location | rate name | rate measurement | rate value
self.__rates[row[1].strip()] = { self.__rates[row[1].strip()] = {
"cost": Decimal(row[3].strip()), "cost": Decimal(row[3].strip()),
"region": row[0].strip(), "region": row[0].strip(),
"measures": row[2].strip() "measures": row[2].strip()
} }
if not self.__rates: if not self.__rates:
raise IndexError("malformed rates CSV!") raise IndexError("malformed rates CSV!")
@ -134,7 +149,8 @@ class RatesFileMixin(object):
except IOError: except IOError:
print "Couldn't open the file!" print "Couldn't open the file!"
raise raise
return self.__rates[name]["cost"] # ignore the regions-ness for now return self.__rates[name]["cost"] # ignore the regions-ness for now
class NamesFileMixin(object): class NamesFileMixin(object):
@ -149,12 +165,13 @@ class NamesFileMixin(object):
if not self.__names: if not self.__names:
self.__names = {} self.__names = {}
try: try:
fh = open(self.config["rates"][ "names" ]) fh = open(self.config["rates"]["names"])
reader = csv.reader(fh, delimiter="|") # Makes no opinions on the file structure # Makes no opinions on the file structure
reader = csv.reader(fh, delimiter="|")
for row in reader: for row in reader:
# The default layout is expected to be: # The default layout is expected to be:
# internal name | external name # internal name | external name
self.__names[ row[0].strip() ] = row[1].strip() self.__names[row[0].strip()] = row[1].strip()
if not self.__names: if not self.__names:
raise IndexError("Malformed names CSV") raise IndexError("Malformed names CSV")
@ -167,4 +184,3 @@ class NamesFileMixin(object):
print "Couldn't open the file!" print "Couldn't open the file!"
raise raise
return self.__names[name] return self.__names[name]

3
artifice/models/tenants.py
View File

@ -1,10 +1,11 @@
from . import Base from . import Base
from sqlalchemy import Column, types, String from sqlalchemy import Column, types, String
class Tenant(Base): class Tenant(Base):
__tablename__ = 'tenants' __tablename__ = 'tenants'
# ID is a uuid # ID is a uuid
id = Column(String, primary_key=True, nullable=False) id = Column(String, primary_key=True, nullable=False)
other_id = Column(String) other_id = Column(String)
# Some reference data to something else? # Some reference data to something else?

22
artifice/models/usage.py
View File

@ -1,11 +1,13 @@
from . import Base from . import Base
from .resources import Resource from .resources import Resource
from sqlalchemy import Column, types, ForeignKey, CheckConstraint, String, Integer, type_coerce, func, Sequence from sqlalchemy import (Column, types, String, Integer, type_coerce,
from sqlalchemy.orm import relationship, backref func, Sequence)
from sqlalchemy.orm import relationship
from sqlalchemy.schema import ForeignKeyConstraint from sqlalchemy.schema import ForeignKeyConstraint
import datetime import datetime
from sqlalchemy.dialects.postgresql import ExcludeConstraint, TSRANGE, ARRAY from sqlalchemy.dialects.postgresql import ExcludeConstraint, TSRANGE
class TSRange(TSRANGE): class TSRange(TSRANGE):
@ -45,16 +47,14 @@ class Usage(Base):
), ),
) )
resource = relationship(Resource, primaryjoin= resource = relationship(Resource,
resource_id == Resource.id) primaryjoin=(resource_id == Resource.id))
tenant = relationship(Resource, primaryjoin= tenant = relationship(Resource,
tenant_id== Resource.tenant_id) primaryjoin=(tenant_id == Resource.tenant_id))
# resource = relationship("Resource", backref=backref("resources", order_by=created)) # resource = relationship("Resource", backref=backref("resources", order_by=created))
# tenant = relationship("Tenant", backref=backref("usage", order_by=created)) # tenant = relationship("Tenant", backref=backref("usage", order_by=created))
def __init__(self, resource, tenant, value, start, end): def __init__(self, resource, tenant, value, start, end):
assert start < end assert start < end
@ -67,7 +67,7 @@ class Usage(Base):
assert resource.tenant.id == tenant.id assert resource.tenant.id == tenant.id
self.resource = resource self.resource = resource
self.tenant = resource # Same resource self.tenant = resource # Same resource
self.time = "[%s,%s]" % (start, end) self.time = "[%s,%s]" % (start, end)
self.created = datetime.datetime.now() self.created = datetime.datetime.now()
self.volume = value self.volume = value

20
bin/bill.py
View File

@ -7,7 +7,7 @@ try:
except ImportError: except ImportError:
loc, fn = os.path.split(__file__) loc, fn = os.path.split(__file__)
print loc print loc
here = os.path.abspath(os.path.join(loc +"/../")) here = os.path.abspath(os.path.join(loc + "/../"))
sys.path.insert(0, here) sys.path.insert(0, here)
# # Are we potentially in a virtualenv? Add that in. # # Are we potentially in a virtualenv? Add that in.
# if os.path.exists( os.path.join(here, "lib/python2.7" ) ): # if os.path.exists( os.path.join(here, "lib/python2.7" ) ):
@ -27,12 +27,13 @@ if __name__ == '__main__':
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
# Takes names to display. # Takes names to display.
# none means display them all. # none means display them all.
parser.add_argument("-t", "--tenant", dest="tenants", help='Tenant to display', action="append", default=[]) parser.add_argument("-t", "--tenant", dest="tenants",
help='Tenant to display', action="append", default=[])
# Add some sections to show data from. # Add some sections to show data from.
# Empty is display all # Empty is display all
parser.add_argument("-s", "--section", dest="sections", help="Sections to display", action="append") parser.add_argument("-s", "--section", dest="sections",
help="Sections to display", action="append")
# Ranging # Ranging
# We want to get stuff from, to. # We want to get stuff from, to.
@ -44,10 +45,14 @@ if __name__ == '__main__':
type=date_fmt_fnc, type=date_fmt_fnc,
default=datetime.datetime.now() - datetime.timedelta(days=31) default=datetime.datetime.now() - datetime.timedelta(days=31)
) )
parser.add_argument("--to", dest="end", help="When to end our date range. Defaults to yesterday.", parser.add_argument(
type=date_fmt_fnc, default=datetime.datetime.now() - datetime.timedelta(days=1) ) "--to", dest="end",
help="When to end our date range. Defaults to yesterday.",
type=date_fmt_fnc,
default=datetime.datetime.now() - datetime.timedelta(days=1))
parser.add_argument("--config", dest="config", help="Config file", default="/opt/stack/artifice/etc/artifice/conf.yaml") parser.add_argument("--config", dest="config", help="Config file",
default="/opt/stack/artifice/etc/artifice/conf.yaml")
args = parser.parse_args() args = parser.parse_args()
@ -58,7 +63,6 @@ if __name__ == '__main__':
print "couldn't load %s " % args.config print "couldn't load %s " % args.config
sys.exit(1) sys.exit(1)
# Make ourselves a nice interaction object # Make ourselves a nice interaction object
# Password needs to be loaded from /etc/artifice/database # Password needs to be loaded from /etc/artifice/database
fh = open(conf["database"]["password_path"]) fh = open(conf["database"]["password_path"])

52
bin/usage.py
View File

@ -1,13 +1,14 @@
#!/usr/bin/env python #!/usr/bin/env python
import os, sys import os
import sys
try: try:
from artifice import interface from artifice import interface
except ImportError: except ImportError:
loc, fn = os.path.split(__file__) loc, fn = os.path.split(__file__)
print loc print loc
here = os.path.abspath(os.path.join(loc +"/../")) here = os.path.abspath(os.path.join(loc + "/../"))
sys.path.insert(0, here) sys.path.insert(0, here)
# # Are we potentially in a virtualenv? Add that in. # # Are we potentially in a virtualenv? Add that in.
# if os.path.exists( os.path.join(here, "lib/python2.7" ) ): # if os.path.exists( os.path.join(here, "lib/python2.7" ) ):
@ -21,6 +22,7 @@ date_format = "%Y-%m-%dT%H:%M:%S"
other_date_format = "%Y-%m-%dT%H:%M:%S.%f" other_date_format = "%Y-%m-%dT%H:%M:%S.%f"
date_fmt = "%Y-%m-%d" date_fmt = "%Y-%m-%d"
def date_fmt_fnc(val): def date_fmt_fnc(val):
return datetime.datetime.strptime(val, date_fmt) return datetime.datetime.strptime(val, date_fmt)
@ -29,12 +31,16 @@ if __name__ == '__main__':
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
# Takes names to display. # Takes names to display.
# none means display them all. # none means display them all.
parser.add_argument("-t", "--tenant", dest="tenants", help='Tenant to display', action="append", default=[]) parser.add_argument(
"-t", "--tenant", dest="tenants",
help='Tenant to display',
action="append", default=[])
# Add some sections to show data from. # Add some sections to show data from.
# Empty is display all # Empty is display all
parser.add_argument("-s", "--section", dest="sections", help="Sections to display", action="append") parser.add_argument(
"-s", "--section", dest="sections",
help="Sections to display", action="append")
# Ranging # Ranging
# We want to get stuff from, to. # We want to get stuff from, to.
@ -46,13 +52,34 @@ if __name__ == '__main__':
type=date_fmt_fnc, type=date_fmt_fnc,
default=datetime.datetime.now() - datetime.timedelta(days=31) default=datetime.datetime.now() - datetime.timedelta(days=31)
) )
parser.add_argument("--to", dest="end", help="When to end our date range. Defaults to yesterday.", parser.add_argument(
type=date_fmt_fnc, default=datetime.datetime.now() - datetime.timedelta(days=1) ) "--to", dest="end",
help="When to end our date range. Defaults to yesterday.",
type=date_fmt_fnc,
default=datetime.datetime.now() - datetime.timedelta(days=1))
parser.add_argument("-c", "--config", dest="config", help="Config file", default="/opt/stack/artifice/etc/artifice/conf.yaml") parser.add_argument(
"-c", "--config", dest="config",
help="Config file",
default="/opt/stack/artifice/etc/artifice/conf.yaml")
args = parser.parse_args() args = parser.parse_args()
print "\n # ----------------- usage summary for: ----------------- # "
def format_title(name, max_length):
numb_lines = (max_length - len(name)) / 2
lines = ""
for i in range(numb_lines):
lines = lines + "-"
if (max_length - len(name)) % 2 == 0:
lines2 = lines
else:
lines2 = lines + "-"
title = "\n # " + lines + " " + name + " " + lines2 + " # "
return title
max_len = 60
print format_title("usage summary for:", max_len)
print "Range: %s -> %s" % (args.start, args.end) print "Range: %s -> %s" % (args.start, args.end)
try: try:
conf = yaml.load(open(args.config).read()) conf = yaml.load(open(args.config).read())
@ -60,7 +87,7 @@ if __name__ == '__main__':
# Whoops # Whoops
print "couldn't load %s " % args.config print "couldn't load %s " % args.config
sys.exit(1) sys.exit(1)
fh = open(conf["database"]["password_path"]) fh = open(conf["database"]["password_path"])
password = fh.read() password = fh.read()
fh.close() fh.close()
@ -81,8 +108,7 @@ if __name__ == '__main__':
tenant = instance.tenant(tenant_name) tenant = instance.tenant(tenant_name)
# Makes a new invoice up for this tenant. # Makes a new invoice up for this tenant.
invoice = tenant.invoice(args.start, args.end) invoice = tenant.invoice(args.start, args.end)
print "\n# ------------------------ Tenant: %s ------------------------ #" % tenant.name print format_title("Tenant: %s" % tenant.name, max_len)
# usage = tenant.usage(start, end) # usage = tenant.usage(start, end)
usage = tenant.usage(args.start, args.end) usage = tenant.usage(args.start, args.end)
@ -94,4 +120,4 @@ if __name__ == '__main__':
# invoice.bill(usage.objects) # invoice.bill(usage.objects)
print "Total invoice value: %s" % invoice.total() print "Total invoice value: %s" % invoice.total()
print "# --------------------- End of Tenant: %s --------------------- #" % tenant.name print format_title("End of Tenant: %s" % tenant.name, max_len)