x/distil
Code Issues Proposed changes

More pep8 formatting and tidying up.

Browse Source
This commit is contained in:
adriant 2014-01-15 17:47:55 +13:00
parent 757adef5e2
commit 051d2c64ff
7 changed files with 147 additions and 86 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
artifice/csv_mixin.py
View File

@ -5,7 +5,7 @@ class CSV_File_mixin(object):
def write_output(self):
try:
read = open(self.filename)
open(self.filename)
raise RuntimeError("Can't write to an existing file!")
except IOError:
pass

74
artifice/interface.py
View File

@ -43,16 +43,17 @@ def get_meter(meter, start, end, auth):
# Meter is a href; in this case, it has a set of fields with it already.
# print meter.link
# print dir(meter)
date_fields = [{
"field": "timestamp",
"op": "ge",
"value": start.strftime(date_format)
},
{
"field": "timestamp",
"op": "lt",
"value": end.strftime(date_format)
}
date_fields = [
{
"field": "timestamp",
"op": "ge",
"value": start.strftime(date_format)
},
{
"field": "timestamp",
"op": "lt",
"value": end.strftime(date_format)
}
]
fields = []
for field in date_fields:
@ -83,7 +84,7 @@ class keystone(KeystoneClient.Client):
url = "%(url)s/tenants?%(query)s" % {
"url": authenticator,
"query": urllib.urlencode({"name": name})
}
}
r = requests.get(url, headers={
"X-Auth-Token": self.auth_token,
"Content-Type": "application/json"
@ -107,18 +108,22 @@ class Artifice(object):
# This is the Keystone client connection, which provides our
# OpenStack authentication
self.auth = keystone(
username= config["openstack"]["username"],
password= config["openstack"]["password"],
tenant_name= config["openstack"]["default_tenant"],
auth_url= config["openstack"]["authentication_url"]
username=config["openstack"]["username"],
password=config["openstack"]["password"],
tenant_name=config["openstack"]["default_tenant"],
auth_url=config["openstack"]["authentication_url"]
)
conn_string = 'postgresql://%(username)s:%(password)s@%(host)s:%(port)s/%(database)s' % {
conn_dict = {
"username": config["database"]["username"],
"password": config["database"]["password"],
"host": config["database"]["host"],
"port": config["database"]["port"],
"host": config["database"]["host"],
"port": config["database"]["port"],
"database": config["database"]["database"]
}
conn_string = ('postgresql://%(username)s:%(password)s@' +
'%(host)s:%(port)s/%(database)s') % conn_dict
engine = create_engine(conn_string)
Session.configure(bind=engine)
self.session = Session()
@ -211,7 +216,8 @@ class Tenant(object):
if self.invoice_type is None:
invoice_type = self.conn.config["main"]["invoice:object"]
if ":" not in invoice_type:
raise AttributeError("Invoice configuration incorrect! %s" % invoice_type)
raise AttributeError("Invoice configuration incorrect! %s" %
invoice_type)
module, call = invoice_type.split(":")
_package = __import__(module, globals(), locals(), [call])
@ -223,10 +229,11 @@ class Tenant(object):
def resources(self, start, end):
if not self._resources:
date_fields = [{"field": "project_id",
"op": "eq",
"value": self.tenant["id"]
},
date_fields = [
{"field": "project_id",
"op": "eq",
"value": self.tenant["id"]
},
]
# Sets up our resources as Ceilometer objects.
# That's cool, I think.
@ -485,7 +492,8 @@ class Artifact(object):
def save(self):
"""
Persists to our database backend. Opinionatedly this is a sql datastore.
Persists to our database backend.
Opinionatedly this is a sql datastore.
"""
value = self.volume()
session = self.resource.conn.session
@ -509,7 +517,8 @@ class Artifact(object):
session.add(tenant)
else:
try:
res = session.query(resources.Resource).filter(resources.Resource.id == resource_id)[0]
matching = resources.Resource.id == resource_id
res = session.query(resources.Resource).filter(matching)[0]
tenant = res.tenant
except IndexError:
res = resources.Resource()
@ -572,21 +581,26 @@ class Gauge(Artifact):
curr = [usage[0]]
last = usage[0]
try:
last["timestamp"] = datetime.datetime.strptime(last["timestamp"], date_format)
last["timestamp"] = datetime.datetime.strptime(last["timestamp"],
date_format)
except ValueError:
last["timestamp"] = datetime.datetime.strptime(last["timestamp"], other_date_format)
last["timestamp"] = datetime.datetime.strptime(last["timestamp"],
other_date_format)
except TypeError:
pass
for val in usage[1:]:
try:
val["timestamp"] = datetime.datetime.strptime(val["timestamp"], date_format)
val["timestamp"] = datetime.datetime.strptime(val["timestamp"],
date_format)
except ValueError:
val["timestamp"] = datetime.datetime.strptime(val["timestamp"], other_date_format)
val["timestamp"] = datetime.datetime.strptime(val["timestamp"],
other_date_format)
except TypeError:
pass
if (val['timestamp'] - last["timestamp"]) > datetime.timedelta(hours=1):
difference = (val['timestamp'] - last["timestamp"])
if difference > datetime.timedelta(hours=1):
blocks.append(curr)
curr = [val]
last = val

60
artifice/invoice.py
View File

@ -12,18 +12,30 @@ an Invoice interface consists of:
from decimal import *
class IntegrityViolation(BaseException): pass
import csv
class BillingOverlap(BaseException): pass
class NoSuchType(KeyError): pass
class IntegrityViolation(BaseException):
pass
class BillingOverlap(BaseException):
pass
class NoSuchType(KeyError):
pass
class NoSuchLocation(KeyError):
pass
class NoSuchLocation(KeyError): pass
costs = {
"cpu_util" : { "nova": "1" }
"cpu_util": {"nova": "1"}
}
class Costs(object):
def cost(self, location, name):
@ -39,6 +51,7 @@ class Costs(object):
required = ["add_line", "close"]
def requirements(name, parents, attrs):
for attr_name in required:
try:
@ -52,6 +65,7 @@ def requirements(name, parents, attrs):
raise RuntimeError("%s is not callable" % (attr_name))
return type(name, parents, attrs)
class Invoice(object):
# __metaclass__ = requirements
@ -77,17 +91,17 @@ class Invoice(object):
# DC is the name of the DC/region. Or the internal code. W/E.
# print datacenter
self.subheading(dc["name"])
for section in dc["sections"]: # will be vm, network, storage
self.add_section( section )
for section in dc["sections"]: # will be vm, network, storage
self.add_section(section)
meters = dc["sections"][section]
for usage in meters:
cost = self.cost( dc["name"], meter["name"] )
self.add_line( "%s per unit " % cost, usage.volume, cost * usage.volume )
self.commit() # Writes to OpenERP? Closes the invoice? Something.
cost = self.cost(dc["name"], meter["name"])
self.add_line("%s per unit " % (cost, usage.volume,
cost * usage.volume))
self.commit() # Writes to OpenERP? Closes the invoice? Something.
def add_line(self, item):
raise NotImplementedError("Not implemented in base class")
@ -98,7 +112,7 @@ class Invoice(object):
def total(self):
raise NotImplementedError("Not implemented in the base class")
import csv
class RatesFileMixin(object):
# Mixin
# Adds a rates file loader, expecting various things from the
@ -112,15 +126,16 @@ class RatesFileMixin(object):
if not self.__rates:
self.__rates = {}
try:
fh = open(self.config["rates"][ "file" ])
reader = csv.reader(fh, delimiter = "|") # Makes no opinions on the file structure
fh = open(self.config["rates"]["file"])
# Makes no opinions on the file structure
reader = csv.reader(fh, delimiter="|")
for row in reader:
# The default layout is expected to be:
# location | rate name | rate measurement | rate value
self.__rates[row[1].strip()] = {
"cost": Decimal(row[3].strip()),
"region": row[0].strip(),
"measures": row[2].strip()
"cost": Decimal(row[3].strip()),
"region": row[0].strip(),
"measures": row[2].strip()
}
if not self.__rates:
raise IndexError("malformed rates CSV!")
@ -134,7 +149,8 @@ class RatesFileMixin(object):
except IOError:
print "Couldn't open the file!"
raise
return self.__rates[name]["cost"] # ignore the regions-ness for now
return self.__rates[name]["cost"] # ignore the regions-ness for now
class NamesFileMixin(object):
@ -149,12 +165,13 @@ class NamesFileMixin(object):
if not self.__names:
self.__names = {}
try:
fh = open(self.config["rates"][ "names" ])
reader = csv.reader(fh, delimiter="|") # Makes no opinions on the file structure
fh = open(self.config["rates"]["names"])
# Makes no opinions on the file structure
reader = csv.reader(fh, delimiter="|")
for row in reader:
# The default layout is expected to be:
# internal name | external name
self.__names[ row[0].strip() ] = row[1].strip()
self.__names[row[0].strip()] = row[1].strip()
if not self.__names:
raise IndexError("Malformed names CSV")
@ -167,4 +184,3 @@ class NamesFileMixin(object):
print "Couldn't open the file!"
raise
return self.__names[name]

3
artifice/models/tenants.py
View File

@ -1,10 +1,11 @@
from . import Base
from sqlalchemy import Column, types, String
class Tenant(Base):
__tablename__ = 'tenants'
# ID is a uuid
id = Column(String, primary_key=True, nullable=False)
other_id = Column(String)
# Some reference data to something else?
# Some reference data to something else?

22
artifice/models/usage.py
View File

@ -1,11 +1,13 @@
from . import Base
from .resources import Resource
from sqlalchemy import Column, types, ForeignKey, CheckConstraint, String, Integer, type_coerce, func, Sequence
from sqlalchemy.orm import relationship, backref
from sqlalchemy import (Column, types, String, Integer, type_coerce,
func, Sequence)
from sqlalchemy.orm import relationship
from sqlalchemy.schema import ForeignKeyConstraint
import datetime
from sqlalchemy.dialects.postgresql import ExcludeConstraint, TSRANGE, ARRAY
from sqlalchemy.dialects.postgresql import ExcludeConstraint, TSRANGE
class TSRange(TSRANGE):
@ -45,16 +47,14 @@ class Usage(Base):
),
)
resource = relationship(Resource, primaryjoin=
resource_id == Resource.id)
tenant = relationship(Resource, primaryjoin=
tenant_id== Resource.tenant_id)
resource = relationship(Resource,
primaryjoin=(resource_id == Resource.id))
tenant = relationship(Resource,
primaryjoin=(tenant_id == Resource.tenant_id))
# resource = relationship("Resource", backref=backref("resources", order_by=created))
# tenant = relationship("Tenant", backref=backref("usage", order_by=created))
def __init__(self, resource, tenant, value, start, end):
assert start < end
@ -67,7 +67,7 @@ class Usage(Base):
assert resource.tenant.id == tenant.id
self.resource = resource
self.tenant = resource # Same resource
self.tenant = resource # Same resource
self.time = "[%s,%s]" % (start, end)
self.created = datetime.datetime.now()
self.volume = value
self.volume = value

20
bin/bill.py
View File

@ -7,7 +7,7 @@ try:
except ImportError:
loc, fn = os.path.split(__file__)
print loc
here = os.path.abspath(os.path.join(loc +"/../"))
here = os.path.abspath(os.path.join(loc + "/../"))
sys.path.insert(0, here)
# # Are we potentially in a virtualenv? Add that in.
# if os.path.exists( os.path.join(here, "lib/python2.7" ) ):
@ -27,12 +27,13 @@ if __name__ == '__main__':
parser = argparse.ArgumentParser()
# Takes names to display.
# none means display them all.
parser.add_argument("-t", "--tenant", dest="tenants", help='Tenant to display', action="append", default=[])
parser.add_argument("-t", "--tenant", dest="tenants",
help='Tenant to display', action="append", default=[])
# Add some sections to show data from.
# Empty is display all
parser.add_argument("-s", "--section", dest="sections", help="Sections to display", action="append")
parser.add_argument("-s", "--section", dest="sections",
help="Sections to display", action="append")
# Ranging
# We want to get stuff from, to.
@ -44,10 +45,14 @@ if __name__ == '__main__':
type=date_fmt_fnc,
default=datetime.datetime.now() - datetime.timedelta(days=31)
)
parser.add_argument("--to", dest="end", help="When to end our date range. Defaults to yesterday.",
type=date_fmt_fnc, default=datetime.datetime.now() - datetime.timedelta(days=1) )
parser.add_argument(
"--to", dest="end",
help="When to end our date range. Defaults to yesterday.",
type=date_fmt_fnc,
default=datetime.datetime.now() - datetime.timedelta(days=1))
parser.add_argument("--config", dest="config", help="Config file", default="/opt/stack/artifice/etc/artifice/conf.yaml")
parser.add_argument("--config", dest="config", help="Config file",
default="/opt/stack/artifice/etc/artifice/conf.yaml")
args = parser.parse_args()
@ -58,7 +63,6 @@ if __name__ == '__main__':
print "couldn't load %s " % args.config
sys.exit(1)
# Make ourselves a nice interaction object
# Password needs to be loaded from /etc/artifice/database
fh = open(conf["database"]["password_path"])

52
bin/usage.py
View File

@ -1,13 +1,14 @@
#!/usr/bin/env python
import os, sys
import os
import sys
try:
from artifice import interface
except ImportError:
loc, fn = os.path.split(__file__)
print loc
here = os.path.abspath(os.path.join(loc +"/../"))
here = os.path.abspath(os.path.join(loc + "/../"))
sys.path.insert(0, here)
# # Are we potentially in a virtualenv? Add that in.
# if os.path.exists( os.path.join(here, "lib/python2.7" ) ):
@ -21,6 +22,7 @@ date_format = "%Y-%m-%dT%H:%M:%S"
other_date_format = "%Y-%m-%dT%H:%M:%S.%f"
date_fmt = "%Y-%m-%d"
def date_fmt_fnc(val):
return datetime.datetime.strptime(val, date_fmt)
@ -29,12 +31,16 @@ if __name__ == '__main__':
parser = argparse.ArgumentParser()
# Takes names to display.
# none means display them all.
parser.add_argument("-t", "--tenant", dest="tenants", help='Tenant to display', action="append", default=[])
parser.add_argument(
"-t", "--tenant", dest="tenants",
help='Tenant to display',
action="append", default=[])
# Add some sections to show data from.
# Empty is display all
parser.add_argument("-s", "--section", dest="sections", help="Sections to display", action="append")
parser.add_argument(
"-s", "--section", dest="sections",
help="Sections to display", action="append")
# Ranging
# We want to get stuff from, to.
@ -46,13 +52,34 @@ if __name__ == '__main__':
type=date_fmt_fnc,
default=datetime.datetime.now() - datetime.timedelta(days=31)
)
parser.add_argument("--to", dest="end", help="When to end our date range. Defaults to yesterday.",
type=date_fmt_fnc, default=datetime.datetime.now() - datetime.timedelta(days=1) )
parser.add_argument(
"--to", dest="end",
help="When to end our date range. Defaults to yesterday.",
type=date_fmt_fnc,
default=datetime.datetime.now() - datetime.timedelta(days=1))
parser.add_argument("-c", "--config", dest="config", help="Config file", default="/opt/stack/artifice/etc/artifice/conf.yaml")
parser.add_argument(
"-c", "--config", dest="config",
help="Config file",
default="/opt/stack/artifice/etc/artifice/conf.yaml")
args = parser.parse_args()
print "\n # ----------------- usage summary for: ----------------- # "
def format_title(name, max_length):
numb_lines = (max_length - len(name)) / 2
lines = ""
for i in range(numb_lines):
lines = lines + "-"
if (max_length - len(name)) % 2 == 0:
lines2 = lines
else:
lines2 = lines + "-"
title = "\n # " + lines + " " + name + " " + lines2 + " # "
return title
max_len = 60
print format_title("usage summary for:", max_len)
print "Range: %s -> %s" % (args.start, args.end)
try:
conf = yaml.load(open(args.config).read())
@ -60,7 +87,7 @@ if __name__ == '__main__':
# Whoops
print "couldn't load %s " % args.config
sys.exit(1)
fh = open(conf["database"]["password_path"])
password = fh.read()
fh.close()
@ -81,8 +108,7 @@ if __name__ == '__main__':
tenant = instance.tenant(tenant_name)
# Makes a new invoice up for this tenant.
invoice = tenant.invoice(args.start, args.end)
print "\n# ------------------------ Tenant: %s ------------------------ #" % tenant.name
print format_title("Tenant: %s" % tenant.name, max_len)
# usage = tenant.usage(start, end)
usage = tenant.usage(args.start, args.end)
@ -94,4 +120,4 @@ if __name__ == '__main__':
# invoice.bill(usage.objects)
print "Total invoice value: %s" % invoice.total()
print "# --------------------- End of Tenant: %s --------------------- #" % tenant.name
print format_title("End of Tenant: %s" % tenant.name, max_len)