Basic fixes to get artifice working. Most just hacky workarounds to get it running for demo purposes.

Some few useful bug fixes such as replacement of password without "\n", and issues with "flavor.name" vs "instance_type".

artifice/billing/csv_invoice.py

@@ -6,13 +6,13 @@ from decimal import *
 
 class Csv(invoice.RatesFileMixin, invoice.NamesFileMixin, invoice.Invoice):
 
-    def __init__(self, tenant, config):
+    def __init__(self, tenant, start, end, config):
         self.config = config
         self.tenant = tenant
         self.lines = []
         self.closed = False
-        self.start = None
+        self.start = start
-        self.end = None
+        self.end = end
         # This has been moved to the mixin
         # try:
         #     fh = open(config["rates"][ "file" ])
@@ -37,6 +37,8 @@ class Csv(invoice.RatesFileMixin, invoice.NamesFileMixin, invoice.Invoice):
                     appendee.append(None)
                     continue
                 # What do we expect element to be?
+                if key == "rate":
+                    appendee.append(self.rate(self.pretty_name(element.type)))                    
                 if key == "type":
                     # Fetch the 'pretty' name from the mappings, if any
                     # The default is that this returns the internal name
@@ -50,8 +52,17 @@ class Csv(invoice.RatesFileMixin, invoice.NamesFileMixin, invoice.Invoice):
                 x = self.config["row_layout"].index("cost")
                 appendee[ x ] = element.amount.volume() * \
                         self.rate( self.pretty_name(element.type) )
-                print self.rate(self.pretty_name(element.type))
+                # print (str(appendee[1]) + " - From: " + str(appendee[2]) +
-                print appendee
+                #        ", Until: " + str(appendee[3]))
+                print ("type: " + str(self.pretty_name(element.get("type"))))
+                try:
+                    print " - name : " + str(element.get("name")) 
+                except:
+                    # Just means it isn't a VM
+                    pass
+                print "   - usage: " + str(element.amount.volume())
+                print "   - rate: " + str(self.rate(self.pretty_name(element.type)))
+                print " - cost: " + str(appendee[ x ])
 
             except ValueError:
                 # Not in this array. Well okay.
@@ -70,10 +81,11 @@ class Csv(invoice.RatesFileMixin, invoice.NamesFileMixin, invoice.Invoice):
     def filename(self):
         fn = os.path.join(
             self.config["output_path"],
-            self.config["output_file"] % dict(tenant=self.tenant,
+            self.config["output_file"] % dict(tenant=self.tenant.tenant['name'],
                 start=self.start, end=self.end)
         )
         return fn
+
     def close(self):
         try:
             read = open( self.filename )
@@ -81,12 +93,24 @@ class Csv(invoice.RatesFileMixin, invoice.NamesFileMixin, invoice.Invoice):
         except IOError:
             pass
         fh = open(self.filename, "w")
-
         csvwriter = writer(fh, dialect='excel', delimiter=',')
+
+        csvwriter.writerow(["", "from", "until"])
+        csvwriter.writerow(["usage range: ", str(self.start), str(self.end)])
+        csvwriter.writerow([])
+
+        
+        csvwriter.writerow(self.config["row_layout"])
         for line in self.lines:
             # Line is expected to be an iterable row
             csvwriter.writerow(line)
 
+        # write a blank line
+        csvwriter.writerow([])
+        # write total
+        total = ["total: ", self.total()]
+        csvwriter.writerow(total)
+
         fh.close()
         self.closed = True
 

artifice/clerk_mixins.py

@@ -1,6 +1,6 @@
 import requests
 from decimal import Decimal
-# from artifice import NotFound
+from artifice import NotFound
 
 
 class ClerkNamesMixin(object):
@@ -13,8 +13,8 @@ class ClerkNamesMixin(object):
         if response.status_code == 200:
             return str(response.json()['pretty_name'])
         elif response.status_code == 404:
-            # raise NotFound
             print "not found"
+            raise NotFound
 
 
 class ClerkRatesMixin(object):
@@ -28,5 +28,5 @@ class ClerkRatesMixin(object):
         if response.status_code == 200:
             return Decimal(response.json()['rate'])
         elif response.status_code == 404:
-            # raise NotFound
             print "not found"
+            raise NotFound

artifice/interface.py

@@ -140,7 +140,7 @@ class Artifice(object):
         """
         # :raises: AttributeError, KeyError
         # How does this get implemented ? Should there be a module injection?
-        return host # For the moment, passthrough
+        return "Data Center 1" # For the moment, passthrough
         # TODO: FIXME.
 
     def tenant(self, name):
@@ -218,22 +218,12 @@ class Tenant(object):
             funct = getattr(_package, call)
             self.invoice_type = funct
         config = self.conn.config["invoice_object"]
-        invoice = self.invoice_type(self, config)
+        invoice = self.invoice_type(self, start, end, config)
         return invoice
 
     def resources(self, start, end):
         if not self._resources:
-            date_fields = [{
+            date_fields = [{   "field": "project_id",
-                "field": "timestamp",
-                    "op": "ge",
-                    "value": start.strftime(date_format)
-                },
-                {
-                    "field": "timestamp",
-                    "op": "lt",
-                    "value": end.strftime(date_format)
-                },
-                {   "field": "project_id",
                     "op": "eq",
                     "value": self.tenant["id"]
                 },
@@ -553,7 +543,18 @@ class Cumulative(Artifact):
     def volume(self):
         measurements = self.usage
         measurements = sorted( measurements, key= lambda x: x["timestamp"] )
-        total_usage = measurements[-1]["counter_volume"] - measurements[0]["counter_volume"]
+        count = 1
+        usage = 0
+        last_measurement = None
+        for measurement in measurements:
+            if measurement["counter_volume"] <= 0 and last_measurement is not None:
+                usage = usage + last_measurement["counter_volume"]
+            count = count + 1
+            last_measurement = measurement
+
+        usage = usage  + measurements[-1]["counter_volume"]
+
+        total_usage = usage - measurements[0]["counter_volume"]
         return total_usage
 
 
@@ -575,7 +576,7 @@ class Gauge(Artifact):
         except ValueError:
             last["timestamp"] = datetime.datetime.strptime(last["timestamp"], other_date_format)
         except TypeError:
-            pass
+            pass        
 
         for val in usage[1:]:
             try:
@@ -603,5 +604,54 @@ class Gauge(Artifact):
         # print totals
         return sum(totals)
 
+    def uptime(self):
+        """THIS IS AN OVERRIDE METHOD FOR A QUICK AND DIRTY DEMO! 
+           DO NOT ACTAULLY USE THIS IN PROD. DELETE IT QUICKLY."""
+        usage = sorted(self.usage, key=lambda x: x["timestamp"])
+
+        blocks = []
+        curr = [usage[0]]
+        last = usage[0]
+        try:
+            last["timestamp"] = datetime.datetime.strptime(last["timestamp"], date_format)
+        except ValueError:
+            last["timestamp"] = datetime.datetime.strptime(last["timestamp"], other_date_format)
+        except TypeError:
+            pass 
+
+        next_hour = True       
+
+        for val in usage[1:]:
+            try:
+                val["timestamp"] = datetime.datetime.strptime(val["timestamp"], date_format)
+            except ValueError:
+                val["timestamp"] = datetime.datetime.strptime(val["timestamp"], other_date_format)
+            except TypeError:
+                pass
+
+            if (val['timestamp'] - last["timestamp"]) > datetime.timedelta(hours=1):
+                blocks.append(curr)
+                curr = [val]
+                last = val
+                next_hour = False
+            else:
+                curr.append(val)
+                next_hour = True
+
+        # We are now sorted into 1-hour blocks
+        totals = []
+        for block in blocks:
+            usage = max( [v["counter_volume"] for v in block])
+            totals.append( usage )
+
+        # totals = [max(x, key=lambda val: val["counter_volume"] ) for x in blocks]
+        # totals is now an array of max values per hour for a given month.
+        # print totals
+        if next_hour:
+            return sum(totals) + 1
+        else:
+            return sum(totals)
+
+
 class Delta(Artifact):
     pass

artifice/invoice.py

@@ -56,7 +56,7 @@ class Invoice(object):
 
     # __metaclass__ = requirements
 
-    def __init__(self, tenant, config):
+    def __init__(self, tenant, start, end, config):
         self.tenant = tenant
         self.config = config
 
@@ -149,7 +149,7 @@ class NamesFileMixin(object):
         if not self.__names:
             self.__names = {}
             try:
-                fh = open(self.config["rates"][ "name" ])
+                fh = open(self.config["rates"][ "names" ])
                 reader = csv.reader(fh, delimiter="|") # Makes no opinions on the file structure
                 for row in reader:
                     # The default layout is expected to be:

artifice/models/resources.py

@@ -34,7 +34,6 @@ class BaseModelConstruct(object):
         # Returns a given name value thing?
         # Based on patterning, this is expected to be a dict of usage information
         # based on a meter, I guess?
-
         return getattr(self, name)
 
     def _fetch_meter_name(self, name):
@@ -68,27 +67,52 @@ class VM(BaseModelConstruct):
     # The only relevant meters of interest are the type of the interest
     # and the amount of network we care about.
     # Oh, and floating IPs.
-    relevant_meters = ["instance:<type>", "network.incoming.bytes", "network.outgoing.bytes"]
+    relevant_meters = [ "instance", "cpu", "instance:<type>", "network.incoming.bytes", "network.outgoing.bytes"]
 
     def _fetch_meter_name(self, name):
         if name == "instance:<type>":
             return "instance:%s" % self.type
         return name
 
+    @property
+    def uptime(self):
+        return self.amount
+
     @property
     def amount(self):
+        # cpu_usage = self.usage()['cpu'].volume()
+        # cpu_usage_in_s = Decimal(cpu_usage / 1000000000)
+
+        # class Amount(object):
+        #     def volume(self):
+        #         return Decimal(cpu_usage_in_s)
+        #     def __str__(self):
+        #         return str(cpu_usage_in_s) + " s"
+        #     def __repr__(self):
+        #         return str(self)
+
+
+        uptime = self.usage()['instance'].uptime()
         class Amount(object):
             def volume(self):
-                return Decimal(1.0)
+                return Decimal(uptime)
             def __str__(self):
-                return "1.0"
+                return str(uptime) + " hr"
             def __repr__(self):
                 return str(self)
         return Amount()
 
     @property
     def type(self):
-        return self._raw["metadata"]["nistance_type"]
+        # TODO FIgure out what the hell is going on with ceilometer here, 
+        # and why flavor.name isn't always there, and why sometimes instance_type
+        # is needed instead....
+        try:             
+            # print "\"flavor.name\" was used"
+            return self._raw["metadata"]["flavor.name"]
+        except KeyError:
+            # print "\"instance_type\" was used"
+            return self._raw["metadata"]["instance_type"]
 
     @property
     def size(self):

bin/bill.py

@@ -37,7 +37,13 @@ if __name__ == '__main__':
     # Ranging
     # We want to get stuff from, to.
 
-    parser.add_argument("--from", dest="start", help="When to start our range, date format %s", type=date_fmt_fnc)
+    parser.add_argument(
+        "--from",
+        dest="start",
+        help="When to start our range, date format %s",
+        type=date_fmt_fnc,
+        default=datetime.datetime.now() - datetime.timedelta(days=31)
+    )
     parser.add_argument("--to", dest="end", help="When to end our date range. Defaults to yesterday.",
         type=date_fmt_fnc, default=datetime.datetime.now() - datetime.timedelta(days=1) )
 
@@ -59,13 +65,16 @@ if __name__ == '__main__':
     password = fh.read()
     fh.close()
     # Make ourselves a nice interaction object
-    conf["database"]["password"] = password
+    conf["database"]["password"] = password.replace("\n", "")
     n = interface.Artifice(conf)
     tenants = args.tenants
     if not args.tenants:
         # only parse this list of tenants
         tenants = n.tenants
 
+    print "\n # ----------------- bill summary for: ----------------- # "
+    print "Range: %s -> %s" % (args.start, args.end)
+
     for tenant_name in tenants:
         # artifact = n.tenant(tenant_name).section(section).usage(args.start, args.end)
         # data should now be an artifact-like construct.
@@ -75,9 +84,8 @@ if __name__ == '__main__':
         tenant = n.tenant(tenant_name)
         # Makes a new invoice up for this tenant.
         invoice = tenant.invoice(args.start, args.end)
-        print "Tenant: %s" % tenant.name
+        print "\n# ------------------------ Tenant: %s ------------------------ #" % tenant.name
-        print "Range: %s -> %s" % (args.start, args.end)
+        
-
         # usage = tenant.usage(start, end)
         usage = tenant.usage(args.start, args.end)
         # A Usage set is the entirety of time for this Tenant.
@@ -86,10 +94,11 @@ if __name__ == '__main__':
         usage.save()
         invoice.bill(usage.vms)
         invoice.bill(usage.volumes)
-        invoice.bill(usage.objects)
+        # invoice.bill(usage.objects)
         invoice.close()
 
         print invoice.total()
+        print "# --------------------- End of Tenant: %s --------------------- #" % tenant.name
 
         # for datacenter, sections in usage.iteritems():
         #     # DC is the name of the DC/region. Or the internal code. W/E.

bin/bill.py~

@@ -0,0 +1,139 @@
+#!/usr/bin/env python
+
+import sys, os
+
+try:
+    from artifice import interface
+except ImportError:
+    loc, fn = os.path.split(__file__)
+    print loc
+    here =  os.path.abspath(os.path.join(loc +"/../"))
+    sys.path.insert(0, here)
+    # # Are we potentially in a virtualenv? Add that in.
+    # if os.path.exists( os.path.join(here, "lib/python2.7" ) ):
+    #     sys.path.insert(1, os.path.join(here, "lib/python2.7"))
+    from artifice import interface
+
+import datetime
+import yaml
+
+date_format = "%Y-%m-%dT%H:%M:%S"
+other_date_format = "%Y-%m-%dT%H:%M:%S.%f"
+
+date_fmt_fnc = lambda x: datetime.datetime.strptime(date_format)
+
+if __name__ == '__main__':
+    import argparse
+    parser = argparse.ArgumentParser()
+    # Takes names to display.
+    # none means display them all.
+    parser.add_argument("-t", "--tenant", dest="tenants", help='Tenant to display', action="append", default=[])
+
+    # Add some sections to show data from.
+    # Empty is display all
+    parser.add_argument("-s", "--section", dest="sections", help="Sections to display", action="append")
+
+
+    # Ranging
+    # We want to get stuff from, to.
+
+    parser.add_argument(
+        "--from",
+        dest="start",
+        help="When to start our range, date format %s",
+        type=date_fmt_fnc,
+        default=datetime.datetime.now() - datetime.timedelta(days=31)
+    )
+    parser.add_argument("--to", dest="end", help="When to end our date range. Defaults to yesterday.",
+        type=date_fmt_fnc, default=datetime.datetime.now() - datetime.timedelta(days=1) )
+
+    parser.add_argument("--config", dest="config", help="Config file", default="/opt/stack/artifice/etc/artifice/conf.yaml")
+
+    args = parser.parse_args()
+
+    try:
+        conf = yaml.load(open(args.config).read())
+    except IOError:
+        # Whoops
+        print "couldn't load %s " % args.config
+        sys.exit(1)
+
+
+    # Make ourselves a nice interaction object
+    # Password needs to be loaded from /etc/artifice/database
+    fh = open(conf["database"]["password_path"])
+    password = fh.read()
+    fh.close()
+    # Make ourselves a nice interaction object
+    conf["database"]["password"] = password
+    n = interface.Artifice(conf)
+    tenants = args.tenants
+    if not args.tenants:
+        # only parse this list of tenants
+        tenants = n.tenants
+
+    print "\n # ----------------- bill summary for: ----------------- # "
+    print "Range: %s -> %s" % (args.start, args.end)
+
+    for tenant_name in tenants:
+        # artifact = n.tenant(tenant_name).section(section).usage(args.start, args.end)
+        # data should now be an artifact-like construct.
+        # Data also knows about Samples, where as an Artifact doesn't.
+
+        # An artifact knows its section
+        tenant = n.tenant(tenant_name)
+        # Makes a new invoice up for this tenant.
+        invoice = tenant.invoice(args.start, args.end)
+        print "\n# ------------------------ Tenant: %s ------------------------ #" % tenant.name
+        
+        # usage = tenant.usage(start, end)
+        usage = tenant.usage(args.start, args.end)
+        # A Usage set is the entirety of time for this Tenant.
+        # It's not time-limited at all.
+        # But the
+        usage.save()
+        invoice.bill(usage.vms)
+        invoice.bill(usage.volumes)
+        # invoice.bill(usage.objects)
+        invoice.close()
+
+        print invoice.total()
+        print "# --------------------- End of Tenant: %s --------------------- #" % tenant.name
+
+        # for datacenter, sections in usage.iteritems():
+        #     # DC is the name of the DC/region. Or the internal code. W/E.
+        #     print datacenter
+
+        #     for section_name in args.sections:
+        #         assert section in sections
+
+        #         # section = sections[ section ]
+        #         print sections[section_name]
+        #         for resources in sections[section_name]:
+        #             for resource in resources:
+        #                 print resource
+        #                 for meter in resource.meters:
+        #                     usage = meter.usage(start, end)
+        #                     if usage.has_been_saved():
+        #                         continue
+        #                     print usage.volume()
+        #                     print usage.cost()
+        #                     usage.save()
+        #                     # Finally, bill it.
+        #                     # All of these things need to be converted to the
+        #                     # publicly-viewable version now.
+        #                     invoice.bill(datacenter, resource, meter, usage)
+
+        #         # Section is going to be in the set of vm, network, storage, image
+        #         # # or just all of them.
+        #         # # It's not going to be an individual meter name.
+        #         # artifacts = section.usage(args.start, args.end)
+        #         # for artifact in artifacts:
+        #         #     if artifact.has_been_saved:
+        #         #         # Does this artifact exist in the DB?
+        #         #         continue
+        #         #     artifact.save() # Save to the Artifact storage
+        #         #     # Saves to the invoice.
+        #         #     invoice.bill ( artifact )
+        #         #     # artifact.bill( invoice.id )
+        #         # print "%s: %s" % (section.name, artifact.volume)

bin/usage.py

@@ -52,6 +52,7 @@ if __name__ == '__main__':
     parser.add_argument("-c", "--config", dest="config", help="Config file", default="/opt/stack/artifice/etc/artifice/conf.yaml")
 
     args = parser.parse_args()
+    print "\n # ----------------- usage summary for: ----------------- # "
     print "Range: %s -> %s" % (args.start, args.end)
     try:
         conf = yaml.load(open(args.config).read())
@@ -64,7 +65,7 @@ if __name__ == '__main__':
     password = fh.read()
     fh.close()
     # Make ourselves a nice interaction object
-    conf["database"]["password"] = password
+    conf["database"]["password"] = password.replace("\n", "")
     instance = interface.Artifice(conf)
     tenants = args.tenants
     if not args.tenants:
@@ -80,7 +81,7 @@ if __name__ == '__main__':
         tenant = instance.tenant(tenant_name)
         # Makes a new invoice up for this tenant.
         invoice = tenant.invoice(args.start, args.end)
-        print "Tenant: %s" % tenant.name
+        print "\n# ------------------------ Tenant: %s ------------------------ #" % tenant.name
 
 
         # usage = tenant.usage(start, end)
@@ -90,6 +91,7 @@ if __name__ == '__main__':
 
         invoice.bill(usage.vms)
         invoice.bill(usage.volumes)
-        invoice.bill(usage.objects)
+        # invoice.bill(usage.objects)
 
         print "Total invoice value: %s" % invoice.total()
+        print "# --------------------- End of Tenant: %s --------------------- #" % tenant.name

bin/usage.py~

@@ -0,0 +1,97 @@
+#!/usr/bin/env python
+
+import os, sys
+
+try:
+    from artifice import interface
+except ImportError:
+    loc, fn = os.path.split(__file__)
+    print loc
+    here =  os.path.abspath(os.path.join(loc +"/../"))
+    sys.path.insert(0, here)
+    # # Are we potentially in a virtualenv? Add that in.
+    # if os.path.exists( os.path.join(here, "lib/python2.7" ) ):
+    #     sys.path.insert(1, os.path.join(here, "lib/python2.7"))
+    from artifice import interface
+
+import datetime
+import yaml
+
+date_format = "%Y-%m-%dT%H:%M:%S"
+other_date_format = "%Y-%m-%dT%H:%M:%S.%f"
+date_fmt = "%Y-%m-%d"
+
+def date_fmt_fnc(val):
+    return datetime.datetime.strptime(val, date_fmt)
+
+if __name__ == '__main__':
+    import argparse
+    parser = argparse.ArgumentParser()
+    # Takes names to display.
+    # none means display them all.
+    parser.add_argument("-t", "--tenant", dest="tenants", help='Tenant to display', action="append", default=[])
+
+    # Add some sections to show data from.
+    # Empty is display all
+    parser.add_argument("-s", "--section", dest="sections", help="Sections to display", action="append")
+
+
+    # Ranging
+    # We want to get stuff from, to.
+
+    parser.add_argument(
+        "--from",
+        dest="start",
+        help="When to start our range, date format %s",
+        type=date_fmt_fnc,
+        default=datetime.datetime.now() - datetime.timedelta(days=31)
+    )
+    parser.add_argument("--to", dest="end", help="When to end our date range. Defaults to yesterday.",
+        type=date_fmt_fnc, default=datetime.datetime.now() - datetime.timedelta(days=1) )
+
+    parser.add_argument("-c", "--config", dest="config", help="Config file", default="/opt/stack/artifice/etc/artifice/conf.yaml")
+
+    args = parser.parse_args()
+    print "\n # ----------------- usage summary for: ----------------- # "
+    print "Range: %s -> %s" % (args.start, args.end)
+    try:
+        conf = yaml.load(open(args.config).read())
+    except IOError:
+        # Whoops
+        print "couldn't load %s " % args.config
+        sys.exit(1)
+    
+    fh = open(conf["database"]["password_path"])
+    password = fh.read()
+    fh.close()
+    # Make ourselves a nice interaction object
+    conf["database"]["password"] = password
+    instance = interface.Artifice(conf)
+    tenants = args.tenants
+    if not args.tenants:
+        # only parse this list of tenants
+        tenants = instance.tenants
+
+    for tenant_name in tenants:
+        # artifact = n.tenant(tenant_name).section(section).usage(args.start, args.end)
+        # data should now be an artifact-like construct.
+        # Data also knows about Samples, where as an Artifact doesn't.
+
+        # An artifact knows its section
+        tenant = instance.tenant(tenant_name)
+        # Makes a new invoice up for this tenant.
+        invoice = tenant.invoice(args.start, args.end)
+        print "\n# ------------------------ Tenant: %s ------------------------ #" % tenant.name
+
+
+        # usage = tenant.usage(start, end)
+        usage = tenant.usage(args.start, args.end)
+        # A Usage set is the entirety of time for this Tenant.
+        # It's not time-limited at all.
+
+        invoice.bill(usage.vms)
+        invoice.bill(usage.volumes)
+        # invoice.bill(usage.objects)
+
+        print "Total invoice value: %s" % invoice.total()
+        print "# --------------------- End of Tenant: %s --------------------- #" % tenant.name