From 4cb3198d51b6a517f2f490cadc6bf612cf69f50e Mon Sep 17 00:00:00 2001 From: Aurynn Shaw Date: Fri, 31 Jan 2014 10:24:04 +1300 Subject: [PATCH] Changing things around. adding a simple web API to handle billing pre-computed usage data as pulled from the DB instead of Ceilometer. --- .gitignore | 1 + 2.0/api/models/resource.py | 0 2.0/api/web.py | 0 api/web.py | 54 ++++++++++++++ artifice/interface.py | 2 +- artifice/models/usage.py | 42 +++++++++++ bin/collect | 9 +++ bin/collect.py | 144 +++++++++++++++++++++++++++++++++++++ 8 files changed, 251 insertions(+), 1 deletion(-) create mode 100644 2.0/api/models/resource.py create mode 100644 2.0/api/web.py create mode 100644 api/web.py create mode 100755 bin/collect create mode 100644 bin/collect.py diff --git a/.gitignore b/.gitignore index 3b377bf..cad237f 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ .vagrant work *.swp +*~ diff --git a/2.0/api/models/resource.py b/2.0/api/models/resource.py new file mode 100644 index 0000000..e69de29 diff --git a/2.0/api/web.py b/2.0/api/web.py new file mode 100644 index 0000000..e69de29 diff --git a/api/web.py b/api/web.py new file mode 100644 index 0000000..2986a80 --- /dev/null +++ b/api/web.py @@ -0,0 +1,54 @@ +from flask import Flask +app = Flask(__name__) + +from artifice.models import Session, usage + +conn_string = ('postgresql://%(username)s:%(password)s@' + + '%(host)s:%(port)s/%(database)s') % conn_dict + +Session.configure(bind=create_engine(conn_string)) + +db = Session() + +config = load_config() + +current_region = "None" # FIXME +def fetch_endpoint(region): + return config.get("keystone_endpoint") + # return "http://0.0.0.0:35357/v2.0" # t\/his ought to be in config. #FIXME + +def keystone(func): + + admin_token = config.get("admin_token") + def _perform_keystone(*args, **kwargs): + headers = flask.request.headers + if not 'user_id' in headers: + flask.abort(401) # authentication required + + endpoint = fetch_endpoint( current_region ) + keystone = keystoneclient.v2_0.client.Client(token=admin_token, + endpoint=endpoint) + + return _perform_keystone + +@app.get("/usage") +@keystone +def retrieve_usage(): + """Retrieves usage for a given tenant ID. + Tenant ID will be passed in via the query string. + Expects a keystone auth string in the headers + and will attempt to perform keystone auth + """ + tenant = flask.request.params.get("tenant", None) + if not tenant: + flask.abort(403, json.dumps({"error":"tenant ID required"})) # Bad request + + +@app.post("/bill") +@keystone +def make_a_bill(): + """Generates a bill for a given user. + Expects a JSON dict, with a tenant id and a time range. + Authentication is expected to be present. + """ + diff --git a/artifice/interface.py b/artifice/interface.py index 9fcec6e..806ac0b 100644 --- a/artifice/interface.py +++ b/artifice/interface.py @@ -506,7 +506,7 @@ class Artifact(object): def __init__(self, resource, usage, start, end): self.resource = resource - self.usage = usage + self.usage = usage # Raw meter data from Ceilometer self.start = start self.end = end diff --git a/artifice/models/usage.py b/artifice/models/usage.py index fcb0c3a..94b887e 100644 --- a/artifice/models/usage.py +++ b/artifice/models/usage.py @@ -21,6 +21,48 @@ class TSRange(TSRANGE): # return "'%s::tsrange'" % bindvalue +class Billed(Base): + + """ + Stores a fully-qualified billable entry. + Is expected to cover a full date range (a month), store the volume + of use for a given resource, the rate it's billed at, and the total billable + value. + Uses postgres' TSRange to assert that there are no overlapping regions in the + billable range. + """ + + __tablename__ = 'billable' + + id_ = Column(Integer, Sequence('billable_id_seq'), primary_key=True) + resource_id = Column(String) + tenant_id = Column(String) + + volume = Column(String, nullable=False) + time = Column(TSRange, nullable=False) + created = Column(types.DateTime, nullable=False) + rate = Column(types.Numeric, nullable=False) + total = Column(Types.Numeric, nullable=False) + + __table_args__ = ( + ExcludeConstraint( + ('tenant_id', '='), + ('resource_id', '='), + ('time', '&&') + ), + ForeignKeyConstraint( + ["resource_id", "tenant_id"], + ["resources.id", "resources.tenant_id"], + name="fk_resource", use_alter=True + ), + ) + + resource = relationship(Resource, + primaryjoin=(resource_id == Resource.id)) + tenant = relationship(Resource, + primaryjoin=(tenant_id == Resource.tenant_id)) + + class Usage(Base): __tablename__ = 'usage' diff --git a/bin/collect b/bin/collect new file mode 100755 index 0000000..9730583 --- /dev/null +++ b/bin/collect @@ -0,0 +1,9 @@ +#!/bin/bash + +INSTALLED="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +ORIGIN=`pwd` + +# Bring up our python environment +# Pass through all our command line opts as expected + +$INSTALLED/../env/bin/python $INSTALLED/bill.py $@ \ No newline at end of file diff --git a/bin/collect.py b/bin/collect.py new file mode 100644 index 0000000..ae600f7 --- /dev/null +++ b/bin/collect.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python + +import sys, os + +try: + from artifice import interface +except ImportError: + loc, fn = os.path.split(__file__) + print loc + here = os.path.abspath(os.path.join(loc + "/../")) + sys.path.insert(0, here) + # # Are we potentially in a virtualenv? Add that in. + # if os.path.exists( os.path.join(here, "lib/python2.7" ) ): + # sys.path.insert(1, os.path.join(here, "lib/python2.7")) + from artifice import interface + +import datetime +import yaml + +date_format = "%Y-%m-%dT%H:%M:%S" +other_date_format = "%Y-%m-%dT%H:%M:%S.%f" + +date_fmt_fnc = lambda x: datetime.datetime.strptime(date_format) + +if __name__ == '__main__': + import argparse + parser = argparse.ArgumentParser() + # Takes names to display. + # none means display them all. + parser.add_argument("-t", "--tenant", dest="tenants", + help='Tenant to display', action="append", default=[]) + + # Add some sections to show data from. + # Empty is display all + parser.add_argument("-s", "--section", dest="sections", + help="Sections to display", action="append") + + # Ranging + # We want to get stuff from, to. + + parser.add_argument( + "--from", + dest="start", + help="When to start our range, date format %s" % ( date_fmt.replace("%","%%") ) , + type=date_fmt_fnc, + default=datetime.datetime.now() - datetime.timedelta(days=31) + ) + parser.add_argument( + "--to", dest="end", + help="When to end our date range. Defaults to yesterday.", + type=date_fmt_fnc, + default=datetime.datetime.now() - datetime.timedelta(days=1)) + + parser.add_argument("--config", dest="config", help="Config file", + default="/opt/stack/artifice/etc/artifice/conf.yaml") + + args = parser.parse_args() + + try: + conf = yaml.load(open(args.config).read()) + except IOError: + # Whoops + print "couldn't load %s " % args.config + sys.exit(1) + + # Make ourselves a nice interaction object + # Password needs to be loaded from /etc/artifice/database + fh = open(conf["database"]["password_path"]) + password = fh.read() + fh.close() + # Make ourselves a nice interaction object + conf["database"]["password"] = password.replace("\n", "") + n = interface.Artifice(conf) + tenants = args.tenants + if not args.tenants: + # only parse this list of tenants + tenants = n.tenants + + print "\n # ----------------- bill summary for: ----------------- # " + print "Range: %s -> %s" % (args.start, args.end) + + for tenant_name in tenants: + # artifact = n.tenant(tenant_name).section(section).usage(args.start, args.end) + # data should now be an artifact-like construct. + # Data also knows about Samples, where as an Artifact doesn't. + + # An artifact knows its section + tenant = n.tenant(tenant_name) + # Makes a new invoice up for this tenant. + invoice = tenant.invoice(args.start, args.end) + print "\n# ------------------------ Tenant: %s ------------------------ #" % tenant.name + + # usage = tenant.usage(start, end) + usage = tenant.usage(args.start, args.end) + # A Usage set is the entirety of time for this Tenant. + # It's not time-limited at all. + # But the + usage.save() + invoice.bill(usage.vms) + invoice.bill(usage.volumes) + invoice.bill(usage.objects) + invoice.bill(usage.networks) + invoice.close() + + print invoice.total() + print "# --------------------- End of Tenant: %s --------------------- #" % tenant.name + + # for datacenter, sections in usage.iteritems(): + # # DC is the name of the DC/region. Or the internal code. W/E. + # print datacenter + + # for section_name in args.sections: + # assert section in sections + + # # section = sections[ section ] + # print sections[section_name] + # for resources in sections[section_name]: + # for resource in resources: + # print resource + # for meter in resource.meters: + # usage = meter.usage(start, end) + # if usage.has_been_saved(): + # continue + # print usage.volume() + # print usage.cost() + # usage.save() + # # Finally, bill it. + # # All of these things need to be converted to the + # # publicly-viewable version now. + # invoice.bill(datacenter, resource, meter, usage) + + # # Section is going to be in the set of vm, network, storage, image + # # # or just all of them. + # # # It's not going to be an individual meter name. + # # artifacts = section.usage(args.start, args.end) + # # for artifact in artifacts: + # # if artifact.has_been_saved: + # # # Does this artifact exist in the DB? + # # continue + # # artifact.save() # Save to the Artifact storage + # # # Saves to the invoice. + # # invoice.bill ( artifact ) + # # # artifact.bill( invoice.id ) + # # print "%s: %s" % (section.name, artifact.volume)