317 lines
12 KiB
Python
317 lines
12 KiB
Python
# Copyright (c) 2013 - Rackspace Inc.
|
|
#
|
|
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
# of this software and associated documentation files (the "Software"), to
|
|
# deal in the Software without restriction, including without limitation the
|
|
# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
|
|
# sell copies of the Software, and to permit persons to whom the Software is
|
|
# furnished to do so, subject to the following conditions:
|
|
#
|
|
# The above copyright notice and this permission notice shall be included in
|
|
# all copies or substantial portions of the Software.
|
|
#
|
|
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
|
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
|
# IN THE SOFTWARE.
|
|
|
|
import argparse
|
|
import datetime
|
|
import functools
|
|
import json
|
|
import sys
|
|
import os
|
|
|
|
sys.path.append(os.environ.get('STACKTACH_INSTALL_DIR', '/stacktach'))
|
|
|
|
import usage_audit
|
|
|
|
from stacktach import datetime_to_decimal as dt
|
|
from stacktach import models
|
|
from stacktach.reconciler import Reconciler
|
|
from stacktach import stacklog
|
|
|
|
OLD_LAUNCHES_QUERY = """
|
|
select stacktach_instanceusage.id,
|
|
stacktach_instanceusage.instance,
|
|
stacktach_instanceusage.launched_at from stacktach_instanceusage
|
|
left outer join stacktach_instancedeletes on
|
|
stacktach_instanceusage.instance = stacktach_instancedeletes.instance
|
|
left outer join stacktach_instancereconcile on
|
|
stacktach_instanceusage.instance = stacktach_instancereconcile.instance
|
|
where (
|
|
stacktach_instancereconcile.deleted_at is null and (
|
|
stacktach_instancedeletes.deleted_at is null or
|
|
stacktach_instancedeletes.deleted_at > %s
|
|
)
|
|
or (stacktach_instancereconcile.deleted_at is not null and
|
|
stacktach_instancereconcile.deleted_at > %s)
|
|
) and stacktach_instanceusage.launched_at < %s;"""
|
|
|
|
OLD_RECONCILES_QUERY = """
|
|
select stacktach_instancereconcile.id,
|
|
stacktach_instancereconcile.instance,
|
|
stacktach_instancereconcile.launched_at from stacktach_instancereconcile
|
|
left outer join stacktach_instancedeletes on
|
|
stacktach_instancereconcile.instance = stacktach_instancedeletes.instance
|
|
where (
|
|
stacktach_instancereconcile.deleted_at is null and (
|
|
stacktach_instancedeletes.deleted_at is null or
|
|
stacktach_instancedeletes.deleted_at > %s
|
|
)
|
|
or (stacktach_instancereconcile.deleted_at is not null and
|
|
stacktach_instancereconcile.deleted_at > %s)
|
|
) and stacktach_instancereconcile.launched_at < %s;"""
|
|
|
|
DEFAULT_UMS_OFFSET = 4 * 60 * 60 # 4 Hours
|
|
|
|
reconciler = None
|
|
|
|
|
|
def _get_new_launches(beginning, ending):
|
|
filters = {
|
|
'launched_at__gte': beginning,
|
|
'launched_at__lte': ending,
|
|
}
|
|
return models.InstanceUsage.objects.filter(**filters)
|
|
|
|
|
|
def _get_deletes(beginning, ending):
|
|
filters = {
|
|
'deleted_at__gte': beginning,
|
|
'deleted_at__lte': ending,
|
|
}
|
|
return models.InstanceDeletes.objects.filter(**filters)
|
|
|
|
|
|
def _get_exists(beginning, ending):
|
|
filters = {
|
|
'audit_period_beginning': beginning,
|
|
'audit_period_ending__gte': beginning,
|
|
'audit_period_ending__lte': ending,
|
|
}
|
|
return models.InstanceExists.objects.filter(**filters)
|
|
|
|
|
|
def _audit_launches_to_exists(launches, exists, beginning):
|
|
fails = []
|
|
for (instance, launches) in launches.items():
|
|
if instance in exists:
|
|
for expected in launches:
|
|
found = False
|
|
for actual in exists[instance]:
|
|
if int(expected['launched_at']) == \
|
|
int(actual['launched_at']):
|
|
# HACK (apmelton): Truncate the decimal because we may not
|
|
# have the milliseconds.
|
|
found = True
|
|
|
|
if not found:
|
|
rec = False
|
|
if reconciler:
|
|
args = (expected['id'], beginning)
|
|
rec = reconciler.missing_exists_for_instance(*args)
|
|
msg = "Couldn't find exists for launch (%s, %s)"
|
|
msg = msg % (instance, expected['launched_at'])
|
|
fails.append(['Launch', expected['id'], msg, 'Y' if rec else 'N'])
|
|
else:
|
|
rec = False
|
|
if reconciler:
|
|
args = (launches[0]['id'], beginning)
|
|
rec = reconciler.missing_exists_for_instance(*args)
|
|
msg = "No exists for instance (%s)" % instance
|
|
fails.append(['Launch', '-', msg, 'Y' if rec else 'N'])
|
|
return fails
|
|
|
|
|
|
def _launch_audit_for_period(beginning, ending):
|
|
launches_dict = {}
|
|
new_launches = _get_new_launches(beginning, ending)
|
|
for launch in new_launches:
|
|
instance = launch.instance
|
|
l = {'id': launch.id, 'launched_at': launch.launched_at}
|
|
if instance in launches_dict:
|
|
launches_dict[instance].append(l)
|
|
else:
|
|
launches_dict[instance] = [l, ]
|
|
|
|
# NOTE (apmelton)
|
|
# Django's safe substitution doesn't allow dict substitution...
|
|
# Thus, we send it 'beginning' three times...
|
|
old_launches = models.InstanceUsage.objects\
|
|
.raw(OLD_LAUNCHES_QUERY,
|
|
[beginning, beginning, beginning])
|
|
|
|
old_launches_dict = {}
|
|
for launch in old_launches:
|
|
instance = launch.instance
|
|
l = {'id': launch.id, 'launched_at': launch.launched_at}
|
|
if instance not in old_launches_dict or \
|
|
(old_launches_dict[instance]['launched_at'] <
|
|
launch.launched_at):
|
|
old_launches_dict[instance] = l
|
|
|
|
# NOTE (apmelton)
|
|
# Django's safe substitution doesn't allow dict substitution...
|
|
# Thus, we send it 'beginning' three times...
|
|
old_recs = models.InstanceReconcile.objects\
|
|
.raw(OLD_RECONCILES_QUERY,
|
|
[beginning, beginning, beginning])
|
|
|
|
for rec in old_recs:
|
|
instance = rec.instance
|
|
l = {'id': rec.id, 'launched_at': rec.launched_at}
|
|
if instance not in old_launches_dict or \
|
|
(old_launches_dict[instance]['launched_at'] <
|
|
rec.launched_at):
|
|
old_launches_dict[instance] = l
|
|
|
|
for instance, launch in old_launches_dict.items():
|
|
if instance in launches_dict:
|
|
launches_dict[instance].append(launch)
|
|
else:
|
|
launches_dict[instance] = [launch, ]
|
|
|
|
exists_dict = {}
|
|
exists = _get_exists(beginning, ending)
|
|
for exist in exists:
|
|
instance = exist.instance
|
|
e = {'id': exist.id,
|
|
'launched_at': exist.launched_at,
|
|
'deleted_at': exist.deleted_at}
|
|
if instance in exists_dict:
|
|
exists_dict[instance].append(e)
|
|
else:
|
|
exists_dict[instance] = [e, ]
|
|
|
|
launch_to_exists_fails = _audit_launches_to_exists(launches_dict,
|
|
exists_dict,
|
|
beginning)
|
|
return launch_to_exists_fails, new_launches.count(), len(old_launches_dict)
|
|
|
|
|
|
def audit_for_period(beginning, ending, ums=False, ums_offset=0):
|
|
beginning_decimal = dt.dt_to_decimal(beginning)
|
|
ending_decimal = dt.dt_to_decimal(ending)
|
|
|
|
if ums:
|
|
verifier_audit_func = functools.partial(
|
|
usage_audit._verifier_audit_for_day_ums, ums_offset=ums_offset
|
|
)
|
|
else:
|
|
verifier_audit_func = usage_audit._verifier_audit_for_day
|
|
|
|
(verify_summary,
|
|
verify_detail) = verifier_audit_func(beginning_decimal, ending_decimal,
|
|
models.InstanceExists)
|
|
detail, new_count, old_count = _launch_audit_for_period(beginning_decimal,
|
|
ending_decimal)
|
|
|
|
summary = {
|
|
'verifier': verify_summary,
|
|
'launch_summary': {
|
|
'new_launches': new_count,
|
|
'old_launches': old_count,
|
|
'failures': len(detail)
|
|
},
|
|
}
|
|
|
|
details = {
|
|
'exist_fails': verify_detail,
|
|
'launch_fails': detail,
|
|
}
|
|
|
|
return summary, details
|
|
|
|
|
|
def store_results(start, end, summary, details):
|
|
values = {
|
|
'json': make_json_report(summary, details),
|
|
'created': dt.dt_to_decimal(datetime.datetime.utcnow()),
|
|
'period_start': start,
|
|
'period_end': end,
|
|
'version': 6,
|
|
'name': 'nova usage audit'
|
|
}
|
|
|
|
report = models.JsonReport(**values)
|
|
report.save()
|
|
|
|
|
|
def make_json_report(summary, details):
|
|
report = [{'summary': summary},
|
|
['Object', 'ID', 'Error Description', 'Reconciled?']]
|
|
report.extend(details['exist_fails'])
|
|
report.extend(details['launch_fails'])
|
|
return json.dumps(report)
|
|
|
|
|
|
def valid_datetime(d):
|
|
try:
|
|
t = datetime.datetime.strptime(d, "%Y-%m-%d %H:%M:%S")
|
|
return t
|
|
except Exception, e:
|
|
raise argparse.ArgumentTypeError(
|
|
"'%s' is not in YYYY-MM-DD HH:MM:SS format." % d)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
parser = argparse.ArgumentParser('StackTach Nova Usage Audit Report')
|
|
parser.add_argument('--period_length',
|
|
choices=['hour', 'day'], default='day')
|
|
parser.add_argument('--utcdatetime',
|
|
help="Override the end time used to generate report.",
|
|
type=valid_datetime, default=None)
|
|
parser.add_argument('--store',
|
|
help="If set to true, report will be stored. "
|
|
"Otherwise, it will just be printed",
|
|
type=bool, default=False)
|
|
parser.add_argument('--reconcile',
|
|
help="Enabled reconciliation",
|
|
type=bool, default=False)
|
|
parser.add_argument('--reconciler_config',
|
|
help="Location of the reconciler config file",
|
|
type=str,
|
|
default='/etc/stacktach/reconciler-config.json')
|
|
parser.add_argument('--ums',
|
|
help="Use query to match UMS, "
|
|
"period length of 'day' required.",
|
|
action='store_true')
|
|
parser.add_argument('--ums-offset',
|
|
help="UMS' fencepost offset in seconds. Default: 4 days",
|
|
type=int,
|
|
default=DEFAULT_UMS_OFFSET)
|
|
args = parser.parse_args()
|
|
|
|
if args.ums and args.period_length != 'day':
|
|
print "UMS query can only be used with period_length of 'day'."
|
|
sys.exit(0)
|
|
|
|
stacklog.set_default_logger_name('nova_usage_audit')
|
|
parent_logger = stacklog.get_logger('nova_usage_audit', is_parent=True)
|
|
log_listener = stacklog.LogListener(parent_logger)
|
|
log_listener.start()
|
|
|
|
if args.reconcile:
|
|
with open(args.reconciler_config) as f:
|
|
reconciler_config = json.load(f)
|
|
reconciler = Reconciler(reconciler_config)
|
|
|
|
if args.utcdatetime is not None:
|
|
time = args.utcdatetime
|
|
else:
|
|
time = datetime.datetime.utcnow()
|
|
|
|
start, end = usage_audit.get_previous_period(time, args.period_length)
|
|
|
|
summary, details = audit_for_period(start, end, ums=args.ums,
|
|
ums_offset=args.ums_offset)
|
|
|
|
if not args.store:
|
|
print make_json_report(summary, details)
|
|
else:
|
|
store_results(start, end, summary, details)
|