337 lines
12 KiB
Python
337 lines
12 KiB
Python
# Licensed to the Apache Software Foundation (ASF) under one
|
|
# or more contributor license agreements. See the NOTICE file
|
|
# distributed with this work for additional information
|
|
# regarding copyright ownership. The ASF licenses this file
|
|
# to you under the Apache License, Version 2.0 (the
|
|
# "License"); you may not use this file except in compliance
|
|
# with the License. You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing,
|
|
# software distributed under the License is distributed on an
|
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
# KIND, either express or implied. See the License for the
|
|
# specific language governing permissions and limitations
|
|
# under the License.
|
|
import argparse
|
|
import datetime
|
|
import functools
|
|
import json
|
|
import sys
|
|
import os
|
|
|
|
sys.path.append(os.environ.get('STACKTACH_INSTALL_DIR', '/stacktach'))
|
|
|
|
import usage_audit
|
|
|
|
from stacktach.models import InstanceUsage
|
|
from stacktach import datetime_to_decimal as dt
|
|
from stacktach import models
|
|
from stacktach.reconciler import Reconciler
|
|
from stacktach import stacklog
|
|
|
|
OLD_LAUNCHES_QUERY = """
|
|
select stacktach_instanceusage.id,
|
|
stacktach_instanceusage.instance,
|
|
stacktach_instanceusage.launched_at from stacktach_instanceusage
|
|
left outer join stacktach_instancedeletes on
|
|
stacktach_instanceusage.instance = stacktach_instancedeletes.instance
|
|
left outer join stacktach_instancereconcile on
|
|
stacktach_instanceusage.instance = stacktach_instancereconcile.instance
|
|
where (
|
|
stacktach_instancereconcile.deleted_at is null and (
|
|
stacktach_instancedeletes.deleted_at is null or
|
|
stacktach_instancedeletes.deleted_at > %s
|
|
)
|
|
or (stacktach_instancereconcile.deleted_at is not null and
|
|
stacktach_instancereconcile.deleted_at > %s)
|
|
) and stacktach_instanceusage.launched_at < %s;"""
|
|
|
|
OLD_RECONCILES_QUERY = """
|
|
select stacktach_instancereconcile.id,
|
|
stacktach_instancereconcile.instance,
|
|
stacktach_instancereconcile.launched_at from stacktach_instancereconcile
|
|
left outer join stacktach_instancedeletes on
|
|
stacktach_instancereconcile.instance = stacktach_instancedeletes.instance
|
|
where (
|
|
stacktach_instancereconcile.deleted_at is null and (
|
|
stacktach_instancedeletes.deleted_at is null or
|
|
stacktach_instancedeletes.deleted_at > %s
|
|
)
|
|
or (stacktach_instancereconcile.deleted_at is not null and
|
|
stacktach_instancereconcile.deleted_at > %s)
|
|
) and stacktach_instancereconcile.launched_at < %s;"""
|
|
|
|
DEFAULT_UMS_OFFSET = 4 * 60 * 60 # 4 Hours
|
|
|
|
reconciler = None
|
|
|
|
|
|
def _get_new_launches(beginning, ending):
|
|
filters = {
|
|
'launched_at__gte': beginning,
|
|
'launched_at__lte': ending,
|
|
}
|
|
return models.InstanceUsage.objects.filter(**filters)
|
|
|
|
|
|
def _get_deletes(beginning, ending):
|
|
filters = {
|
|
'deleted_at__gte': beginning,
|
|
'deleted_at__lte': ending,
|
|
}
|
|
return models.InstanceDeletes.objects.filter(**filters)
|
|
|
|
|
|
def _get_exists(beginning, ending):
|
|
filters = {
|
|
'audit_period_beginning': beginning,
|
|
'audit_period_ending__gte': beginning,
|
|
'audit_period_ending__lte': ending,
|
|
}
|
|
return models.InstanceExists.objects.filter(**filters)
|
|
|
|
|
|
def cell_and_compute(instance, launched_at):
|
|
usage_list = InstanceUsage.find(instance, launched_at)
|
|
deployment = None
|
|
usage = None
|
|
if usage_list:
|
|
usage = usage_list[0]
|
|
deployment = usage.latest_deployment_for_request_id()
|
|
cell = (deployment and deployment.name) or '-'
|
|
compute = (usage and usage.host()) or '-'
|
|
return cell, compute
|
|
|
|
|
|
def _audit_launches_to_exists(launches, exists, beginning):
|
|
fails = []
|
|
for (instance, launches) in launches.items():
|
|
if instance in exists:
|
|
for expected in launches:
|
|
found = False
|
|
for actual in exists[instance]:
|
|
if int(expected['launched_at']) == \
|
|
int(actual['launched_at']):
|
|
# HACK (apmelton): Truncate the decimal because we may not
|
|
# have the milliseconds.
|
|
found = True
|
|
|
|
if not found:
|
|
rec = False
|
|
if reconciler:
|
|
args = (expected['id'], beginning)
|
|
rec = reconciler.missing_exists_for_instance(*args)
|
|
launched_at = dt.dt_from_decimal(expected['launched_at'])
|
|
msg = "Couldn't find exists for launch (%s, %s)"
|
|
msg = msg % (instance, launched_at)
|
|
cell, compute = cell_and_compute(instance, launched_at)
|
|
fails.append(['Launch', expected['id'], msg,
|
|
'Y' if rec else 'N', cell, compute])
|
|
else:
|
|
rec = False
|
|
if reconciler:
|
|
args = (launches[0]['id'], beginning)
|
|
rec = reconciler.missing_exists_for_instance(*args)
|
|
msg = "No exists for instance (%s)" % instance
|
|
launched_at = dt.dt_from_decimal(launches[0]['launched_at'])
|
|
cell, compute = cell_and_compute(instance, launched_at)
|
|
fails.append(['-', msg, 'Y' if rec else 'N',
|
|
cell, compute])
|
|
return fails
|
|
|
|
|
|
def _launch_audit_for_period(beginning, ending):
|
|
launches_dict = {}
|
|
new_launches = _get_new_launches(beginning, ending)
|
|
for launch in new_launches:
|
|
instance = launch.instance
|
|
l = {'id': launch.id, 'launched_at': launch.launched_at}
|
|
if instance in launches_dict:
|
|
launches_dict[instance].append(l)
|
|
else:
|
|
launches_dict[instance] = [l, ]
|
|
|
|
# NOTE (apmelton)
|
|
# Django's safe substitution doesn't allow dict substitution...
|
|
# Thus, we send it 'beginning' three times...
|
|
old_launches = models.InstanceUsage.objects\
|
|
.raw(OLD_LAUNCHES_QUERY,
|
|
[beginning, beginning, beginning])
|
|
|
|
old_launches_dict = {}
|
|
for launch in old_launches:
|
|
instance = launch.instance
|
|
l = {'id': launch.id, 'launched_at': launch.launched_at}
|
|
if instance not in old_launches_dict or \
|
|
(old_launches_dict[instance]['launched_at'] <
|
|
launch.launched_at):
|
|
old_launches_dict[instance] = l
|
|
|
|
# NOTE (apmelton)
|
|
# Django's safe substitution doesn't allow dict substitution...
|
|
# Thus, we send it 'beginning' three times...
|
|
old_recs = models.InstanceReconcile.objects\
|
|
.raw(OLD_RECONCILES_QUERY,
|
|
[beginning, beginning, beginning])
|
|
|
|
for rec in old_recs:
|
|
instance = rec.instance
|
|
l = {'id': rec.id, 'launched_at': rec.launched_at}
|
|
if instance not in old_launches_dict or \
|
|
(old_launches_dict[instance]['launched_at'] <
|
|
rec.launched_at):
|
|
old_launches_dict[instance] = l
|
|
|
|
for instance, launch in old_launches_dict.items():
|
|
if instance in launches_dict:
|
|
launches_dict[instance].append(launch)
|
|
else:
|
|
launches_dict[instance] = [launch, ]
|
|
|
|
exists_dict = {}
|
|
exists = _get_exists(beginning, ending)
|
|
for exist in exists:
|
|
instance = exist.instance
|
|
e = {'id': exist.id,
|
|
'launched_at': exist.launched_at,
|
|
'deleted_at': exist.deleted_at}
|
|
if instance in exists_dict:
|
|
exists_dict[instance].append(e)
|
|
else:
|
|
exists_dict[instance] = [e, ]
|
|
|
|
launch_to_exists_fails = _audit_launches_to_exists(launches_dict,
|
|
exists_dict,
|
|
beginning)
|
|
return launch_to_exists_fails, new_launches.count(), len(old_launches_dict)
|
|
|
|
|
|
def audit_for_period(beginning, ending, ums=False, ums_offset=0):
|
|
beginning_decimal = dt.dt_to_decimal(beginning)
|
|
ending_decimal = dt.dt_to_decimal(ending)
|
|
|
|
if ums:
|
|
verifier_audit_func = functools.partial(
|
|
usage_audit._verifier_audit_for_day_ums, ums_offset=ums_offset
|
|
)
|
|
else:
|
|
verifier_audit_func = usage_audit._verifier_audit_for_day
|
|
|
|
(verify_summary,
|
|
verify_detail) = verifier_audit_func(beginning_decimal, ending_decimal,
|
|
models.InstanceExists)
|
|
detail, new_count, old_count = _launch_audit_for_period(beginning_decimal,
|
|
ending_decimal)
|
|
|
|
summary = {
|
|
'verifier': verify_summary,
|
|
'launch_summary': {
|
|
'new_launches': new_count,
|
|
'old_launches': old_count,
|
|
'failures': len(detail)
|
|
},
|
|
}
|
|
|
|
details = {
|
|
'exist_fails': verify_detail,
|
|
'launch_fails': detail,
|
|
}
|
|
|
|
return summary, details
|
|
|
|
|
|
def store_results(start, end, summary, details):
|
|
values = {
|
|
'json': make_json_report(summary, details),
|
|
'created': dt.dt_to_decimal(datetime.datetime.utcnow()),
|
|
'period_start': start,
|
|
'period_end': end,
|
|
'version': 7,
|
|
'name': 'nova usage audit'
|
|
}
|
|
|
|
report = models.JsonReport(**values)
|
|
report.save()
|
|
|
|
|
|
def make_json_report(summary, details):
|
|
report = {
|
|
'summary': summary,
|
|
'exist_fail_headers': ['Exists Row ID', 'Error Description', 'Cell',
|
|
'Compute'],
|
|
'exist_fails': details['exist_fails'],
|
|
'launch_fail_headers': ['Launch Row ID', 'Error Description',
|
|
'Reconciled?', 'Cell', 'Compute'],
|
|
'launch_fails': details['launch_fails']
|
|
}
|
|
return json.dumps(report)
|
|
|
|
|
|
def valid_datetime(d):
|
|
try:
|
|
t = datetime.datetime.strptime(d, "%Y-%m-%d %H:%M:%S")
|
|
return t
|
|
except Exception, e:
|
|
raise argparse.ArgumentTypeError(
|
|
"'%s' is not in YYYY-MM-DD HH:MM:SS format." % d)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
parser = argparse.ArgumentParser('StackTach Nova Usage Audit Report')
|
|
parser.add_argument('--period_length',
|
|
choices=['hour', 'day'], default='day')
|
|
parser.add_argument('--utcdatetime',
|
|
help="Override the end time used to generate report.",
|
|
type=valid_datetime, default=None)
|
|
parser.add_argument('--store',
|
|
help="If set to true, report will be stored. "
|
|
"Otherwise, it will just be printed",
|
|
type=bool, default=False)
|
|
parser.add_argument('--reconcile',
|
|
help="Enabled reconciliation",
|
|
type=bool, default=False)
|
|
parser.add_argument('--reconciler_config',
|
|
help="Location of the reconciler config file",
|
|
type=str,
|
|
default='/etc/stacktach/reconciler-config.json')
|
|
parser.add_argument('--ums',
|
|
help="Use query to match UMS, "
|
|
"period length of 'day' required.",
|
|
action='store_true')
|
|
parser.add_argument('--ums-offset',
|
|
help="UMS' fencepost offset in seconds. Default: 4 days",
|
|
type=int,
|
|
default=DEFAULT_UMS_OFFSET)
|
|
args = parser.parse_args()
|
|
|
|
if args.ums and args.period_length != 'day':
|
|
print "UMS query can only be used with period_length of 'day'."
|
|
sys.exit(0)
|
|
|
|
stacklog.set_default_logger_name('nova_usage_audit')
|
|
parent_logger = stacklog.get_logger('nova_usage_audit', is_parent=True)
|
|
log_listener = stacklog.LogListener(parent_logger)
|
|
log_listener.start()
|
|
|
|
if args.reconcile:
|
|
with open(args.reconciler_config) as f:
|
|
reconciler_config = json.load(f)
|
|
reconciler = Reconciler(reconciler_config)
|
|
|
|
if args.utcdatetime is not None:
|
|
time = args.utcdatetime
|
|
else:
|
|
time = datetime.datetime.utcnow()
|
|
|
|
start, end = usage_audit.get_previous_period(time, args.period_length)
|
|
|
|
summary, details = audit_for_period(start, end, ums=args.ums,
|
|
ums_offset=args.ums_offset)
|
|
|
|
if not args.store:
|
|
print make_json_report(summary, details)
|
|
else:
|
|
store_results(start, end, summary, details)
|