8a0f06ac79
Added instance hours report Initial version of report to calculate unit hours used for nova instances Breakdown by flavor, flavor class, account/billing types and by tenant. Moved license so script has shebang as the first line Add tenant info cache. Refactor Instance hr report. Added cache table for basic tenant info for reports. Refactor instance_hours report to use table. Improve performance of tenant info update. use bulk sql operations to speed up the tenant info update, as it's taking ~40s/1000 tenants to update on a decent machine. Fix some tests broken by rebase. Fix unittests broken by rebase. Also, renumber migration due to collision. Add Apache license header to new files. Fixed bug with fetching deployment information in reconciler. Reverted old method for fetching current usage's deployment and added new method to fetch latest deployment information for a request_id. Made the field mismatch error message more readable Refactored nova and glance verifier tests the exists are updated with 201 send_status as part of stacktach down repair mechanism Revert "Fixed bug with fetching deployment information in" Revert "Adding host and deployment info to missing exists entries in the nova usage audit" Revert "Added column headers for host and deployment in json reports" Only log ERROR on last retry fixed the wrong status name for sent_failed variable in audit report fixing documentation for urls that are not available for glance deprecating stacky urls (usage, deletes, exists) that are not used anymore Revert "Revert "Added column headers for host and deployment in json reports"" Revert "Revert "Adding host and deployment info to missing exists entries in the nova usage audit"" Revert "Revert "Fixed bug with fetching deployment information in"" Cell and compute info added for verification failures as well. If that is not present(request_id is not populated for an InstanceUsage entry), the cells display '-' Add tox support for move to stackforge Add tox support for move to stackforge Change-Id: Id94c2a7f1f9061e972e90c3f54e39c9dec11943b
370 lines
14 KiB
Python
370 lines
14 KiB
Python
# Licensed to the Apache Software Foundation (ASF) under one
|
|
# or more contributor license agreements. See the NOTICE file
|
|
# distributed with this work for additional information
|
|
# regarding copyright ownership. The ASF licenses this file
|
|
# to you under the Apache License, Version 2.0 (the
|
|
# "License"); you may not use this file except in compliance
|
|
# with the License. You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing,
|
|
# software distributed under the License is distributed on an
|
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
# KIND, either express or implied. See the License for the
|
|
# specific language governing permissions and limitations
|
|
# under the License.
|
|
import datetime
|
|
import json
|
|
import os
|
|
import sys
|
|
import uuid
|
|
|
|
|
|
POSSIBLE_TOPDIR = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]),
|
|
os.pardir, os.pardir))
|
|
if os.path.exists(os.path.join(POSSIBLE_TOPDIR, 'stacktach')):
|
|
sys.path.insert(0, POSSIBLE_TOPDIR)
|
|
|
|
from verifier import base_verifier
|
|
from verifier import config
|
|
from verifier import NullFieldException
|
|
from stacktach import models
|
|
from stacktach import stacklog
|
|
from stacktach import datetime_to_decimal as dt
|
|
from verifier import FieldMismatch
|
|
from verifier import AmbiguousResults
|
|
from verifier import NotFound
|
|
from verifier import VerificationException
|
|
from stacktach import message_service
|
|
|
|
stacklog.set_default_logger_name('verifier')
|
|
|
|
|
|
def _get_child_logger():
|
|
return stacklog.get_logger('verifier', is_parent=False)
|
|
|
|
|
|
def _verify_field_mismatch(exists, launch):
|
|
flavor_field_name = config.flavor_field_name()
|
|
if not base_verifier._verify_date_field(
|
|
launch.launched_at, exists.launched_at, same_second=True):
|
|
raise FieldMismatch(
|
|
'launched_at',
|
|
{'name': 'exists', 'value': exists.launched_at},
|
|
{'name': 'launches', 'value': launch.launched_at},
|
|
exists.instance)
|
|
|
|
if getattr(launch, flavor_field_name) != \
|
|
getattr(exists, flavor_field_name):
|
|
raise FieldMismatch(
|
|
flavor_field_name,
|
|
{'name': 'exists', 'value': getattr(exists, flavor_field_name)},
|
|
{'name': 'launches', 'value': getattr(launch, flavor_field_name)},
|
|
exists.instance)
|
|
|
|
if launch.tenant != exists.tenant:
|
|
raise FieldMismatch(
|
|
'tenant',
|
|
{'name': 'exists', 'value': exists.tenant},
|
|
{'name': 'launches', 'value': launch.tenant},
|
|
exists.instance)
|
|
|
|
if launch.rax_options != exists.rax_options:
|
|
raise FieldMismatch(
|
|
'rax_options',
|
|
{'name': 'exists', 'value': exists.rax_options},
|
|
{'name': 'launches', 'value': launch.rax_options},
|
|
exists.instance)
|
|
|
|
if launch.os_architecture != exists.os_architecture:
|
|
raise FieldMismatch(
|
|
'os_architecture',
|
|
{'name': 'exists', 'value': exists.os_architecture},
|
|
{'name': 'launches', 'value': launch.os_architecture},
|
|
exists.instance)
|
|
|
|
if launch.os_version != exists.os_version:
|
|
raise FieldMismatch(
|
|
'os_version',
|
|
{'name': 'exists', 'value': exists.os_version},
|
|
{'name': 'launches', 'value': launch.os_version},
|
|
exists.instance)
|
|
|
|
if launch.os_distro != exists.os_distro:
|
|
raise FieldMismatch(
|
|
'os_distro',
|
|
{'name': 'exists', 'value': exists.os_distro},
|
|
{'name': 'launches', 'value': launch.os_distro},
|
|
exists.instance)
|
|
|
|
|
|
def _verify_for_launch(exist, launch=None,
|
|
launch_type="InstanceUsage"):
|
|
|
|
if not launch and exist.usage:
|
|
launch = exist.usage
|
|
elif not launch:
|
|
if models.InstanceUsage.objects\
|
|
.filter(instance=exist.instance).count() > 0:
|
|
launches = models.InstanceUsage.find(
|
|
exist.instance, dt.dt_from_decimal(exist.launched_at))
|
|
count = launches.count()
|
|
query = {
|
|
'instance': exist.instance,
|
|
'launched_at': exist.launched_at
|
|
}
|
|
if count > 1:
|
|
raise AmbiguousResults(launch_type, query)
|
|
elif count == 0:
|
|
raise NotFound(launch_type, query)
|
|
launch = launches[0]
|
|
else:
|
|
raise NotFound(launch_type, {'instance': exist.instance})
|
|
|
|
_verify_field_mismatch(exist, launch)
|
|
|
|
|
|
def _verify_for_delete(exist, delete=None,
|
|
delete_type="InstanceDeletes"):
|
|
|
|
if not delete and exist.delete:
|
|
# We know we have a delete and we have it's id
|
|
delete = exist.delete
|
|
elif not delete:
|
|
if exist.deleted_at:
|
|
# We received this exists before the delete, go find it
|
|
deletes = models.InstanceDeletes.find(
|
|
exist.instance, dt.dt_from_decimal(exist.launched_at))
|
|
if deletes.count() == 1:
|
|
delete = deletes[0]
|
|
else:
|
|
query = {
|
|
'instance': exist.instance,
|
|
'launched_at': exist.launched_at
|
|
}
|
|
raise NotFound(delete_type, query)
|
|
else:
|
|
# We don't know if this is supposed to have a delete or not.
|
|
# Thus, we need to check if we have a delete for this instance.
|
|
# We need to be careful though, since we could be verifying an
|
|
# exist event that we got before the delete. So, we restrict the
|
|
# search to only deletes before this exist's audit period ended.
|
|
# If we find any, we fail validation
|
|
launched_at = dt.dt_from_decimal(exist.launched_at)
|
|
deleted_at_max = dt.dt_from_decimal(exist.audit_period_ending)
|
|
deletes = models.InstanceDeletes.find(exist.instance, launched_at,
|
|
deleted_at_max)
|
|
if deletes.count() > 0:
|
|
reason = 'Found %s for non-delete exist' % delete_type
|
|
raise VerificationException(reason)
|
|
|
|
if delete:
|
|
if not base_verifier._verify_date_field(
|
|
delete.launched_at, exist.launched_at, same_second=True):
|
|
raise FieldMismatch(
|
|
'launched_at',
|
|
{'name': 'exists', 'value': exist.launched_at},
|
|
{'name': 'deletes', 'value': delete.launched_at},
|
|
exist.instance)
|
|
|
|
if not base_verifier._verify_date_field(
|
|
delete.deleted_at, exist.deleted_at, same_second=True):
|
|
raise FieldMismatch(
|
|
'deleted_at',
|
|
{'name': 'exists', 'value': exist.deleted_at},
|
|
{'name': 'deletes', 'value': delete.deleted_at},
|
|
exist.instance)
|
|
|
|
|
|
def _verify_basic_validity(exist):
|
|
flavor_field_name = config.flavor_field_name()
|
|
fields = {
|
|
'tenant': exist.tenant,
|
|
'launched_at': exist.launched_at,
|
|
flavor_field_name: getattr(exist, flavor_field_name)
|
|
}
|
|
for (field_name, field_value) in fields.items():
|
|
if field_value is None:
|
|
raise NullFieldException(field_name, exist.id, exist.instance)
|
|
base_verifier._is_hex_owner_id(
|
|
'tenant', exist.tenant, exist.id, exist.instance)
|
|
base_verifier._is_like_date(
|
|
'launched_at', exist.launched_at, exist.id, exist.instance)
|
|
if exist.deleted_at is not None:
|
|
base_verifier._is_like_date(
|
|
'deleted_at', exist.deleted_at, exist.id, exist.instance)
|
|
|
|
|
|
def _verify_optional_validity(exist):
|
|
is_image_type_import = exist.is_image_type_import()
|
|
fields = {exist.rax_options: 'rax_options',
|
|
exist.os_architecture: 'os_architecture'
|
|
}
|
|
if not is_image_type_import:
|
|
fields.update({exist.os_distro: 'os_distro',
|
|
exist.os_version: 'os_version'})
|
|
for (field_value, field_name) in fields.items():
|
|
if field_value == '':
|
|
raise NullFieldException(field_name, exist.id, exist.instance)
|
|
base_verifier._is_int_in_char(
|
|
'rax_options', exist.rax_options, exist.id, exist.instance)
|
|
base_verifier._is_alphanumeric(
|
|
'os_architecture', exist.os_architecture, exist.id, exist.instance)
|
|
if not is_image_type_import:
|
|
base_verifier._is_alphanumeric(
|
|
'os_distro', exist.os_distro, exist.id, exist.instance)
|
|
base_verifier._is_alphanumeric(
|
|
'os_version', exist.os_version, exist.id, exist.instance)
|
|
|
|
|
|
def _verify_validity(exist, validation_level):
|
|
if validation_level == 'none':
|
|
return
|
|
if validation_level == 'basic':
|
|
_verify_basic_validity(exist)
|
|
if validation_level == 'all':
|
|
_verify_basic_validity(exist)
|
|
_verify_optional_validity(exist)
|
|
|
|
|
|
def _verify_with_reconciled_data(exist):
|
|
if not exist.launched_at:
|
|
raise VerificationException("Exists without a launched_at")
|
|
|
|
query = models.InstanceReconcile.objects.filter(instance=exist.instance)
|
|
if query.count() > 0:
|
|
recs = models.InstanceReconcile.find(exist.instance,
|
|
dt.dt_from_decimal((
|
|
exist.launched_at)))
|
|
search_query = {'instance': exist.instance,
|
|
'launched_at': exist.launched_at}
|
|
count = recs.count()
|
|
if count > 1:
|
|
raise AmbiguousResults('InstanceReconcile', search_query)
|
|
elif count == 0:
|
|
raise NotFound('InstanceReconcile', search_query)
|
|
reconcile = recs[0]
|
|
else:
|
|
raise NotFound('InstanceReconcile', {'instance': exist.instance})
|
|
|
|
_verify_for_launch(exist, launch=reconcile,
|
|
launch_type="InstanceReconcile")
|
|
delete = None
|
|
if reconcile.deleted_at is not None:
|
|
delete = reconcile
|
|
_verify_for_delete(exist, delete=delete, delete_type="InstanceReconcile")
|
|
|
|
|
|
def _attempt_reconciled_verify(exist, orig_e):
|
|
verified = False
|
|
try:
|
|
# Attempt to verify against reconciled data
|
|
_verify_with_reconciled_data(exist)
|
|
verified = True
|
|
exist.mark_verified(reconciled=True)
|
|
except NotFound, rec_e:
|
|
# No reconciled data, just mark it failed
|
|
exist.mark_failed(reason=str(orig_e))
|
|
except VerificationException, rec_e:
|
|
# Verification failed against reconciled data, mark it failed
|
|
# using the second failure.
|
|
exist.mark_failed(reason=str(rec_e))
|
|
except Exception, rec_e:
|
|
exist.mark_failed(reason=rec_e.__class__.__name__)
|
|
_get_child_logger().exception("nova: %s" % rec_e)
|
|
return verified
|
|
|
|
|
|
def _verify(exist, validation_level):
|
|
verified = False
|
|
try:
|
|
if not exist.launched_at:
|
|
raise VerificationException("Exists without a launched_at")
|
|
_verify_validity(exist, validation_level)
|
|
_verify_for_launch(exist)
|
|
_verify_for_delete(exist)
|
|
|
|
verified = True
|
|
exist.mark_verified()
|
|
except VerificationException, orig_e:
|
|
# Something is wrong with the InstanceUsage record
|
|
verified = _attempt_reconciled_verify(exist, orig_e)
|
|
except Exception, e:
|
|
exist.mark_failed(reason=e.__class__.__name__)
|
|
_get_child_logger().exception("nova: %s" % e)
|
|
|
|
return verified, exist
|
|
|
|
|
|
class NovaVerifier(base_verifier.Verifier):
|
|
def __init__(self, config, pool=None, reconciler=None):
|
|
super(NovaVerifier, self).__init__(config,
|
|
pool=pool,
|
|
reconciler=reconciler)
|
|
|
|
def send_verified_notification(self, exist, connection, exchange,
|
|
routing_keys=None):
|
|
# NOTE (apmelton)
|
|
# The exist we're provided from the callback may have cached queries
|
|
# from before it was serialized. We don't want to use them as
|
|
# they could have been lost somewhere in the process forking.
|
|
# So, grab a new InstanceExists object from the database and use it.
|
|
body = models.InstanceExists.objects.get(id=exist.id).raw.json
|
|
json_body = json.loads(body)
|
|
json_body[1]['event_type'] = self.config.nova_event_type()
|
|
json_body[1]['original_message_id'] = json_body[1]['message_id']
|
|
json_body[1]['message_id'] = str(uuid.uuid4())
|
|
if routing_keys is None:
|
|
message_service.send_notification(
|
|
json_body[1], json_body[0], connection, exchange)
|
|
else:
|
|
for key in routing_keys:
|
|
message_service.send_notification(
|
|
json_body[1], key, connection, exchange)
|
|
|
|
def verify_exists(self, callback, exists, verifying_status):
|
|
count = exists.count()
|
|
added = 0
|
|
update_interval = datetime.timedelta(seconds=30)
|
|
next_update = datetime.datetime.utcnow() + update_interval
|
|
_get_child_logger().info("nova: Adding %s exists to queue." % count)
|
|
while added < count:
|
|
for exist in exists[0:1000]:
|
|
exist.update_status(verifying_status)
|
|
exist.save()
|
|
validation_level = self.config.validation_level()
|
|
result = self.pool.apply_async(
|
|
_verify, args=(exist, validation_level),
|
|
callback=callback)
|
|
self.results.append(result)
|
|
added += 1
|
|
if datetime.datetime.utcnow() > next_update:
|
|
values = ((added,) + self.clean_results())
|
|
msg = "nova: N: %s, P: %s, S: %s, E: %s" % values
|
|
_get_child_logger().info(msg)
|
|
next_update = datetime.datetime.utcnow() + update_interval
|
|
return count
|
|
|
|
def verify_for_range(self, ending_max, callback=None):
|
|
sent_unverified_exists = models.InstanceExists.find(
|
|
ending_max=ending_max, status=
|
|
models.InstanceExists.SENT_UNVERIFIED)
|
|
sent_unverified_count = self.verify_exists(None,
|
|
sent_unverified_exists,
|
|
models.InstanceExists.
|
|
SENT_VERIFYING)
|
|
exists = models.InstanceExists.find(
|
|
ending_max=ending_max, status=models.InstanceExists.PENDING)
|
|
count = self.verify_exists(callback, exists,
|
|
models.InstanceExists.VERIFYING)
|
|
return count+sent_unverified_count
|
|
|
|
def reconcile_failed(self):
|
|
for failed_exist in self.failed:
|
|
self.reconciler.failed_validation(failed_exist)
|
|
self.failed = []
|
|
|
|
def exchange(self):
|
|
return 'nova'
|