Query Elastic to compare software-metadata
Method to query ElasticSearch for a specific set of browbeat_uuids and compare the metadata to determine if there are differences. This work will also tell the user if a option or value is missing. Eventually, I would like to see us query Elastic for collectd data to see if there has been CPU/Memory/DiskIO increases during a specific Browbeat run -- this is a longer-term goal. Example of this : https://gist.github.com/jtaleric/ffc1508eba3cba9515ca24cfcf23583c Change-Id: Ie65e2c3d505aa2f19ba10109276ba982ee4ab67b
This commit is contained in:
Joe Talerico
parent
6f9aef7987
commit
21900bffad
69
browbeat.py
69
browbeat.py
@ -11,24 +11,25 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from lib.Elastic import browbeat_uuid
|
||||
import argparse
|
||||
import datetime
|
||||
import lib.Elastic
|
||||
import lib.PerfKit
|
||||
import lib.Rally
|
||||
import lib.Shaker
|
||||
import lib.Yoda
|
||||
import lib.WorkloadBase
|
||||
import lib.Tools
|
||||
import argparse
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import datetime
|
||||
import os
|
||||
|
||||
_workload_opts = ['perfkit', 'rally', 'shaker', 'yoda']
|
||||
_config_file = 'browbeat-config.yaml'
|
||||
debug_log_file = 'log/debug.log'
|
||||
|
||||
|
||||
def main():
|
||||
tools = lib.Tools.Tools()
|
||||
parser = argparse.ArgumentParser(
|
||||
@ -41,14 +42,21 @@ def main():
|
||||
help='Provide Browbeat YAML configuration file. Default is ./{}'.format(_config_file))
|
||||
parser.add_argument('workloads', nargs='*', help='Browbeat workload(s). Takes a space separated'
|
||||
' list of workloads ({}) or \"all\"'.format(', '.join(_workload_opts)))
|
||||
parser.add_argument('--debug', action='store_true', help='Enable Debug messages')
|
||||
parser.add_argument('-p','--postprocess',
|
||||
dest="path",help="Path to process, ie results/20170101/")
|
||||
parser.add_argument('--debug', action='store_true',
|
||||
help='Enable Debug messages')
|
||||
parser.add_argument('-p', '--postprocess',
|
||||
dest="path", help="Path to process, ie results/20170101/")
|
||||
parser.add_argument('-c', '--compare',
|
||||
help="Compare metadata", dest="compare",
|
||||
choices=['software-metadata'])
|
||||
parser.add_argument('-u', '--uuid',
|
||||
help="UUIDs to pass", dest="uuids", nargs=2)
|
||||
_cli_args = parser.parse_args()
|
||||
|
||||
_logger = logging.getLogger('browbeat')
|
||||
_logger.setLevel(logging.DEBUG)
|
||||
_formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)7s - %(message)s')
|
||||
_formatter = logging.Formatter(
|
||||
'%(asctime)s - %(name)s - %(levelname)7s - %(message)s')
|
||||
_formatter.converter = time.gmtime
|
||||
_dbg_file = logging.FileHandler(debug_log_file)
|
||||
_dbg_file.setLevel(logging.DEBUG)
|
||||
@ -67,24 +75,35 @@ def main():
|
||||
# Load Browbeat yaml config file:
|
||||
_config = tools._load_config(_cli_args.setup)
|
||||
|
||||
if _cli_args.compare == "software-metadata":
|
||||
es = lib.Elastic.Elastic(_config, "BrowbeatCLI")
|
||||
es.compare_metadata("_all", 'controller', _cli_args.uuids)
|
||||
exit(0)
|
||||
|
||||
if _cli_args.compare:
|
||||
parser.print_help()
|
||||
exit(1)
|
||||
|
||||
# Default to all workloads
|
||||
if _cli_args.workloads == []:
|
||||
_cli_args.workloads.append('all')
|
||||
if _cli_args.path :
|
||||
if _cli_args.path:
|
||||
return tools.post_process(_cli_args)
|
||||
|
||||
if len(_cli_args.workloads) == 1 and 'all' in _cli_args.workloads:
|
||||
_cli_args.workloads = _workload_opts
|
||||
invalid_wkld = [wkld for wkld in _cli_args.workloads if wkld not in _workload_opts]
|
||||
invalid_wkld = [
|
||||
wkld for wkld in _cli_args.workloads if wkld not in _workload_opts]
|
||||
if invalid_wkld:
|
||||
_logger.error("Invalid workload(s) specified: {}".format(invalid_wkld))
|
||||
if 'all' in _cli_args.workloads:
|
||||
_logger.error("If you meant 'all' use: './browbeat.py all' or './browbeat.py'")
|
||||
_logger.error(
|
||||
"If you meant 'all' use: './browbeat.py all' or './browbeat.py'")
|
||||
exit(1)
|
||||
else:
|
||||
time_stamp = datetime.datetime.utcnow().strftime("%Y%m%d-%H%M%S")
|
||||
_logger.info("Browbeat test suite kicked off")
|
||||
_logger.info("Browbeat UUID: {}".format(browbeat_uuid))
|
||||
_logger.info("Browbeat UUID: {}".format(lib.Elastic.browbeat_uuid))
|
||||
if _config['elasticsearch']['enabled']:
|
||||
_logger.info("Checking for Metadata")
|
||||
metadata_exists = tools.check_metadata()
|
||||
@ -93,11 +112,12 @@ def main():
|
||||
" metadata files do not exist")
|
||||
_logger.info("Gathering Metadata")
|
||||
tools.gather_metadata()
|
||||
elif _config['elasticsearch']['regather'] :
|
||||
elif _config['elasticsearch']['regather']:
|
||||
_logger.info("Regathering Metadata")
|
||||
tools.gather_metadata()
|
||||
|
||||
_logger.info("Running workload(s): {}".format(','.join(_cli_args.workloads)))
|
||||
_logger.info("Running workload(s): {}".format(
|
||||
','.join(_cli_args.workloads)))
|
||||
for wkld_provider in _cli_args.workloads:
|
||||
if wkld_provider in _config:
|
||||
if _config[wkld_provider]['enabled']:
|
||||
@ -106,11 +126,12 @@ def main():
|
||||
_logger.warning("{} is not enabled in {}".format(wkld_provider,
|
||||
_cli_args.setup))
|
||||
else:
|
||||
_logger.error("{} is missing in {}".format(wkld_provider, _cli_args.setup))
|
||||
_logger.error("{} is missing in {}".format(
|
||||
wkld_provider, _cli_args.setup))
|
||||
result_dir = _config['browbeat']['results']
|
||||
lib.WorkloadBase.WorkloadBase.print_report(result_dir, time_stamp)
|
||||
_logger.info("Saved browbeat result summary to {}".format(
|
||||
os.path.join(result_dir,time_stamp + '.' + 'report')))
|
||||
os.path.join(result_dir, time_stamp + '.' + 'report')))
|
||||
lib.WorkloadBase.WorkloadBase.print_summary()
|
||||
|
||||
browbeat_rc = 0
|
||||
@ -120,15 +141,17 @@ def main():
|
||||
browbeat_rc = 2
|
||||
|
||||
if browbeat_rc == 1:
|
||||
_logger.info("Browbeat finished with test failures, UUID: {}".format(browbeat_uuid))
|
||||
sys.exit(browbeat_rc)
|
||||
_logger.info("Browbeat finished with test failures, UUID: {}".format(
|
||||
lib.Elastic.browbeat_uuid))
|
||||
sys.exit(browbeat_rc)
|
||||
elif browbeat_rc == 2:
|
||||
_logger.info("Browbeat finished with Elasticsearch indexing failures, UUID: {}"
|
||||
.format(browbeat_uuid))
|
||||
sys.exit(browbeat_rc)
|
||||
_logger.info("Browbeat finished with Elasticsearch indexing failures, UUID: {}"
|
||||
.format(lib.Elastic.browbeat_uuid))
|
||||
sys.exit(browbeat_rc)
|
||||
else:
|
||||
_logger.info("Browbeat finished successfully, UUID: {}".format(browbeat_uuid))
|
||||
sys.exit(0)
|
||||
_logger.info("Browbeat finished successfully, UUID: {}".format(
|
||||
lib.Elastic.browbeat_uuid))
|
||||
sys.exit(0)
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
|
||||
@ -38,7 +38,7 @@ Run performance stress tests through Browbeat
|
||||
(browbeat-venv)[stack@ospd browbeat]$ ./browbeat.py <workload> #perfkit, rally, shaker or "all"
|
||||
|
||||
Running PerfKitBenchmarker
|
||||
==========================
|
||||
---------------------------
|
||||
|
||||
Work is on-going to utilize PerfKitBenchmarker as a workload provider to
|
||||
Browbeat. Many benchmarks work out of the box with Browbeat. You must
|
||||
@ -69,7 +69,8 @@ browbeat-config.yaml:
|
||||
(browbeat-venv)[stack@ospd browbeat]$ ./browbeat.py perfkit -s browbeat-config.yaml
|
||||
|
||||
Running Shaker
|
||||
==============
|
||||
---------------
|
||||
|
||||
Running Shaker requires the shaker image to be built, which in turn requires
|
||||
instances to be able to access the internet. The playbooks for this installation
|
||||
have been described in the installation documentation but for the sake of
|
||||
@ -215,7 +216,7 @@ I would suggest bulk introspection for testing documented TripleO workflows and
|
||||
individual introspection to test the performance of introspection itself.
|
||||
|
||||
Interpreting Browbeat Results
|
||||
=============================
|
||||
------------------------------
|
||||
|
||||
By default results for each test will be placed in a timestamped folder `results/` inside your Browbeat folder.
|
||||
Each run folder will contain output files from the various workloads and benchmarks that ran during that Browbeat
|
||||
@ -232,7 +233,7 @@ and Kibana to view them more easily.
|
||||
|
||||
|
||||
Working with Multiple Clouds
|
||||
============================
|
||||
-----------------------------
|
||||
|
||||
If you are running playbooks from your local machine you can run against more
|
||||
than one cloud at the same time. To do this, you should create a directory
|
||||
@ -249,3 +250,32 @@ per-cloud and clone Browbeat into that specific directory:
|
||||
[browbeat@laptop ansible]$ ssh -F ssh-config overcloud-controller-0 # Takes you to first controller
|
||||
|
||||
Repeat the above steps for as many clouds as you have to run playbooks against your clouds.
|
||||
|
||||
Compare software-metadata from two different runs
|
||||
--------------------------------------------------
|
||||
|
||||
Browbeat's metadata is great to help build visuals in Kibana by querying on specific metadata fields, but sometimes
|
||||
we need to see what the difference between two builds might be. Kibana doesn't have a good way to show this, so we
|
||||
added an option to Browbeat CLI to query ElasticSearch.
|
||||
|
||||
To use :
|
||||
|
||||
::
|
||||
|
||||
$ python browbeat.py --compare software-metadata --uuid "browbeat-uuid-1" "browbeat-uuid-2"
|
||||
|
||||
Real world use-case, we had two builds in our CI that used the exact same DLRN hash, however the later build had a
|
||||
10x performance hit for two Neutron operations, router-create and add-interface-to-router. Given we had exactly the
|
||||
same DLRN hash, the only difference could be how things were configured. Using this new code, we could quickly identify
|
||||
the difference -- TripleO enabled l3_ha.
|
||||
|
||||
::
|
||||
|
||||
[rocketship:browbeat] jtaleric:browbeat$ python browbeat.py --compare software-metadata --uuid "3fc2f149-7091-4e16-855a-60738849af17" "6738eed7-c8dd-4747-abde-47c996975a57"
|
||||
2017-05-25 02:34:47,230 - browbeat.Tools - INFO - Validating the configuration file passed by the user
|
||||
2017-05-25 02:34:47,311 - browbeat.Tools - INFO - Validation successful
|
||||
2017-05-25 02:34:47,311 - browbeat.Elastic - INFO - Querying Elastic : index [_all] : role [controller] : uuid [3fc2f149-7091-4e16-855a-60738849af17]
|
||||
2017-05-25 02:34:55,684 - browbeat.Elastic - INFO - Querying Elastic : index [_all] : role [controller] : uuid [6738eed7-c8dd-4747-abde-47c996975a57]
|
||||
2017-05-25 02:35:01,165 - browbeat.Elastic - INFO - Difference found : Host [overcloud-controller-2] Service [neutron] l3_ha [False]
|
||||
2017-05-25 02:35:01,168 - browbeat.Elastic - INFO - Difference found : Host [overcloud-controller-1] Service [neutron] l3_ha [False]
|
||||
2017-05-25 02:35:01,172 - browbeat.Elastic - INFO - Difference found : Host [overcloud-controller-0] Service [neutron] l3_ha [False]
|
||||
|
||||
170
lib/Elastic.py
170
lib/Elastic.py
@ -18,6 +18,7 @@ import uuid
|
||||
import sys
|
||||
import time
|
||||
import os
|
||||
import re
|
||||
|
||||
browbeat_uuid = uuid.uuid4()
|
||||
|
||||
@ -34,7 +35,8 @@ class Elastic(object):
|
||||
)
|
||||
self.workload = workload
|
||||
today = datetime.datetime.today()
|
||||
self.index = "{}-{}-{}".format(tool, workload, today.strftime('%Y.%m.%d'))
|
||||
self.index = "{}-{}-{}".format(tool,
|
||||
workload, today.strftime('%Y.%m.%d'))
|
||||
|
||||
def load_json(self, result):
|
||||
json_data = None
|
||||
@ -63,15 +65,23 @@ class Elastic(object):
|
||||
result[_meta['name']] = json.load(jdata)
|
||||
except Exception:
|
||||
self.logger.error(
|
||||
"Error loading Metadata file : {}".format(_meta['file']))
|
||||
self.logger.error("Please make sure the metadata file exists and"
|
||||
" is valid JSON or run the playbook ansible/gather/site.yml"
|
||||
" before running the Browbeat test Suite")
|
||||
"Error loading Metadata file : {}".format(
|
||||
_meta['file']))
|
||||
self.logger.error(
|
||||
"Please make sure the metadata file exists and"
|
||||
" is valid JSON or run the playbook ansible/gather/site.yml"
|
||||
" before running the Browbeat test Suite")
|
||||
sys.exit(1)
|
||||
return result
|
||||
|
||||
def index_result(self, result, test_name, result_dir, identifier='', _type='result',
|
||||
_id=None):
|
||||
def index_result(
|
||||
self,
|
||||
result,
|
||||
test_name,
|
||||
result_dir,
|
||||
identifier='',
|
||||
_type='result',
|
||||
_id=None):
|
||||
retry = 2
|
||||
result['browbeat_uuid'] = str(browbeat_uuid)
|
||||
result['cloud_name'] = self.config['browbeat']['cloud_name']
|
||||
@ -88,18 +98,138 @@ class Elastic(object):
|
||||
format(self.index, result['browbeat_uuid']))
|
||||
return True
|
||||
except Exception as Err:
|
||||
self.logger.error("Error pushing data to Elasticsearch, going to retry"
|
||||
" in 10 seconds")
|
||||
self.logger.error(
|
||||
"Error pushing data to Elasticsearch, going to retry"
|
||||
" in 10 seconds")
|
||||
self.logger.error("Exception: {}".format(Err))
|
||||
time.sleep(10)
|
||||
if i == (retry-1):
|
||||
self.logger.error("Pushing Data to Elasticsearch failed in spite of retry,"
|
||||
" dumping JSON")
|
||||
elastic_file = os.path.join(result_dir,
|
||||
test_name + '-' + identifier + '-elastic' +
|
||||
'.' + 'json')
|
||||
with open(elastic_file, 'w') as result_file:
|
||||
json.dump(result, result_file, indent=4, sort_keys=True)
|
||||
self.logger.info("Saved Elasticsearch consumable result JSON to {}".
|
||||
format(elastic_file))
|
||||
return False
|
||||
if i == (retry - 1):
|
||||
self.logger.error(
|
||||
"Pushing Data to Elasticsearch failed in spite of retry,"
|
||||
" dumping JSON")
|
||||
elastic_file = os.path.join(
|
||||
result_dir, test_name + '-' + identifier + '-elastic' + '.' + 'json')
|
||||
with open(elastic_file, 'w') as result_file:
|
||||
json.dump(result, result_file,
|
||||
indent=4, sort_keys=True)
|
||||
self.logger.info(
|
||||
"Saved Elasticsearch consumable result JSON to {}".format(elastic_file))
|
||||
return False
|
||||
|
||||
def get_software_metadata(self, index, role, browbeat_uuid):
|
||||
nodes = {}
|
||||
results = self.query_uuid(index, browbeat_uuid)
|
||||
pattern = re.compile(".*{}.*".format(role))
|
||||
if results:
|
||||
for result in results:
|
||||
for metadata in result['_source']['software-metadata']:
|
||||
for service in metadata:
|
||||
if pattern.match(metadata[service]['node_name']):
|
||||
if metadata[service]['node_name'] not in nodes:
|
||||
nodes[metadata[service][
|
||||
'node_name']] = metadata
|
||||
return nodes
|
||||
else:
|
||||
self.logger.error("UUID {} wasn't found".format(browbeat_uuid))
|
||||
return False
|
||||
|
||||
"""
|
||||
Currently this function will only compare two uuids. I (rook) am not convinced it is worth
|
||||
the effort to engineer anything > 2.
|
||||
"""
|
||||
|
||||
def compare_metadata(self, index, role, uuids):
|
||||
meta = []
|
||||
for browbeat_uuid in uuids:
|
||||
self.logger.info(
|
||||
"Querying Elastic : index [{}] : role [{}] : browbeat_uuid [{}] ".format(
|
||||
index, role, browbeat_uuid))
|
||||
software_metadata = self.get_software_metadata(index, role, browbeat_uuid)
|
||||
if software_metadata:
|
||||
meta.append(software_metadata)
|
||||
else:
|
||||
return False
|
||||
ignore = [
|
||||
"connection",
|
||||
"admin_url",
|
||||
"bind_host",
|
||||
"rabbit_hosts",
|
||||
"auth_url",
|
||||
"public_bind_host",
|
||||
"host",
|
||||
"key",
|
||||
"url",
|
||||
"auth_uri",
|
||||
"coordination_url",
|
||||
"swift_authurl",
|
||||
"admin_token",
|
||||
"memcached_servers",
|
||||
"api_servers",
|
||||
"osapi_volume_listen",
|
||||
"nova_url",
|
||||
"coordination_url",
|
||||
"memcache_servers",
|
||||
"novncproxy_host",
|
||||
"backend_url",
|
||||
"novncproxy_base_url",
|
||||
"metadata_listen",
|
||||
"osapi_compute_listen",
|
||||
"admin_bind_host",
|
||||
"glance_api_servers",
|
||||
"iscsi_ip_address",
|
||||
"registry_host",
|
||||
"auth_address",
|
||||
"swift_key",
|
||||
"auth_encryption_key",
|
||||
"metadata_proxy_shared_secret",
|
||||
"telemetry_secret",
|
||||
"heat_metadata_server_url",
|
||||
"heat_waitcondition_server_url",
|
||||
"transport_url"]
|
||||
if len(meta) < 2:
|
||||
self.logger.error("Unable to compare data-sets")
|
||||
return False
|
||||
for host in meta[0]:
|
||||
if host not in meta[1]:
|
||||
self.logger.error("Deployment differs: "
|
||||
"Host [{}] missing ".format(host))
|
||||
continue
|
||||
for service in meta[0][host]:
|
||||
for options in meta[0][host][service].keys():
|
||||
if options not in meta[1][host][service]:
|
||||
self.logger.error(
|
||||
"Missing Option : "
|
||||
"Host [{}] Service [{}] {}".format(
|
||||
host, service, options))
|
||||
continue
|
||||
if isinstance(meta[0][host][service][options], dict):
|
||||
for key in meta[0][host][service][options].keys():
|
||||
if key not in ignore:
|
||||
if key in meta[1][host][service][options]:
|
||||
value = meta[0][host][
|
||||
service][options][key]
|
||||
new_value = meta[1][host][
|
||||
service][options][key]
|
||||
if value != new_value:
|
||||
self.logger.info(
|
||||
"Difference found : "
|
||||
"Host [{}] Service [{}] Section {} {} [{}]".format(
|
||||
host,
|
||||
service,
|
||||
options,
|
||||
key,
|
||||
meta[0][host][service][options][key]))
|
||||
else:
|
||||
self.logger.info(
|
||||
"Missing Value : "
|
||||
"Host [{}] Service [{}] {} [{}]".format(
|
||||
host, service, options, key))
|
||||
|
||||
def query_uuid(self, index, browbeat_uuid):
|
||||
body = {'query': {"match": {"browbeat_uuid": {
|
||||
"query": browbeat_uuid, "type": "phrase"}}}}
|
||||
results = self.es.search(index=index, doc_type='result', body=body)
|
||||
if len(results['hits']['hits']) > 0:
|
||||
return results['hits']['hits']
|
||||
else:
|
||||
return False
|
||||
|
||||
Loading…
Reference in New Issue
Block a user