From 21900bffad9bbd50da07c2343d0c2c4d688fbc57 Mon Sep 17 00:00:00 2001 From: Joe Talerico Date: Wed, 24 May 2017 14:12:41 -0400 Subject: [PATCH] Query Elastic to compare software-metadata Method to query ElasticSearch for a specific set of browbeat_uuids and compare the metadata to determine if there are differences. This work will also tell the user if a option or value is missing. Eventually, I would like to see us query Elastic for collectd data to see if there has been CPU/Memory/DiskIO increases during a specific Browbeat run -- this is a longer-term goal. Example of this : https://gist.github.com/jtaleric/ffc1508eba3cba9515ca24cfcf23583c Change-Id: Ie65e2c3d505aa2f19ba10109276ba982ee4ab67b --- browbeat.py | 69 ++++++++++++------ doc/source/usage.rst | 38 +++++++++- lib/Elastic.py | 170 ++++++++++++++++++++++++++++++++++++++----- 3 files changed, 230 insertions(+), 47 deletions(-) diff --git a/browbeat.py b/browbeat.py index f7735dea8..c1535cabd 100755 --- a/browbeat.py +++ b/browbeat.py @@ -11,24 +11,25 @@ # See the License for the specific language governing permissions and # limitations under the License. -from lib.Elastic import browbeat_uuid +import argparse +import datetime +import lib.Elastic import lib.PerfKit import lib.Rally import lib.Shaker import lib.Yoda import lib.WorkloadBase import lib.Tools -import argparse import logging +import os import sys import time -import datetime -import os _workload_opts = ['perfkit', 'rally', 'shaker', 'yoda'] _config_file = 'browbeat-config.yaml' debug_log_file = 'log/debug.log' + def main(): tools = lib.Tools.Tools() parser = argparse.ArgumentParser( @@ -41,14 +42,21 @@ def main(): help='Provide Browbeat YAML configuration file. Default is ./{}'.format(_config_file)) parser.add_argument('workloads', nargs='*', help='Browbeat workload(s). Takes a space separated' ' list of workloads ({}) or \"all\"'.format(', '.join(_workload_opts))) - parser.add_argument('--debug', action='store_true', help='Enable Debug messages') - parser.add_argument('-p','--postprocess', - dest="path",help="Path to process, ie results/20170101/") + parser.add_argument('--debug', action='store_true', + help='Enable Debug messages') + parser.add_argument('-p', '--postprocess', + dest="path", help="Path to process, ie results/20170101/") + parser.add_argument('-c', '--compare', + help="Compare metadata", dest="compare", + choices=['software-metadata']) + parser.add_argument('-u', '--uuid', + help="UUIDs to pass", dest="uuids", nargs=2) _cli_args = parser.parse_args() _logger = logging.getLogger('browbeat') _logger.setLevel(logging.DEBUG) - _formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)7s - %(message)s') + _formatter = logging.Formatter( + '%(asctime)s - %(name)s - %(levelname)7s - %(message)s') _formatter.converter = time.gmtime _dbg_file = logging.FileHandler(debug_log_file) _dbg_file.setLevel(logging.DEBUG) @@ -67,24 +75,35 @@ def main(): # Load Browbeat yaml config file: _config = tools._load_config(_cli_args.setup) + if _cli_args.compare == "software-metadata": + es = lib.Elastic.Elastic(_config, "BrowbeatCLI") + es.compare_metadata("_all", 'controller', _cli_args.uuids) + exit(0) + + if _cli_args.compare: + parser.print_help() + exit(1) + # Default to all workloads if _cli_args.workloads == []: _cli_args.workloads.append('all') - if _cli_args.path : + if _cli_args.path: return tools.post_process(_cli_args) if len(_cli_args.workloads) == 1 and 'all' in _cli_args.workloads: _cli_args.workloads = _workload_opts - invalid_wkld = [wkld for wkld in _cli_args.workloads if wkld not in _workload_opts] + invalid_wkld = [ + wkld for wkld in _cli_args.workloads if wkld not in _workload_opts] if invalid_wkld: _logger.error("Invalid workload(s) specified: {}".format(invalid_wkld)) if 'all' in _cli_args.workloads: - _logger.error("If you meant 'all' use: './browbeat.py all' or './browbeat.py'") + _logger.error( + "If you meant 'all' use: './browbeat.py all' or './browbeat.py'") exit(1) else: time_stamp = datetime.datetime.utcnow().strftime("%Y%m%d-%H%M%S") _logger.info("Browbeat test suite kicked off") - _logger.info("Browbeat UUID: {}".format(browbeat_uuid)) + _logger.info("Browbeat UUID: {}".format(lib.Elastic.browbeat_uuid)) if _config['elasticsearch']['enabled']: _logger.info("Checking for Metadata") metadata_exists = tools.check_metadata() @@ -93,11 +112,12 @@ def main(): " metadata files do not exist") _logger.info("Gathering Metadata") tools.gather_metadata() - elif _config['elasticsearch']['regather'] : + elif _config['elasticsearch']['regather']: _logger.info("Regathering Metadata") tools.gather_metadata() - _logger.info("Running workload(s): {}".format(','.join(_cli_args.workloads))) + _logger.info("Running workload(s): {}".format( + ','.join(_cli_args.workloads))) for wkld_provider in _cli_args.workloads: if wkld_provider in _config: if _config[wkld_provider]['enabled']: @@ -106,11 +126,12 @@ def main(): _logger.warning("{} is not enabled in {}".format(wkld_provider, _cli_args.setup)) else: - _logger.error("{} is missing in {}".format(wkld_provider, _cli_args.setup)) + _logger.error("{} is missing in {}".format( + wkld_provider, _cli_args.setup)) result_dir = _config['browbeat']['results'] lib.WorkloadBase.WorkloadBase.print_report(result_dir, time_stamp) _logger.info("Saved browbeat result summary to {}".format( - os.path.join(result_dir,time_stamp + '.' + 'report'))) + os.path.join(result_dir, time_stamp + '.' + 'report'))) lib.WorkloadBase.WorkloadBase.print_summary() browbeat_rc = 0 @@ -120,15 +141,17 @@ def main(): browbeat_rc = 2 if browbeat_rc == 1: - _logger.info("Browbeat finished with test failures, UUID: {}".format(browbeat_uuid)) - sys.exit(browbeat_rc) + _logger.info("Browbeat finished with test failures, UUID: {}".format( + lib.Elastic.browbeat_uuid)) + sys.exit(browbeat_rc) elif browbeat_rc == 2: - _logger.info("Browbeat finished with Elasticsearch indexing failures, UUID: {}" - .format(browbeat_uuid)) - sys.exit(browbeat_rc) + _logger.info("Browbeat finished with Elasticsearch indexing failures, UUID: {}" + .format(lib.Elastic.browbeat_uuid)) + sys.exit(browbeat_rc) else: - _logger.info("Browbeat finished successfully, UUID: {}".format(browbeat_uuid)) - sys.exit(0) + _logger.info("Browbeat finished successfully, UUID: {}".format( + lib.Elastic.browbeat_uuid)) + sys.exit(0) if __name__ == '__main__': sys.exit(main()) diff --git a/doc/source/usage.rst b/doc/source/usage.rst index b55973677..8748b8bf5 100644 --- a/doc/source/usage.rst +++ b/doc/source/usage.rst @@ -38,7 +38,7 @@ Run performance stress tests through Browbeat (browbeat-venv)[stack@ospd browbeat]$ ./browbeat.py #perfkit, rally, shaker or "all" Running PerfKitBenchmarker -========================== +--------------------------- Work is on-going to utilize PerfKitBenchmarker as a workload provider to Browbeat. Many benchmarks work out of the box with Browbeat. You must @@ -69,7 +69,8 @@ browbeat-config.yaml: (browbeat-venv)[stack@ospd browbeat]$ ./browbeat.py perfkit -s browbeat-config.yaml Running Shaker -============== +--------------- + Running Shaker requires the shaker image to be built, which in turn requires instances to be able to access the internet. The playbooks for this installation have been described in the installation documentation but for the sake of @@ -215,7 +216,7 @@ I would suggest bulk introspection for testing documented TripleO workflows and individual introspection to test the performance of introspection itself. Interpreting Browbeat Results -============================= +------------------------------ By default results for each test will be placed in a timestamped folder `results/` inside your Browbeat folder. Each run folder will contain output files from the various workloads and benchmarks that ran during that Browbeat @@ -232,7 +233,7 @@ and Kibana to view them more easily. Working with Multiple Clouds -============================ +----------------------------- If you are running playbooks from your local machine you can run against more than one cloud at the same time. To do this, you should create a directory @@ -249,3 +250,32 @@ per-cloud and clone Browbeat into that specific directory: [browbeat@laptop ansible]$ ssh -F ssh-config overcloud-controller-0 # Takes you to first controller Repeat the above steps for as many clouds as you have to run playbooks against your clouds. + +Compare software-metadata from two different runs +-------------------------------------------------- + +Browbeat's metadata is great to help build visuals in Kibana by querying on specific metadata fields, but sometimes +we need to see what the difference between two builds might be. Kibana doesn't have a good way to show this, so we +added an option to Browbeat CLI to query ElasticSearch. + +To use : + +:: + + $ python browbeat.py --compare software-metadata --uuid "browbeat-uuid-1" "browbeat-uuid-2" + +Real world use-case, we had two builds in our CI that used the exact same DLRN hash, however the later build had a +10x performance hit for two Neutron operations, router-create and add-interface-to-router. Given we had exactly the +same DLRN hash, the only difference could be how things were configured. Using this new code, we could quickly identify +the difference -- TripleO enabled l3_ha. + +:: + + [rocketship:browbeat] jtaleric:browbeat$ python browbeat.py --compare software-metadata --uuid "3fc2f149-7091-4e16-855a-60738849af17" "6738eed7-c8dd-4747-abde-47c996975a57" + 2017-05-25 02:34:47,230 - browbeat.Tools - INFO - Validating the configuration file passed by the user + 2017-05-25 02:34:47,311 - browbeat.Tools - INFO - Validation successful + 2017-05-25 02:34:47,311 - browbeat.Elastic - INFO - Querying Elastic : index [_all] : role [controller] : uuid [3fc2f149-7091-4e16-855a-60738849af17] + 2017-05-25 02:34:55,684 - browbeat.Elastic - INFO - Querying Elastic : index [_all] : role [controller] : uuid [6738eed7-c8dd-4747-abde-47c996975a57] + 2017-05-25 02:35:01,165 - browbeat.Elastic - INFO - Difference found : Host [overcloud-controller-2] Service [neutron] l3_ha [False] + 2017-05-25 02:35:01,168 - browbeat.Elastic - INFO - Difference found : Host [overcloud-controller-1] Service [neutron] l3_ha [False] + 2017-05-25 02:35:01,172 - browbeat.Elastic - INFO - Difference found : Host [overcloud-controller-0] Service [neutron] l3_ha [False] diff --git a/lib/Elastic.py b/lib/Elastic.py index d8c1fdd85..bfd9141d8 100644 --- a/lib/Elastic.py +++ b/lib/Elastic.py @@ -18,6 +18,7 @@ import uuid import sys import time import os +import re browbeat_uuid = uuid.uuid4() @@ -34,7 +35,8 @@ class Elastic(object): ) self.workload = workload today = datetime.datetime.today() - self.index = "{}-{}-{}".format(tool, workload, today.strftime('%Y.%m.%d')) + self.index = "{}-{}-{}".format(tool, + workload, today.strftime('%Y.%m.%d')) def load_json(self, result): json_data = None @@ -63,15 +65,23 @@ class Elastic(object): result[_meta['name']] = json.load(jdata) except Exception: self.logger.error( - "Error loading Metadata file : {}".format(_meta['file'])) - self.logger.error("Please make sure the metadata file exists and" - " is valid JSON or run the playbook ansible/gather/site.yml" - " before running the Browbeat test Suite") + "Error loading Metadata file : {}".format( + _meta['file'])) + self.logger.error( + "Please make sure the metadata file exists and" + " is valid JSON or run the playbook ansible/gather/site.yml" + " before running the Browbeat test Suite") sys.exit(1) return result - def index_result(self, result, test_name, result_dir, identifier='', _type='result', - _id=None): + def index_result( + self, + result, + test_name, + result_dir, + identifier='', + _type='result', + _id=None): retry = 2 result['browbeat_uuid'] = str(browbeat_uuid) result['cloud_name'] = self.config['browbeat']['cloud_name'] @@ -88,18 +98,138 @@ class Elastic(object): format(self.index, result['browbeat_uuid'])) return True except Exception as Err: - self.logger.error("Error pushing data to Elasticsearch, going to retry" - " in 10 seconds") + self.logger.error( + "Error pushing data to Elasticsearch, going to retry" + " in 10 seconds") self.logger.error("Exception: {}".format(Err)) time.sleep(10) - if i == (retry-1): - self.logger.error("Pushing Data to Elasticsearch failed in spite of retry," - " dumping JSON") - elastic_file = os.path.join(result_dir, - test_name + '-' + identifier + '-elastic' + - '.' + 'json') - with open(elastic_file, 'w') as result_file: - json.dump(result, result_file, indent=4, sort_keys=True) - self.logger.info("Saved Elasticsearch consumable result JSON to {}". - format(elastic_file)) - return False + if i == (retry - 1): + self.logger.error( + "Pushing Data to Elasticsearch failed in spite of retry," + " dumping JSON") + elastic_file = os.path.join( + result_dir, test_name + '-' + identifier + '-elastic' + '.' + 'json') + with open(elastic_file, 'w') as result_file: + json.dump(result, result_file, + indent=4, sort_keys=True) + self.logger.info( + "Saved Elasticsearch consumable result JSON to {}".format(elastic_file)) + return False + + def get_software_metadata(self, index, role, browbeat_uuid): + nodes = {} + results = self.query_uuid(index, browbeat_uuid) + pattern = re.compile(".*{}.*".format(role)) + if results: + for result in results: + for metadata in result['_source']['software-metadata']: + for service in metadata: + if pattern.match(metadata[service]['node_name']): + if metadata[service]['node_name'] not in nodes: + nodes[metadata[service][ + 'node_name']] = metadata + return nodes + else: + self.logger.error("UUID {} wasn't found".format(browbeat_uuid)) + return False + + """ + Currently this function will only compare two uuids. I (rook) am not convinced it is worth + the effort to engineer anything > 2. + """ + + def compare_metadata(self, index, role, uuids): + meta = [] + for browbeat_uuid in uuids: + self.logger.info( + "Querying Elastic : index [{}] : role [{}] : browbeat_uuid [{}] ".format( + index, role, browbeat_uuid)) + software_metadata = self.get_software_metadata(index, role, browbeat_uuid) + if software_metadata: + meta.append(software_metadata) + else: + return False + ignore = [ + "connection", + "admin_url", + "bind_host", + "rabbit_hosts", + "auth_url", + "public_bind_host", + "host", + "key", + "url", + "auth_uri", + "coordination_url", + "swift_authurl", + "admin_token", + "memcached_servers", + "api_servers", + "osapi_volume_listen", + "nova_url", + "coordination_url", + "memcache_servers", + "novncproxy_host", + "backend_url", + "novncproxy_base_url", + "metadata_listen", + "osapi_compute_listen", + "admin_bind_host", + "glance_api_servers", + "iscsi_ip_address", + "registry_host", + "auth_address", + "swift_key", + "auth_encryption_key", + "metadata_proxy_shared_secret", + "telemetry_secret", + "heat_metadata_server_url", + "heat_waitcondition_server_url", + "transport_url"] + if len(meta) < 2: + self.logger.error("Unable to compare data-sets") + return False + for host in meta[0]: + if host not in meta[1]: + self.logger.error("Deployment differs: " + "Host [{}] missing ".format(host)) + continue + for service in meta[0][host]: + for options in meta[0][host][service].keys(): + if options not in meta[1][host][service]: + self.logger.error( + "Missing Option : " + "Host [{}] Service [{}] {}".format( + host, service, options)) + continue + if isinstance(meta[0][host][service][options], dict): + for key in meta[0][host][service][options].keys(): + if key not in ignore: + if key in meta[1][host][service][options]: + value = meta[0][host][ + service][options][key] + new_value = meta[1][host][ + service][options][key] + if value != new_value: + self.logger.info( + "Difference found : " + "Host [{}] Service [{}] Section {} {} [{}]".format( + host, + service, + options, + key, + meta[0][host][service][options][key])) + else: + self.logger.info( + "Missing Value : " + "Host [{}] Service [{}] {} [{}]".format( + host, service, options, key)) + + def query_uuid(self, index, browbeat_uuid): + body = {'query': {"match": {"browbeat_uuid": { + "query": browbeat_uuid, "type": "phrase"}}}} + results = self.es.search(index=index, doc_type='result', body=body) + if len(results['hits']['hits']) > 0: + return results['hits']['hits'] + else: + return False