From 0b5609727216b79ec6edebe43e6d1c473c36f41e Mon Sep 17 00:00:00 2001 From: agopi Date: Fri, 21 Sep 2018 08:56:03 -0400 Subject: [PATCH] Adding stockpile to collect data This would facilitate browbeat users to take advantage of work being done wrt browbench utilities. To use stockpile just update metadata_playbook to ansible/gather/stockpile_gather.yml Change-Id: I4c12920007f66bc3378439b437676e4cb162b082 --- ansible/gather/stockpile.yml | 16 ++ ansible/generate_tripleo_hostfile.sh | 10 +- ansible/install/browbeat.yml | 1 + .../install/roles/stockpile/tasks/main.yml | 8 + ansible/oooq/install-browbeat.yml | 1 + .../templates/browbeat-basic.yaml.j2 | 2 +- .../templates/browbeat-minimal-ci.yaml.j2 | 2 +- browbeat-complete.yaml | 1 + browbeat-config.yaml | 1 + browbeat/prescribe.py | 174 ++++++++++++++++++ browbeat/tools.py | 1 - doc/source/contributing.rst | 7 + 12 files changed, 219 insertions(+), 5 deletions(-) create mode 100644 ansible/gather/stockpile.yml create mode 100644 ansible/install/roles/stockpile/tasks/main.yml create mode 100644 browbeat/prescribe.py diff --git a/ansible/gather/stockpile.yml b/ansible/gather/stockpile.yml new file mode 100644 index 000000000..705a6c4cf --- /dev/null +++ b/ansible/gather/stockpile.yml @@ -0,0 +1,16 @@ +--- + +- hosts: stockpile + tasks: + - name: setting fact for metadata folder + set_fact: + md_output_path: "{{ browbeat_path }}/metadata/machine_facts.json" + +- import_playbook: stockpile/stockpile.yml + vars: + stockpile_output_path: "{{ md_output_path }}" + +- hosts: stockpile + tasks: + - name: run prescribe + command: python {{ browbeat_path }}/browbeat/prescribe.py {{ browbeat_path }}/metadata diff --git a/ansible/generate_tripleo_hostfile.sh b/ansible/generate_tripleo_hostfile.sh index cce14b6a0..1d3a69876 100755 --- a/ansible/generate_tripleo_hostfile.sh +++ b/ansible/generate_tripleo_hostfile.sh @@ -304,11 +304,11 @@ if [ "${uncomment_localhost}" = true ]; then echo "#undercloud" | tee -a ${ansible_inventory_file} else echo "#localhost" | tee -a ${ansible_inventory_file} - echo "undercloud" | tee -a ${ansible_inventory_file} + echo "undercloud ansible_user=${user}" | tee -a ${ansible_inventory_file} fi echo "" | tee -a ${ansible_inventory_file} echo "[undercloud]" | tee -a ${ansible_inventory_file} -echo "undercloud" | tee -a ${ansible_inventory_file} +echo "undercloud ansible_user=${user}" | tee -a ${ansible_inventory_file} echo "" | tee -a ${ansible_inventory_file} echo "[controller]" | tee -a ${ansible_inventory_file} if [[ ${#controller_hn} -gt 0 ]]; then @@ -426,6 +426,9 @@ if [[ ${#controller_hn} -gt 0 ]] || [[ ${#blockstorage_hn} -gt 0 ]] || [[ ${#obj if [[ ${#compute_hn} -gt 0 ]]; then echo "compute" | tee -a ${ansible_inventory_file} fi + echo "" | tee -a ${ansible_inventory_file} + echo "[overcloud:vars]" | tee -a ${ansible_inventory_file} + echo "ansible_user=heat-admin" | tee -a ${ansible_inventory_file} fi echo "" | tee -a ${ansible_inventory_file} echo "[other]" | tee -a ${ansible_inventory_file} @@ -456,6 +459,9 @@ echo "" | tee -a ${ansible_inventory_file} echo "[elk-client]" | tee -a ${ansible_inventory_file} echo "## example host entry." | tee -a ${ansible_inventory_file} echo "#host-02" | tee -a ${ansible_inventory_file} +echo "" | tee -a ${ansible_inventory_file} +echo "[stockpile]" | tee -a ${ansible_inventory_file} +echo "undercloud ansible_user=${user}" | tee -a ${ansible_inventory_file} echo "---------------------------" echo "IMPORTANT: If you plan on deploying ELK and ELK clients, update hosts and make sure" diff --git a/ansible/install/browbeat.yml b/ansible/install/browbeat.yml index 6c0046d7a..db3e42026 100644 --- a/ansible/install/browbeat.yml +++ b/ansible/install/browbeat.yml @@ -7,6 +7,7 @@ remote_user: "{{ browbeat_user }}" roles: - common + - stockpile - browbeat - { role: browbeat-results, when: browbeat_results_in_httpd} - firewall diff --git a/ansible/install/roles/stockpile/tasks/main.yml b/ansible/install/roles/stockpile/tasks/main.yml new file mode 100644 index 000000000..edc17ae8d --- /dev/null +++ b/ansible/install/roles/stockpile/tasks/main.yml @@ -0,0 +1,8 @@ +--- + +- name: Clone stockpile + git: + repo: 'http://github.com/redhat-performance/stockpile.git' + dest: "{{ browbeat_path }}/ansible/gather/stockpile" + version: master + force: yes diff --git a/ansible/oooq/install-browbeat.yml b/ansible/oooq/install-browbeat.yml index 6a6f4eeb2..1e61b8ce0 100644 --- a/ansible/oooq/install-browbeat.yml +++ b/ansible/oooq/install-browbeat.yml @@ -6,6 +6,7 @@ statsd_host: "{{ graphite_host }}" roles: - browbeat/common + - browbeat/stockpile - browbeat/browbeat - browbeat/firewall - browbeat/perfkitbenchmarker diff --git a/ansible/oooq/roles/template-configs/templates/browbeat-basic.yaml.j2 b/ansible/oooq/roles/template-configs/templates/browbeat-basic.yaml.j2 index 6c76f53a4..165cad49c 100644 --- a/ansible/oooq/roles/template-configs/templates/browbeat-basic.yaml.j2 +++ b/ansible/oooq/roles/template-configs/templates/browbeat-basic.yaml.j2 @@ -6,7 +6,7 @@ browbeat: rerun_type: iteration ansible: hosts: ansible/hosts - metadata_playbook: ansible/gather/site.yml + metadata_playbook: ansible/gather/stockpile.yml ssh_config: ansible/ssh-config elasticsearch: enabled: {{ elastic_enabled }} diff --git a/ansible/oooq/roles/template-configs/templates/browbeat-minimal-ci.yaml.j2 b/ansible/oooq/roles/template-configs/templates/browbeat-minimal-ci.yaml.j2 index 2fb40579d..262fe0c7e 100644 --- a/ansible/oooq/roles/template-configs/templates/browbeat-minimal-ci.yaml.j2 +++ b/ansible/oooq/roles/template-configs/templates/browbeat-minimal-ci.yaml.j2 @@ -6,7 +6,7 @@ browbeat: rerun_type: iteration ansible: hosts: ansible/hosts - metadata_playbook: ansible/gather/site.yml + metadata_playbook: ansible/gather/stockpile.yml ssh_config: ansible/ssh-config elasticsearch: enabled: {{ elastic_enabled }} diff --git a/browbeat-complete.yaml b/browbeat-complete.yaml index c566338d4..6beb2cba0 100644 --- a/browbeat-complete.yaml +++ b/browbeat-complete.yaml @@ -5,6 +5,7 @@ browbeat: rerun_type: iteration ansible: hosts: ansible/hosts + # can use ansible/gather/stockpile.yml to stockpile's roles metadata_playbook: ansible/gather/site.yml ssh_config: ansible/ssh-config elasticsearch: diff --git a/browbeat-config.yaml b/browbeat-config.yaml index 575efdfb7..70c189920 100644 --- a/browbeat-config.yaml +++ b/browbeat-config.yaml @@ -9,6 +9,7 @@ browbeat: rerun_type: iteration ansible: hosts: ansible/hosts + # can use ansible/gather/stockpile.yml to stockpile's roles metadata_playbook: ansible/gather/site.yml ssh_config: ansible/ssh-config elasticsearch: diff --git a/browbeat/prescribe.py b/browbeat/prescribe.py new file mode 100644 index 000000000..eeec6e4e4 --- /dev/null +++ b/browbeat/prescribe.py @@ -0,0 +1,174 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import json +import os +import sys + + +class Metadata(object): + + def __init__(self): + pass + + def load_file(self, filename): + try: + with open(filename) as f: + system_data = json.load(f) + except IOError: + print("machine_facts.json is missing") + exit(1) + return system_data + + def get_hardware_metadata(self, sys_data): + hard_dict = {} + for item, dictionary in sys_data.iteritems(): + if 'hardware_details' not in hard_dict: + hard_dict['hardware_details'] = [] + hardware_dict = {} + hardware_dict['label'] = sys_data[item]['inventory_hostname'] + hardware_dict['virtualization_role'] = sys_data[item]['ansible_virtualization_role'] + hardware_dict['virtualization_type'] = sys_data[item]['ansible_virtualization_type'] + hardware_dict['total_mem'] = sys_data[item][ + 'ansible_memory_mb']['real']['total'] + hardware_dict['total_logical_cores'] = sys_data[item][ + 'facter_processorcount'] + hardware_dict['os_name'] = sys_data[item]['ansible_distribution'] + \ + sys_data[item]['ansible_distribution_version'] + hardware_dict['ip'] = sys_data[item]['ansible_default_ipv4']['address'] + hardware_dict['num_interface'] = len(sys_data[item]['ansible_interfaces']) + hardware_dict['machine_make'] = sys_data[item]['ansible_product_name'] + hardware_dict['processor_type'] = ' '.join(sys_data[item]['facter_processor0'].split()) + hard_dict['hardware_details'].append(hardware_dict) + return hard_dict + + def get_environment_metadata(self, sys_data): + env_dict = {} + for item, dictionary in sys_data.iteritems(): + if 'environment_setup' not in env_dict: + env_dict['environment_setup'] = {} + for key, value in sys_data[item].iteritems(): + if 'stockpile_osp_env' in key: + for nodes, number in value.iteritems(): + env_dict['environment_setup'][nodes] = number + return env_dict + + def get_software_metadata(self, sys_data): + soft_all_dict = [] + bad_output_list = [{},[],""] + for item, dictionary in sys_data.iteritems(): + nodes = ['controller', 'undercloud', 'compute'] + if any(node in sys_data[item]['group_names'] for node in nodes): + software_dict = {} + sample_vuln_dict = {} + node = sys_data[item]['inventory_hostname'] + for key, output in sys_data[item].iteritems(): + if 'stockpile_yum' in key and output not in bad_output_list: + software_dict['repos_enabled'] = {} + software_dict['repos_enabled']['repos'] = [] + for repo in output: + if repo['state'] in 'enabled': + software_dict['repos_enabled']['repos'].append(repo['repoid']) + software_dict['repos_enabled']['node_name'] = node + if 'stockpile_cpu_vuln' in key and output not in bad_output_list: + if 'vulnerability' not in sample_vuln_dict.keys(): + sample_vuln_dict['vulnerability'] = {} + for vuln in output: + vuln = vuln.split('/sys/devices/system/cpu/vulnerabilities/') + vuln = vuln[1].split(':', 1) + sample_vuln_dict['vulnerability'][vuln[0]] = {} + if 'Mitigation: ' in vuln[1]: + sample_vuln_dict['vulnerability'][vuln[0]]['mitigation'] = \ + vuln[1].split('Mitigation: ')[1] + if 'Vulnerable: ' in vuln[1]: + vuln_value = vuln[1].split('Vulnerable: ')[1] + if vuln_value != "": + sample_vuln_dict['vulnerability'][vuln[0]]['type'] = \ + vuln_value + if 'stockpile_openstack_' in key: + service = key.split('stockpile_openstack_')[1] + # Either it's a dict or a value + if isinstance(output, dict): + if '_' in service: + # data from outside config files + service = service.split('_', 1) + service_name = service[0] + key_name = service[1] + if service_name not in software_dict.keys(): + software_dict[service_name] = {} + if key_name not in software_dict[service_name].keys(): + software_dict[service_name][key_name] = {} + for obj, value in output.iteritems(): + software_dict[service_name][key_name][obj] = value + else: + for obj, value in output.iteritems(): + if obj not in software_dict.keys(): + software_dict[obj] = value + software_dict[obj]['node_name'] = node + else: + software_dict[obj].update(value) + else: + if '_' in service: + service = service.split('_') + if len(service) > 1: + service_name = service[0] + key_name = service[1] + if service_name not in software_dict.keys(): + software_dict[service_name] = {} + if "DEFAULT" not in software_dict[service_name].keys(): + software_dict[service_name]["DEFAULT"] = {} + software_dict[service_name]["DEFAULT"][key_name] = output + if 'node_name' not in software_dict[service_name]: + software_dict[service_name]['node_name'] = node + if 'kernel' not in software_dict.keys(): + software_dict['kernel'] = {} + software_dict['kernel']['version'] = sys_data[item]['ansible_kernel'] + software_dict['kernel']['architecture'] = sys_data[item]['ansible_architecture'] + software_dict['kernel']['cpu_vulnerabilities'] = sample_vuln_dict['vulnerability'] + software_dict['kernel']['node_name'] = node + soft_all_dict.append(software_dict) + return soft_all_dict + + def write_metadata_file(self, data, filename): + with open(filename, 'w') as json_file: + json.dump(data, json_file, indent=4, sort_keys=True) + + +def main(): + parser = argparse.ArgumentParser(description='Metadata Generation') + parser.add_argument(dest='path', help='Location of machine-facts') + args = parser.parse_args() + _filename = os.path.join(args.path, 'machine_facts.json') + metadata = Metadata() + sysdata = metadata.load_file(_filename) + env_data = metadata.get_environment_metadata(sysdata) + + metadata.write_metadata_file( + env_data, os.path.join(args.path, 'environment-metadata.json')) + hardware_data = metadata.get_hardware_metadata(sysdata) + metadata.write_metadata_file( + hardware_data, os.path.join(args.path, 'hardware-metadata.json')) + software_data = metadata.get_software_metadata(sysdata) + # Just a simple check to ensure that stockpile was able to collect osp data + check = False + for node in software_data: + if "nova" in node.keys(): + check = True + if not check: + return "Issue with machine_facts.json, no nova data found." + metadata.write_metadata_file( + software_data, os.path.join(args.path, 'software-metadata.json')) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/browbeat/tools.py b/browbeat/tools.py index 1f9e54c22..8c677ce27 100644 --- a/browbeat/tools.py +++ b/browbeat/tools.py @@ -114,7 +114,6 @@ class Tools(object): def gather_metadata(self): os.putenv("ANSIBLE_SSH_ARGS", " -F {}".format(self.config['ansible']['ssh_config'])) - ansible_cmd = \ 'ansible-playbook -i {} {}' \ .format(self.config['ansible']['hosts'], self.config['ansible']['metadata_playbook']) diff --git a/doc/source/contributing.rst b/doc/source/contributing.rst index 0b4441b82..828413a6e 100644 --- a/doc/source/contributing.rst +++ b/doc/source/contributing.rst @@ -142,3 +142,10 @@ If you are adding new functionality to Browbeat please add testing for that func $ ci-scripts/install-and-check.sh See the README.rst in the ci-scripts folder for more details on the structure of the script and how to add additional tests. + +Contributing to stockpile +------------------------- + +We currently use `stockpile `_ +to gather config. Please follow `instructions `_ +to contribute to stockpile.