diff --git a/ansible/gather/roles/ceilometer/tasks/main.yml b/ansible/gather/roles/ceilometer/tasks/main.yml index df5031fc3..fcbd454ba 100644 --- a/ansible/gather/roles/ceilometer/tasks/main.yml +++ b/ansible/gather/roles/ceilometer/tasks/main.yml @@ -9,7 +9,7 @@ - name: Parse Ceilometer config become: true - shell: python /tmp/openstack-config-parser.py ceilometer /etc/ceilometer/ceilometer.conf /tmp/out.yml + shell: python /tmp/openstack-config-parser.py ceilometer /tmp/out.yml when: ceilometer_config.stat.exists - name: Fetch output diff --git a/ansible/gather/roles/cinder/tasks/main.yml b/ansible/gather/roles/cinder/tasks/main.yml index b82a507a3..47de30617 100644 --- a/ansible/gather/roles/cinder/tasks/main.yml +++ b/ansible/gather/roles/cinder/tasks/main.yml @@ -9,7 +9,7 @@ - name: Parse Cinder config become: true - shell: python /tmp/openstack-config-parser.py cinder /etc/cinder/cinder.conf /tmp/out.yml + shell: python /tmp/openstack-config-parser.py cinder /tmp/out.yml when: cinder_config.stat.exists - name: Fetch output diff --git a/ansible/gather/roles/common/files/openstack-config-parser.py b/ansible/gather/roles/common/files/openstack-config-parser.py index c56298f87..6249fa7d0 100644 --- a/ansible/gather/roles/common/files/openstack-config-parser.py +++ b/ansible/gather/roles/common/files/openstack-config-parser.py @@ -12,31 +12,44 @@ # limitations under the License. import sys -# usage: openstack-config-parser.py [service] [config file] [output file] +import os +import subprocess +# usage: openstack-config-parser.py [service] [output file] +def run_cmd(cmd): + process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + stdout, stderr = process.communicate() + output_dict = {} + output_dict['stdout'] = stdout.strip() + output_dict['stderr'] = stderr.strip() + output_dict['rc'] = process.returncode + return output_dict + +def strip_chars(line): + forbidden_chars = ['#', '\n', '"', '\\', ' ', '<', '>'] + for char in forbidden_chars: + line = line.replace(char, '') + return line def parse_config(serviceName, fileName): - # a dict containing key/value - # pairs, last value is what is - # stored. + # a dict containing key/value pairs, last value is what is stored. values = {} with open(fileName) as config: section = None for line in config: - pair = line.replace('#', '') - pair = pair.replace('\n', '') - pair = pair.replace('"', '') - pair = pair.replace('\\', '') - pair = pair.replace(' ', '') - pair = pair.replace('<', '') - pair = pair.replace('>', '') + pair = strip_chars(line) pair = pair.split('=') # excludes any line without a key/val pair - valid_line = not line.startswith( - "# ") and '[' not in line and line != '\n' and line != '#\n' and "password" not in line.lower() + valid_line = not line.startswith("# ") and \ + '[' not in line and line != '\n' \ + and line != '#\n' and "password" \ + not in line.lower() if line.startswith('['): section = line.replace('[','').replace(']','').replace('\n','') - if '#' not in line and valid_line and not section == None: + if '#' not in line and valid_line and not section == None and len(pair) == 2: + pair[0] = strip_chars(pair[0]) + pair[1] = strip_chars(pair[1]) values["openstack_S_" + serviceName + "_S_" + section + "_S_" + pair[0]] = pair[1] return values @@ -45,26 +58,97 @@ def try_type(val): try: int(val) return val - except ValueError: + except (ValueError, TypeError): try: float(val) return val - except ValueError: - if val.lower() in ("true", "false"): + except (ValueError, TypeError): + if type(val) is list: + return "\"" + str(val) + "\"" + elif val.lower() in ("true", "false"): return val else: return "\"" + val + "\"" +def add_conf_location(serviceName, fileName, values): + # Stores the exact location we gathered this config from. + index = "openstack_S_" + serviceName + "_S_" + "Browbeat" + "_S_" + "gather_conf_path" + if index in values: + values[index].append(fileName) + else: + values[index] = [fileName] + def print_vars_file(values, fileName): with open(fileName, 'w') as output: for key in values: output.write(key + ": " + try_type(values[key]) + "\n") +def is_containerized(service_name): + out = run_cmd("docker ps") + if service_name in out['stdout']: + return True + else: + return False + +def get_configs_list(path, extension='.conf'): + configs = [] + for item in os.listdir(path): + if item.endswith(extension): + configs.extend([item]) + return configs + +def get_neutron_plugin(output, cfg_path): + plugin = output['openstack_S_neutron_S_DEFAULT_S_core_plugin'] + plugin_path = "{}/plugins/{}/".format(cfg_path, plugin) + for item in get_configs_list(plugin_path, extension='.ini'): + full_path = "{}/{}".format(plugin_path, + item) + output.update(parse_config("neutron-plugin", full_path)) + add_conf_location("neutron", full_path, output) + return output def main(): - output = parse_config(sys.argv[1], sys.argv[2]) - print_vars_file(output, sys.argv[3]) + if len(sys.argv) < 3: + print("usage: openstack-config-parser.py [service] [output file]") + exit(1) + + service_name = sys.argv[1] + outfile = sys.argv[2] + + # This is a list of services that require exceptions from the usual + # pattern when gathering their config files + pattern_exceptions = ['glance'] + in_container = is_containerized(service_name) + + + if 'undercloud' in service_name: + cfg_path = "/home/stack" + elif in_container and service_name not in pattern_exceptions: + cfg_path = "/var/lib/config-data/{}/etc/{}".format(service_name, + service_name) + # Glance has all configs in a folder named glance_api, ps shows no + # processes outside of the container, so I assume those are the right + # configs, even though the container is also named glance-api + # jkilpatr 7/13/17 + elif in_container and 'glance' in service_name: + cfg_path = "/var/lib/config-data/glance_api/etc/glance" + else: + cfg_path = "/etc/{}".format(service_name) + + print("Parsing all .conf files in {}".format(cfg_path)) + output = {} + for item in get_configs_list(cfg_path): + full_path = "{}/{}".format(cfg_path, + item) + output.update(parse_config(service_name, full_path)) + add_conf_location(service_name, full_path, output) + # Required to find and load the active neutron plugin file. + if 'neutron' in service_name: + output.update(get_neutron_plugin(output, cfg_path)) + + print_vars_file(output, outfile) if __name__ == '__main__': sys.exit(main()) + diff --git a/ansible/gather/roles/common/tasks/main.yml b/ansible/gather/roles/common/tasks/main.yml index de1c0bf94..b7c510c4a 100644 --- a/ansible/gather/roles/common/tasks/main.yml +++ b/ansible/gather/roles/common/tasks/main.yml @@ -1,3 +1,18 @@ --- - name: Copy config parser script to remote copy: src=openstack-config-parser.py dest=/tmp/openstack-config-parser.py + +- name: Determine if docker is running + shell: docker ps | wc -l + register: docker_ps + +- name: Set var for container deployment + set_fact: + containers: True + config_path: /var/lib/config-data/ + when: docker_ps.stdout|int > 1 + +- name: Set fact for non-container deployment + set_fact: + config_path: /etc + when: docker_ps.stdout|int < 2 diff --git a/ansible/gather/roles/gather/tasks/main.yml b/ansible/gather/roles/gather/tasks/main.yml new file mode 100644 index 000000000..83ca18b08 --- /dev/null +++ b/ansible/gather/roles/gather/tasks/main.yml @@ -0,0 +1,65 @@ +--- +# +# Tasks to get facts +# + +- name: Check for the config - container + become: true + stat: path="container_config_paths[item].config" + register: config_containers + when: hostvars[inventory_hostname]['containers'] is defined + with_items: "{{ container_config_paths }}" + +- name: Check for the config + become: true + stat: path="{{hostvars[inventory_hostname]['config_path']}}nova/nova.conf" + register: config + when: hostvars[inventory_hostname]['containers'] is not defined + +- name: Create tmp dir + become: true + shell: mktemp -d -p /tmp -t XXX-metadata + register: tmp + +- name: Parse config - containers + become: true + shell: "python /tmp/openstack-config-parser.py {{item}} {{container_config_paths[item].config}} {{tmp.stdout}}/{{item}}.yml" + when: hostvars[inventory_hostname]['containers'] is defined + ignore_errors: true + with_items: "{{ container_config_paths }}" + +- name: Parse config + become: true + shell: python /tmp/openstack-config-parser.py {{config_paths[item]}} {{hostvars[inventory_hostname]['config_path']}}/nova/nova.conf /tmp/out.yml + when: config.stat.exists and hostvars[inventory_hostname]['containers'] is not defined + ignore_errors: true + +- name: Create local tmp dir + become: false + local_action: shell mktemp -d -p /tmp -t XXX-metadata + register: localtmp + +- name: Fetch output - containers + fetch: src={{tmp.stdout}}/{{item}}.yml dest={{localtmp.stdout}}/{{item}}.yml flat=yes + when: hostvars[inventory_hostname]['containers'] is defined + ignore_errors: true + with_items: "{{ container_config_paths }}" + +- name: Assemble metadata - containers + local_action: assemble src="{{localtmp.stdout}}" dest="{{localtmp.stdout}}/out.yml" + +- name: Load configuration variables - containers + include_vars: "{{localtmp.stdout}}/out.yml" + when: hostvars[inventory_hostname]['containers'] is defined + ignore_errors: true + +- name: Fetch output + fetch: src=/tmp/out.yml dest=/tmp/out.yml flat=yes + when: hostvars[inventory_hostname]['containers'] is not defined + ignore_errors: true + +- name: Load configuration variables + include_vars: /tmp/out.yml + when: hostvars[inventory_hostname]['containers'] is not defined + ignore_errors: true + diff --git a/ansible/gather/roles/glance/tasks/main.yml b/ansible/gather/roles/glance/tasks/main.yml index 1ce5c5104..3330aad7a 100644 --- a/ansible/gather/roles/glance/tasks/main.yml +++ b/ansible/gather/roles/glance/tasks/main.yml @@ -3,19 +3,12 @@ # Tasks to get Glance facts # -- name: Get config files for Glance - shell: "ls /etc/glance/*.conf" - register: glance_config - - name: Parse Glance config files become: true - shell: "python /tmp/openstack-config-parser.py glance {{ item }} /tmp/{{ item | basename }}.yml" - with_items: "{{ glance_config.stdout_lines }}" + shell: "python /tmp/openstack-config-parser.py glance /tmp/out.yml" - name: Fetch output - fetch: "src=/tmp/{{ item | basename }}.yml dest=/tmp/{{ item | basename }}.yml flat=yes" - with_items: "{{ glance_config.stdout_lines }}" + fetch: "src=/tmp/out.yml dest=/tmp/out.yml flat=yes" - name: Load configuration variables - include_vars: "/tmp/{{ item | basename }}.yml" - with_items: "{{ glance_config.stdout_lines }}" + include_vars: "/tmp/out.yml" diff --git a/ansible/gather/roles/gnocchi/tasks/main.yml b/ansible/gather/roles/gnocchi/tasks/main.yml index 6574f6f00..29894a75a 100644 --- a/ansible/gather/roles/gnocchi/tasks/main.yml +++ b/ansible/gather/roles/gnocchi/tasks/main.yml @@ -10,7 +10,7 @@ - name: Parse Gnocchi config become: true - shell: python /tmp/openstack-config-parser.py gnocchi /etc/gnocchi/gnocchi.conf /tmp/out.yml + shell: python /tmp/openstack-config-parser.py gnocchi /tmp/out.yml when: gnocchi_config.stat.exists - name: Fetch output diff --git a/ansible/gather/roles/heat/tasks/main.yml b/ansible/gather/roles/heat/tasks/main.yml index 6f3030bc6..26c4a7a75 100644 --- a/ansible/gather/roles/heat/tasks/main.yml +++ b/ansible/gather/roles/heat/tasks/main.yml @@ -10,7 +10,7 @@ - name: Parse Heat config become: true - shell: python /tmp/openstack-config-parser.py heat /etc/heat/heat.conf /tmp/out.yml + shell: python /tmp/openstack-config-parser.py heat /tmp/out.yml when: heat_config.stat.exists - name: Fetch output diff --git a/ansible/gather/roles/keystone/tasks/main.yml b/ansible/gather/roles/keystone/tasks/main.yml index e8c9da47b..f09901a84 100644 --- a/ansible/gather/roles/keystone/tasks/main.yml +++ b/ansible/gather/roles/keystone/tasks/main.yml @@ -10,7 +10,7 @@ - name: Parse Keystone config become: true - shell: python /tmp/openstack-config-parser.py keystone /etc/keystone/keystone.conf /tmp/out.yml + shell: python /tmp/openstack-config-parser.py keystone /tmp/out.yml when: keystone_config.stat.exists - name: Fetch output diff --git a/ansible/gather/roles/mysql/tasks/main.yml b/ansible/gather/roles/mysql/tasks/main.yml index 4d59dfa9d..492bdd871 100644 --- a/ansible/gather/roles/mysql/tasks/main.yml +++ b/ansible/gather/roles/mysql/tasks/main.yml @@ -1,3 +1,5 @@ +--- + # # Get mysql facts # @@ -5,10 +7,23 @@ shell: mysql -e "show variables like 'max_connections';" | grep max_connections | awk '{print $2}' register: max_conn ignore_errors: true + when: hostvars[inventory_hostname]['containers'] is not defined + + - name: Get max_connections on the database + shell: docker exec mysql cat /etc/my.cnf.d/galera.cnf| grep max_connections | awk -F ' = ' '{print $2}' + register: max_conn_container + ignore_errors: true + when: hostvars[inventory_hostname]['containers'] is defined - name: Set max database connections set_fact: openstack_mysql_max_connections: "{{ max_conn.stdout }}" + when: hostvars[inventory_hostname]['containers'] is not defined + + - name: Set max database connections + set_fact: + openstack_mysql_max_connections: "{{ max_conn_container.stdout }}" + when: hostvars[inventory_hostname]['containers'] is defined - name : Get file descriptors for the mysql process shell: cat /proc/$(pgrep mysqld_safe)/limits | grep "open files" | awk '{print $4}' @@ -17,5 +32,3 @@ - name: Set file descriptors fact for mysql set_fact: openstack_mysql_file_descriptors: "{{ mysql_desc.stdout }}" - - diff --git a/ansible/gather/roles/neutron/tasks/main.yml b/ansible/gather/roles/neutron/tasks/main.yml index 7113158b8..028636dcb 100644 --- a/ansible/gather/roles/neutron/tasks/main.yml +++ b/ansible/gather/roles/neutron/tasks/main.yml @@ -10,7 +10,7 @@ - name: Parse Neutron config become: true - shell: python /tmp/openstack-config-parser.py neutron /etc/neutron/neutron.conf /tmp/out.yml + shell: python /tmp/openstack-config-parser.py neutron /tmp/out.yml when: neutron_config.stat.exists - name: Fetch output @@ -21,12 +21,3 @@ include_vars: /tmp/out.yml when: neutron_config.stat.exists -- name: Parse Neutron plugin.ini - become: true - shell: python /tmp/openstack-config-parser.py neutron-plugin /etc/neutron/plugin.ini /tmp/out.yml - -- name: Fetch output - fetch: src=/tmp/out.yml dest=/tmp/out.yml flat=yes - -- name: Load configuration variables - include_vars: /tmp/out.yml diff --git a/ansible/gather/roles/nova/tasks/main.yml b/ansible/gather/roles/nova/tasks/main.yml index 1b09f5846..1611bdcd5 100644 --- a/ansible/gather/roles/nova/tasks/main.yml +++ b/ansible/gather/roles/nova/tasks/main.yml @@ -10,7 +10,7 @@ - name: Parse Nova config become: true - shell: python /tmp/openstack-config-parser.py nova /etc/nova/nova.conf /tmp/out.yml + shell: python /tmp/openstack-config-parser.py nova /tmp/out.yml when: nova_config.stat.exists - name: Fetch output diff --git a/ansible/gather/roles/rabbitmq/tasks/main.yml b/ansible/gather/roles/rabbitmq/tasks/main.yml index 5d1e26141..25dec670f 100644 --- a/ansible/gather/roles/rabbitmq/tasks/main.yml +++ b/ansible/gather/roles/rabbitmq/tasks/main.yml @@ -6,9 +6,20 @@ shell: rabbitmqctl status | grep file_descriptors | awk -F',' '{print $3}' | sed 's/.$//' register: rabbitmq_desc ignore_errors: true + when: hostvars[inventory_hostname]['containers'] is not defined + + - name : Get rabbitmq file descriptors - containers + shell: docker exec rabbitmq rabbitmqctl status | grep total_limit | awk -F',' '{print $2}'| sed 's/.$//' + register: rabbitmq_desc_container + ignore_errors: true + when: hostvars[inventory_hostname]['containers'] is defined - name: Set rabbitmq file descriptors set_fact: openstack_rabbitmq_file_descriptors: "{{ rabbitmq_desc.stdout }}" + when: hostvars[inventory_hostname]['containers'] is not defined - + - name: Set rabbitmq file descriptors - containers + set_fact: + openstack_rabbitmq_file_descriptors: "{{ rabbitmq_desc_container.stdout }}" + when: hostvars[inventory_hostname]['containers'] is defined diff --git a/ansible/gather/roles/undercloud/tasks/main.yml b/ansible/gather/roles/undercloud/tasks/main.yml index 194c787e8..e55bce228 100644 --- a/ansible/gather/roles/undercloud/tasks/main.yml +++ b/ansible/gather/roles/undercloud/tasks/main.yml @@ -11,7 +11,7 @@ - name: Undercloud.conf become: true - shell: python /tmp/openstack-config-parser.py undercloud /home/stack/undercloud.conf /tmp/out.yml + shell: python /tmp/openstack-config-parser.py undercloud /tmp/out.yml when: undercloud_conf.stat.exists - name: Fetch output diff --git a/ansible/gather/site.yml b/ansible/gather/site.yml index 07ed43296..e384dd9fb 100644 --- a/ansible/gather/site.yml +++ b/ansible/gather/site.yml @@ -3,6 +3,7 @@ remote_user: "{{ host_remote_user }}" become: true roles: + - common - compute - hosts: controller