Merge "Gather script for containers"

ansible/gather/roles/ceilometer/tasks/main.yml

@@ -9,7 +9,7 @@
 
 - name: Parse Ceilometer config
   become: true
-  shell: python /tmp/openstack-config-parser.py ceilometer /etc/ceilometer/ceilometer.conf /tmp/out.yml
+  shell: python /tmp/openstack-config-parser.py ceilometer /tmp/out.yml
   when: ceilometer_config.stat.exists
 
 - name: Fetch output

ansible/gather/roles/cinder/tasks/main.yml

@@ -9,7 +9,7 @@
 
 - name: Parse Cinder config
   become: true
-  shell: python /tmp/openstack-config-parser.py cinder /etc/cinder/cinder.conf /tmp/out.yml
+  shell: python /tmp/openstack-config-parser.py cinder /tmp/out.yml
   when: cinder_config.stat.exists
 
 - name: Fetch output

ansible/gather/roles/common/files/openstack-config-parser.py

@@ -12,31 +12,44 @@
 #   limitations under the License.
 
 import sys
-# usage: openstack-config-parser.py [service] [config file] [output file]
+import os
+import subprocess
+# usage: openstack-config-parser.py [service] [output file]
 
+def run_cmd(cmd):
+    process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE,
+                               stderr=subprocess.PIPE)
+    stdout, stderr = process.communicate()
+    output_dict = {}
+    output_dict['stdout'] = stdout.strip()
+    output_dict['stderr'] = stderr.strip()
+    output_dict['rc'] = process.returncode
+    return output_dict
+
+def strip_chars(line):
+    forbidden_chars = ['#', '\n', '"', '\\', ' ', '<', '>']
+    for char in forbidden_chars:
+        line = line.replace(char, '')
+    return line
 
 def parse_config(serviceName, fileName):
-    # a dict containing key/value
-    # pairs, last value is what is
-    # stored.
+    # a dict containing key/value pairs, last value is what is stored.
     values = {}
     with open(fileName) as config:
         section = None
         for line in config:
-            pair = line.replace('#', '')
-            pair = pair.replace('\n', '')
-            pair = pair.replace('"', '')
-            pair = pair.replace('\\', '')
-            pair = pair.replace(' ', '')
-            pair = pair.replace('<', '')
-            pair = pair.replace('>', '')
+            pair = strip_chars(line)
             pair = pair.split('=')
             # excludes any line without a key/val pair
-            valid_line = not line.startswith(
-                "# ") and '[' not in line and line != '\n' and line != '#\n' and "password" not in line.lower()
+            valid_line = not line.startswith("# ") and \
+                         '[' not in line and line != '\n' \
+                         and line != '#\n' and "password" \
+                         not in line.lower()
             if line.startswith('['):
                 section = line.replace('[','').replace(']','').replace('\n','')
-            if '#' not in line and valid_line and not section == None:
+            if '#' not in line and valid_line and not section == None and len(pair) == 2:
+                pair[0] = strip_chars(pair[0])
+                pair[1] = strip_chars(pair[1])
                 values["openstack_S_" + serviceName + "_S_" + section + "_S_" + pair[0]] = pair[1]
     return values
 
@@ -45,26 +58,97 @@ def try_type(val):
     try:
         int(val)
         return val
-    except ValueError:
+    except (ValueError, TypeError):
         try:
             float(val)
             return val
-        except ValueError:
-            if val.lower() in ("true", "false"):
+        except (ValueError, TypeError):
+            if type(val) is list:
+                return "\"" + str(val) + "\""
+            elif val.lower() in ("true", "false"):
                 return val
             else:
                 return "\"" + val + "\""
 
 
+def add_conf_location(serviceName, fileName, values): 
+    # Stores the exact location we gathered this config from.
+    index = "openstack_S_" + serviceName + "_S_" + "Browbeat" + "_S_" + "gather_conf_path"
+    if index in values:
+        values[index].append(fileName)
+    else:
+        values[index] = [fileName]
+
 def print_vars_file(values, fileName):
     with open(fileName, 'w') as output:
         for key in values:
             output.write(key + ": " + try_type(values[key]) + "\n")
 
+def is_containerized(service_name):
+    out = run_cmd("docker ps")
+    if service_name in out['stdout']:
+        return True
+    else:
+        return False
+
+def get_configs_list(path, extension='.conf'):
+    configs = []
+    for item in os.listdir(path):
+        if item.endswith(extension):
+            configs.extend([item])
+    return configs
+
+def get_neutron_plugin(output, cfg_path):
+    plugin = output['openstack_S_neutron_S_DEFAULT_S_core_plugin']
+    plugin_path = "{}/plugins/{}/".format(cfg_path, plugin)
+    for item in get_configs_list(plugin_path, extension='.ini'):
+        full_path = "{}/{}".format(plugin_path,
+                                   item)
+        output.update(parse_config("neutron-plugin", full_path))
+        add_conf_location("neutron", full_path, output)
+    return output
 
 def main():
-    output = parse_config(sys.argv[1], sys.argv[2])
-    print_vars_file(output, sys.argv[3])
+    if len(sys.argv) < 3:
+        print("usage: openstack-config-parser.py [service] [output file]")
+        exit(1)
+
+    service_name = sys.argv[1]
+    outfile = sys.argv[2]
+
+    # This is a list of services that require exceptions from the usual
+    # pattern when gathering their config files
+    pattern_exceptions = ['glance']
+    in_container = is_containerized(service_name)
+
+
+    if 'undercloud' in service_name:
+        cfg_path = "/home/stack"
+    elif in_container and service_name not in pattern_exceptions:
+        cfg_path = "/var/lib/config-data/{}/etc/{}".format(service_name,
+                                                           service_name)
+    # Glance has all configs in a folder named glance_api, ps shows no
+    # processes outside of the container, so I assume those are the right
+    # configs, even though the container is also named glance-api
+    # jkilpatr 7/13/17
+    elif in_container and 'glance' in service_name:
+        cfg_path = "/var/lib/config-data/glance_api/etc/glance"
+    else:
+        cfg_path = "/etc/{}".format(service_name)
+
+    print("Parsing all .conf files in {}".format(cfg_path))
+    output = {}
+    for item in get_configs_list(cfg_path):
+        full_path = "{}/{}".format(cfg_path,
+                                   item)
+        output.update(parse_config(service_name, full_path))
+        add_conf_location(service_name, full_path, output)
+    # Required to find and load the active neutron plugin file.
+    if 'neutron' in service_name:
+        output.update(get_neutron_plugin(output, cfg_path))
+
+    print_vars_file(output, outfile)
 
 if __name__ == '__main__':
     sys.exit(main())
+

ansible/gather/roles/common/tasks/main.yml

@@ -1,3 +1,18 @@
 ---
 - name: Copy config parser script to remote
   copy: src=openstack-config-parser.py dest=/tmp/openstack-config-parser.py
+
+- name: Determine if docker is running
+  shell: docker ps | wc -l
+  register: docker_ps
+
+- name: Set var for container deployment
+  set_fact:
+    containers: True
+    config_path: /var/lib/config-data/
+  when: docker_ps.stdout|int > 1
+
+- name: Set fact for non-container deployment
+  set_fact:
+    config_path: /etc
+  when: docker_ps.stdout|int < 2

ansible/gather/roles/gather/tasks/main.yml

@@ -0,0 +1,65 @@
+---
+#
+# Tasks to get facts
+#
+
+- name: Check for the config - container
+  become: true
+  stat: path="container_config_paths[item].config"
+  register: config_containers
+  when: hostvars[inventory_hostname]['containers'] is defined
+  with_items: "{{ container_config_paths }}"
+
+- name: Check for the config
+  become: true
+  stat: path="{{hostvars[inventory_hostname]['config_path']}}nova/nova.conf"
+  register: config
+  when: hostvars[inventory_hostname]['containers'] is not defined
+
+- name: Create tmp dir
+  become: true
+  shell: mktemp -d -p /tmp -t XXX-metadata
+  register: tmp
+
+- name: Parse config - containers
+  become: true
+  shell: "python /tmp/openstack-config-parser.py {{item}} {{container_config_paths[item].config}} {{tmp.stdout}}/{{item}}.yml"
+  when: hostvars[inventory_hostname]['containers'] is defined
+  ignore_errors: true
+  with_items: "{{ container_config_paths }}"
+
+- name: Parse config
+  become: true
+  shell: python /tmp/openstack-config-parser.py {{config_paths[item]}} {{hostvars[inventory_hostname]['config_path']}}/nova/nova.conf /tmp/out.yml
+  when: config.stat.exists and hostvars[inventory_hostname]['containers'] is not defined
+  ignore_errors: true
+
+- name: Create local tmp dir
+  become: false
+  local_action: shell mktemp -d -p /tmp -t XXX-metadata
+  register: localtmp
+
+- name: Fetch output - containers
+  fetch: src={{tmp.stdout}}/{{item}}.yml dest={{localtmp.stdout}}/{{item}}.yml flat=yes
+  when: hostvars[inventory_hostname]['containers'] is defined
+  ignore_errors: true
+  with_items: "{{ container_config_paths }}"
+
+- name: Assemble metadata - containers
+  local_action: assemble src="{{localtmp.stdout}}" dest="{{localtmp.stdout}}/out.yml"
+
+- name: Load configuration variables - containers
+  include_vars: "{{localtmp.stdout}}/out.yml"
+  when: hostvars[inventory_hostname]['containers'] is defined
+  ignore_errors: true
+
+- name: Fetch output
+  fetch: src=/tmp/out.yml dest=/tmp/out.yml flat=yes
+  when: hostvars[inventory_hostname]['containers'] is not defined
+  ignore_errors: true
+
+- name: Load configuration variables
+  include_vars: /tmp/out.yml
+  when: hostvars[inventory_hostname]['containers'] is not defined
+  ignore_errors: true
+

ansible/gather/roles/glance/tasks/main.yml

@@ -3,19 +3,12 @@
 # Tasks to get Glance facts
 #
 
-- name: Get config files for Glance
-  shell: "ls /etc/glance/*.conf"
-  register: glance_config
-
 - name: Parse Glance config files
   become: true
-  shell: "python /tmp/openstack-config-parser.py glance {{ item }} /tmp/{{ item | basename }}.yml"
-  with_items: "{{ glance_config.stdout_lines }}"
+  shell: "python /tmp/openstack-config-parser.py glance /tmp/out.yml"
 
 - name: Fetch output
-  fetch: "src=/tmp/{{ item | basename }}.yml dest=/tmp/{{ item | basename  }}.yml flat=yes"
-  with_items: "{{ glance_config.stdout_lines }}"
+  fetch: "src=/tmp/out.yml dest=/tmp/out.yml flat=yes"
 
 - name: Load configuration variables
-  include_vars: "/tmp/{{ item | basename }}.yml"
-  with_items: "{{ glance_config.stdout_lines }}"
+  include_vars: "/tmp/out.yml"

ansible/gather/roles/gnocchi/tasks/main.yml

@@ -10,7 +10,7 @@
 
 - name: Parse Gnocchi config
   become: true
-  shell: python /tmp/openstack-config-parser.py gnocchi /etc/gnocchi/gnocchi.conf /tmp/out.yml
+  shell: python /tmp/openstack-config-parser.py gnocchi /tmp/out.yml
   when: gnocchi_config.stat.exists
 
 - name: Fetch output

ansible/gather/roles/heat/tasks/main.yml

@@ -10,7 +10,7 @@
 
 - name: Parse Heat config
   become: true
-  shell: python /tmp/openstack-config-parser.py heat /etc/heat/heat.conf /tmp/out.yml
+  shell: python /tmp/openstack-config-parser.py heat /tmp/out.yml
   when: heat_config.stat.exists
 
 - name: Fetch output

ansible/gather/roles/keystone/tasks/main.yml

@@ -10,7 +10,7 @@
 
 - name: Parse Keystone config
   become: true
-  shell: python /tmp/openstack-config-parser.py keystone /etc/keystone/keystone.conf /tmp/out.yml
+  shell: python /tmp/openstack-config-parser.py keystone /tmp/out.yml
   when: keystone_config.stat.exists
 
 - name: Fetch output

ansible/gather/roles/mysql/tasks/main.yml

@@ -1,3 +1,5 @@
+---
+
 #
 # Get mysql facts
 #
@@ -5,10 +7,23 @@
     shell: mysql -e "show variables like 'max_connections';" | grep max_connections | awk '{print $2}'
     register: max_conn
     ignore_errors: true
+    when: hostvars[inventory_hostname]['containers'] is not defined
+
+  - name: Get max_connections on the database
+    shell: docker exec mysql cat /etc/my.cnf.d/galera.cnf| grep max_connections | awk -F ' = ' '{print $2}'
+    register: max_conn_container
+    ignore_errors: true
+    when: hostvars[inventory_hostname]['containers'] is defined
 
   - name: Set max database connections
     set_fact:
       openstack_mysql_max_connections: "{{ max_conn.stdout }}"
+    when: hostvars[inventory_hostname]['containers'] is not defined
+
+  - name: Set max database connections
+    set_fact:
+      openstack_mysql_max_connections: "{{ max_conn_container.stdout }}"
+    when: hostvars[inventory_hostname]['containers'] is defined
 
   - name : Get file descriptors for the mysql process
     shell: cat /proc/$(pgrep mysqld_safe)/limits | grep "open files" | awk '{print $4}'
@@ -17,5 +32,3 @@
   - name: Set file descriptors fact for mysql
     set_fact:
       openstack_mysql_file_descriptors: "{{ mysql_desc.stdout }}"
-
-

ansible/gather/roles/neutron/tasks/main.yml

@@ -10,7 +10,7 @@
 
 - name: Parse Neutron config
   become: true
-  shell: python /tmp/openstack-config-parser.py neutron /etc/neutron/neutron.conf /tmp/out.yml
+  shell: python /tmp/openstack-config-parser.py neutron /tmp/out.yml
   when: neutron_config.stat.exists
 
 - name: Fetch output
@@ -21,12 +21,3 @@
   include_vars: /tmp/out.yml
   when: neutron_config.stat.exists
 
-- name: Parse Neutron plugin.ini
-  become: true
-  shell: python /tmp/openstack-config-parser.py neutron-plugin /etc/neutron/plugin.ini /tmp/out.yml
-
-- name: Fetch output
-  fetch: src=/tmp/out.yml dest=/tmp/out.yml flat=yes
-
-- name: Load configuration variables
-  include_vars: /tmp/out.yml

ansible/gather/roles/nova/tasks/main.yml

@@ -10,7 +10,7 @@
 
 - name: Parse Nova config
   become: true
-  shell: python /tmp/openstack-config-parser.py nova /etc/nova/nova.conf /tmp/out.yml
+  shell: python /tmp/openstack-config-parser.py nova /tmp/out.yml
   when: nova_config.stat.exists
 
 - name: Fetch output

ansible/gather/roles/rabbitmq/tasks/main.yml

@@ -6,9 +6,20 @@
     shell: rabbitmqctl status | grep file_descriptors | awk -F',' '{print $3}' | sed 's/.$//'
     register: rabbitmq_desc
     ignore_errors: true
+    when: hostvars[inventory_hostname]['containers'] is not defined
+
+  - name : Get rabbitmq file descriptors - containers
+    shell: docker exec rabbitmq rabbitmqctl status | grep total_limit | awk -F',' '{print $2}'| sed 's/.$//'
+    register: rabbitmq_desc_container
+    ignore_errors: true
+    when: hostvars[inventory_hostname]['containers'] is defined
 
   - name: Set rabbitmq file descriptors
     set_fact:
       openstack_rabbitmq_file_descriptors: "{{ rabbitmq_desc.stdout }}"
+    when: hostvars[inventory_hostname]['containers'] is not defined
 
-
+  - name: Set rabbitmq file descriptors - containers
+    set_fact:
+      openstack_rabbitmq_file_descriptors: "{{ rabbitmq_desc_container.stdout }}"
+    when: hostvars[inventory_hostname]['containers'] is defined

ansible/gather/roles/undercloud/tasks/main.yml

@@ -11,7 +11,7 @@
 
   - name: Undercloud.conf
     become: true
-    shell: python /tmp/openstack-config-parser.py undercloud /home/stack/undercloud.conf /tmp/out.yml
+    shell: python /tmp/openstack-config-parser.py undercloud /tmp/out.yml
     when: undercloud_conf.stat.exists
 
   - name: Fetch output

ansible/gather/site.yml

@@ -3,6 +3,7 @@
   remote_user: "{{ host_remote_user }}"
   become: true
   roles:
+    - common
     - compute
 
 - hosts: controller