From 8c668d51aa05429618f4547a1ef84725ec91829e Mon Sep 17 00:00:00 2001
From: Joe Talerico <jtaleric@redhat.com>
Date: Fri, 30 Jun 2017 15:05:40 -0400
Subject: [PATCH] Gather script for containers

Currently our gather playbook assumes all configuration files will be in
/etc. This assumption isn't true with container deployments. Currently
the configuration files are located in
/var/lib/config-data/<service>/etc/<service>/.

The Gather script needs to support both container and non-container
deployments. This patchset updates the config parser python script
to check if a service is in the running containers list and then
determine it's appropriate path, grab all of multiple config files
in that path, then parse and drop them off for ansible to use.

This method automagically supports all possible mixes of containerized
uncontainerized services and will always grab the correct config even
if that changes build to build or run to run.

It's also easily extensible for many possible config locations or
different container types by adding another condition or additional
search paths.

Change-Id: I95a3059c7fc263733ac64aa894c6dbf11e2a909f
Closes-bug: #1701264
---
 .../gather/roles/ceilometer/tasks/main.yml    |   2 +-
 ansible/gather/roles/cinder/tasks/main.yml    |   2 +-
 .../common/files/openstack-config-parser.py   | 122 +++++++++++++++---
 ansible/gather/roles/common/tasks/main.yml    |  15 +++
 ansible/gather/roles/gather/tasks/main.yml    |  65 ++++++++++
 ansible/gather/roles/glance/tasks/main.yml    |  13 +-
 ansible/gather/roles/gnocchi/tasks/main.yml   |   2 +-
 ansible/gather/roles/heat/tasks/main.yml      |   2 +-
 ansible/gather/roles/keystone/tasks/main.yml  |   2 +-
 ansible/gather/roles/mysql/tasks/main.yml     |  17 ++-
 ansible/gather/roles/neutron/tasks/main.yml   |  11 +-
 ansible/gather/roles/nova/tasks/main.yml      |   2 +-
 ansible/gather/roles/rabbitmq/tasks/main.yml  |  13 +-
 .../gather/roles/undercloud/tasks/main.yml    |   2 +-
 ansible/gather/site.yml                       |   1 +
 15 files changed, 222 insertions(+), 49 deletions(-)
 create mode 100644 ansible/gather/roles/gather/tasks/main.yml

diff --git a/ansible/gather/roles/ceilometer/tasks/main.yml b/ansible/gather/roles/ceilometer/tasks/main.yml
index df5031fc3..fcbd454ba 100644
--- a/ansible/gather/roles/ceilometer/tasks/main.yml
+++ b/ansible/gather/roles/ceilometer/tasks/main.yml
@@ -9,7 +9,7 @@
 
 - name: Parse Ceilometer config
   become: true
-  shell: python /tmp/openstack-config-parser.py ceilometer /etc/ceilometer/ceilometer.conf /tmp/out.yml
+  shell: python /tmp/openstack-config-parser.py ceilometer /tmp/out.yml
   when: ceilometer_config.stat.exists
 
 - name: Fetch output
diff --git a/ansible/gather/roles/cinder/tasks/main.yml b/ansible/gather/roles/cinder/tasks/main.yml
index b82a507a3..47de30617 100644
--- a/ansible/gather/roles/cinder/tasks/main.yml
+++ b/ansible/gather/roles/cinder/tasks/main.yml
@@ -9,7 +9,7 @@
 
 - name: Parse Cinder config
   become: true
-  shell: python /tmp/openstack-config-parser.py cinder /etc/cinder/cinder.conf /tmp/out.yml
+  shell: python /tmp/openstack-config-parser.py cinder /tmp/out.yml
   when: cinder_config.stat.exists
 
 - name: Fetch output
diff --git a/ansible/gather/roles/common/files/openstack-config-parser.py b/ansible/gather/roles/common/files/openstack-config-parser.py
index c56298f87..6249fa7d0 100644
--- a/ansible/gather/roles/common/files/openstack-config-parser.py
+++ b/ansible/gather/roles/common/files/openstack-config-parser.py
@@ -12,31 +12,44 @@
 #   limitations under the License.
 
 import sys
-# usage: openstack-config-parser.py [service] [config file] [output file]
+import os
+import subprocess
+# usage: openstack-config-parser.py [service] [output file]
 
+def run_cmd(cmd):
+    process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE,
+                               stderr=subprocess.PIPE)
+    stdout, stderr = process.communicate()
+    output_dict = {}
+    output_dict['stdout'] = stdout.strip()
+    output_dict['stderr'] = stderr.strip()
+    output_dict['rc'] = process.returncode
+    return output_dict
+
+def strip_chars(line):
+    forbidden_chars = ['#', '\n', '"', '\\', ' ', '<', '>']
+    for char in forbidden_chars:
+        line = line.replace(char, '')
+    return line
 
 def parse_config(serviceName, fileName):
-    # a dict containing key/value
-    # pairs, last value is what is
-    # stored.
+    # a dict containing key/value pairs, last value is what is stored.
     values = {}
     with open(fileName) as config:
         section = None
         for line in config:
-            pair = line.replace('#', '')
-            pair = pair.replace('\n', '')
-            pair = pair.replace('"', '')
-            pair = pair.replace('\\', '')
-            pair = pair.replace(' ', '')
-            pair = pair.replace('<', '')
-            pair = pair.replace('>', '')
+            pair = strip_chars(line)
             pair = pair.split('=')
             # excludes any line without a key/val pair
-            valid_line = not line.startswith(
-                "# ") and '[' not in line and line != '\n' and line != '#\n' and "password" not in line.lower()
+            valid_line = not line.startswith("# ") and \
+                         '[' not in line and line != '\n' \
+                         and line != '#\n' and "password" \
+                         not in line.lower()
             if line.startswith('['):
                 section = line.replace('[','').replace(']','').replace('\n','')
-            if '#' not in line and valid_line and not section == None:
+            if '#' not in line and valid_line and not section == None and len(pair) == 2:
+                pair[0] = strip_chars(pair[0])
+                pair[1] = strip_chars(pair[1])
                 values["openstack_S_" + serviceName + "_S_" + section + "_S_" + pair[0]] = pair[1]
     return values
 
@@ -45,26 +58,97 @@ def try_type(val):
     try:
         int(val)
         return val
-    except ValueError:
+    except (ValueError, TypeError):
         try:
             float(val)
             return val
-        except ValueError:
-            if val.lower() in ("true", "false"):
+        except (ValueError, TypeError):
+            if type(val) is list:
+                return "\"" + str(val) + "\""
+            elif val.lower() in ("true", "false"):
                 return val
             else:
                 return "\"" + val + "\""
 
 
+def add_conf_location(serviceName, fileName, values): 
+    # Stores the exact location we gathered this config from.
+    index = "openstack_S_" + serviceName + "_S_" + "Browbeat" + "_S_" + "gather_conf_path"
+    if index in values:
+        values[index].append(fileName)
+    else:
+        values[index] = [fileName]
+
 def print_vars_file(values, fileName):
     with open(fileName, 'w') as output:
         for key in values:
             output.write(key + ": " + try_type(values[key]) + "\n")
 
+def is_containerized(service_name):
+    out = run_cmd("docker ps")
+    if service_name in out['stdout']:
+        return True
+    else:
+        return False
+
+def get_configs_list(path, extension='.conf'):
+    configs = []
+    for item in os.listdir(path):
+        if item.endswith(extension):
+            configs.extend([item])
+    return configs
+
+def get_neutron_plugin(output, cfg_path):
+    plugin = output['openstack_S_neutron_S_DEFAULT_S_core_plugin']
+    plugin_path = "{}/plugins/{}/".format(cfg_path, plugin)
+    for item in get_configs_list(plugin_path, extension='.ini'):
+        full_path = "{}/{}".format(plugin_path,
+                                   item)
+        output.update(parse_config("neutron-plugin", full_path))
+        add_conf_location("neutron", full_path, output)
+    return output
 
 def main():
-    output = parse_config(sys.argv[1], sys.argv[2])
-    print_vars_file(output, sys.argv[3])
+    if len(sys.argv) < 3:
+        print("usage: openstack-config-parser.py [service] [output file]")
+        exit(1)
+
+    service_name = sys.argv[1]
+    outfile = sys.argv[2]
+
+    # This is a list of services that require exceptions from the usual
+    # pattern when gathering their config files
+    pattern_exceptions = ['glance']
+    in_container = is_containerized(service_name)
+
+
+    if 'undercloud' in service_name:
+        cfg_path = "/home/stack"
+    elif in_container and service_name not in pattern_exceptions:
+        cfg_path = "/var/lib/config-data/{}/etc/{}".format(service_name,
+                                                           service_name)
+    # Glance has all configs in a folder named glance_api, ps shows no
+    # processes outside of the container, so I assume those are the right
+    # configs, even though the container is also named glance-api
+    # jkilpatr 7/13/17
+    elif in_container and 'glance' in service_name:
+        cfg_path = "/var/lib/config-data/glance_api/etc/glance"
+    else:
+        cfg_path = "/etc/{}".format(service_name)
+
+    print("Parsing all .conf files in {}".format(cfg_path))
+    output = {}
+    for item in get_configs_list(cfg_path):
+        full_path = "{}/{}".format(cfg_path,
+                                   item)
+        output.update(parse_config(service_name, full_path))
+        add_conf_location(service_name, full_path, output)
+    # Required to find and load the active neutron plugin file.
+    if 'neutron' in service_name:
+        output.update(get_neutron_plugin(output, cfg_path))
+
+    print_vars_file(output, outfile)
 
 if __name__ == '__main__':
     sys.exit(main())
+
diff --git a/ansible/gather/roles/common/tasks/main.yml b/ansible/gather/roles/common/tasks/main.yml
index de1c0bf94..b7c510c4a 100644
--- a/ansible/gather/roles/common/tasks/main.yml
+++ b/ansible/gather/roles/common/tasks/main.yml
@@ -1,3 +1,18 @@
 ---
 - name: Copy config parser script to remote
   copy: src=openstack-config-parser.py dest=/tmp/openstack-config-parser.py
+
+- name: Determine if docker is running
+  shell: docker ps | wc -l
+  register: docker_ps
+
+- name: Set var for container deployment
+  set_fact:
+    containers: True
+    config_path: /var/lib/config-data/
+  when: docker_ps.stdout|int > 1
+
+- name: Set fact for non-container deployment
+  set_fact:
+    config_path: /etc
+  when: docker_ps.stdout|int < 2
diff --git a/ansible/gather/roles/gather/tasks/main.yml b/ansible/gather/roles/gather/tasks/main.yml
new file mode 100644
index 000000000..83ca18b08
--- /dev/null
+++ b/ansible/gather/roles/gather/tasks/main.yml
@@ -0,0 +1,65 @@
+---
+#
+# Tasks to get facts
+#
+
+- name: Check for the config - container
+  become: true
+  stat: path="container_config_paths[item].config"
+  register: config_containers
+  when: hostvars[inventory_hostname]['containers'] is defined
+  with_items: "{{ container_config_paths }}"
+
+- name: Check for the config
+  become: true
+  stat: path="{{hostvars[inventory_hostname]['config_path']}}nova/nova.conf"
+  register: config
+  when: hostvars[inventory_hostname]['containers'] is not defined
+
+- name: Create tmp dir
+  become: true
+  shell: mktemp -d -p /tmp -t XXX-metadata
+  register: tmp
+
+- name: Parse config - containers
+  become: true
+  shell: "python /tmp/openstack-config-parser.py {{item}} {{container_config_paths[item].config}} {{tmp.stdout}}/{{item}}.yml"
+  when: hostvars[inventory_hostname]['containers'] is defined
+  ignore_errors: true
+  with_items: "{{ container_config_paths }}"
+
+- name: Parse config
+  become: true
+  shell: python /tmp/openstack-config-parser.py {{config_paths[item]}} {{hostvars[inventory_hostname]['config_path']}}/nova/nova.conf /tmp/out.yml
+  when: config.stat.exists and hostvars[inventory_hostname]['containers'] is not defined
+  ignore_errors: true
+
+- name: Create local tmp dir
+  become: false
+  local_action: shell mktemp -d -p /tmp -t XXX-metadata
+  register: localtmp
+
+- name: Fetch output - containers
+  fetch: src={{tmp.stdout}}/{{item}}.yml dest={{localtmp.stdout}}/{{item}}.yml flat=yes
+  when: hostvars[inventory_hostname]['containers'] is defined
+  ignore_errors: true
+  with_items: "{{ container_config_paths }}"
+
+- name: Assemble metadata - containers
+  local_action: assemble src="{{localtmp.stdout}}" dest="{{localtmp.stdout}}/out.yml"
+
+- name: Load configuration variables - containers
+  include_vars: "{{localtmp.stdout}}/out.yml"
+  when: hostvars[inventory_hostname]['containers'] is defined
+  ignore_errors: true
+
+- name: Fetch output
+  fetch: src=/tmp/out.yml dest=/tmp/out.yml flat=yes
+  when: hostvars[inventory_hostname]['containers'] is not defined
+  ignore_errors: true
+
+- name: Load configuration variables
+  include_vars: /tmp/out.yml
+  when: hostvars[inventory_hostname]['containers'] is not defined
+  ignore_errors: true
+
diff --git a/ansible/gather/roles/glance/tasks/main.yml b/ansible/gather/roles/glance/tasks/main.yml
index 1ce5c5104..3330aad7a 100644
--- a/ansible/gather/roles/glance/tasks/main.yml
+++ b/ansible/gather/roles/glance/tasks/main.yml
@@ -3,19 +3,12 @@
 # Tasks to get Glance facts
 #
 
-- name: Get config files for Glance
-  shell: "ls /etc/glance/*.conf"
-  register: glance_config
-
 - name: Parse Glance config files
   become: true
-  shell: "python /tmp/openstack-config-parser.py glance {{ item }} /tmp/{{ item | basename }}.yml"
-  with_items: "{{ glance_config.stdout_lines }}"
+  shell: "python /tmp/openstack-config-parser.py glance /tmp/out.yml"
 
 - name: Fetch output
-  fetch: "src=/tmp/{{ item | basename }}.yml dest=/tmp/{{ item | basename  }}.yml flat=yes"
-  with_items: "{{ glance_config.stdout_lines }}"
+  fetch: "src=/tmp/out.yml dest=/tmp/out.yml flat=yes"
 
 - name: Load configuration variables
-  include_vars: "/tmp/{{ item | basename }}.yml"
-  with_items: "{{ glance_config.stdout_lines }}"
+  include_vars: "/tmp/out.yml"
diff --git a/ansible/gather/roles/gnocchi/tasks/main.yml b/ansible/gather/roles/gnocchi/tasks/main.yml
index 6574f6f00..29894a75a 100644
--- a/ansible/gather/roles/gnocchi/tasks/main.yml
+++ b/ansible/gather/roles/gnocchi/tasks/main.yml
@@ -10,7 +10,7 @@
 
 - name: Parse Gnocchi config
   become: true
-  shell: python /tmp/openstack-config-parser.py gnocchi /etc/gnocchi/gnocchi.conf /tmp/out.yml
+  shell: python /tmp/openstack-config-parser.py gnocchi /tmp/out.yml
   when: gnocchi_config.stat.exists
 
 - name: Fetch output
diff --git a/ansible/gather/roles/heat/tasks/main.yml b/ansible/gather/roles/heat/tasks/main.yml
index 6f3030bc6..26c4a7a75 100644
--- a/ansible/gather/roles/heat/tasks/main.yml
+++ b/ansible/gather/roles/heat/tasks/main.yml
@@ -10,7 +10,7 @@
 
 - name: Parse Heat config
   become: true
-  shell: python /tmp/openstack-config-parser.py heat /etc/heat/heat.conf /tmp/out.yml
+  shell: python /tmp/openstack-config-parser.py heat /tmp/out.yml
   when: heat_config.stat.exists
 
 - name: Fetch output
diff --git a/ansible/gather/roles/keystone/tasks/main.yml b/ansible/gather/roles/keystone/tasks/main.yml
index e8c9da47b..f09901a84 100644
--- a/ansible/gather/roles/keystone/tasks/main.yml
+++ b/ansible/gather/roles/keystone/tasks/main.yml
@@ -10,7 +10,7 @@
 
 - name: Parse Keystone config
   become: true
-  shell: python /tmp/openstack-config-parser.py keystone /etc/keystone/keystone.conf /tmp/out.yml
+  shell: python /tmp/openstack-config-parser.py keystone /tmp/out.yml
   when: keystone_config.stat.exists
 
 - name: Fetch output
diff --git a/ansible/gather/roles/mysql/tasks/main.yml b/ansible/gather/roles/mysql/tasks/main.yml
index 4d59dfa9d..492bdd871 100644
--- a/ansible/gather/roles/mysql/tasks/main.yml
+++ b/ansible/gather/roles/mysql/tasks/main.yml
@@ -1,3 +1,5 @@
+---
+
 #
 # Get mysql facts
 #
@@ -5,10 +7,23 @@
     shell: mysql -e "show variables like 'max_connections';" | grep max_connections | awk '{print $2}'
     register: max_conn
     ignore_errors: true
+    when: hostvars[inventory_hostname]['containers'] is not defined
+
+  - name: Get max_connections on the database
+    shell: docker exec mysql cat /etc/my.cnf.d/galera.cnf| grep max_connections | awk -F ' = ' '{print $2}'
+    register: max_conn_container
+    ignore_errors: true
+    when: hostvars[inventory_hostname]['containers'] is defined
 
   - name: Set max database connections
     set_fact:
       openstack_mysql_max_connections: "{{ max_conn.stdout }}"
+    when: hostvars[inventory_hostname]['containers'] is not defined
+
+  - name: Set max database connections
+    set_fact:
+      openstack_mysql_max_connections: "{{ max_conn_container.stdout }}"
+    when: hostvars[inventory_hostname]['containers'] is defined
 
   - name : Get file descriptors for the mysql process
     shell: cat /proc/$(pgrep mysqld_safe)/limits | grep "open files" | awk '{print $4}'
@@ -17,5 +32,3 @@
   - name: Set file descriptors fact for mysql
     set_fact:
       openstack_mysql_file_descriptors: "{{ mysql_desc.stdout }}"
-
-
diff --git a/ansible/gather/roles/neutron/tasks/main.yml b/ansible/gather/roles/neutron/tasks/main.yml
index 7113158b8..028636dcb 100644
--- a/ansible/gather/roles/neutron/tasks/main.yml
+++ b/ansible/gather/roles/neutron/tasks/main.yml
@@ -10,7 +10,7 @@
 
 - name: Parse Neutron config
   become: true
-  shell: python /tmp/openstack-config-parser.py neutron /etc/neutron/neutron.conf /tmp/out.yml
+  shell: python /tmp/openstack-config-parser.py neutron /tmp/out.yml
   when: neutron_config.stat.exists
 
 - name: Fetch output
@@ -21,12 +21,3 @@
   include_vars: /tmp/out.yml
   when: neutron_config.stat.exists
 
-- name: Parse Neutron plugin.ini
-  become: true
-  shell: python /tmp/openstack-config-parser.py neutron-plugin /etc/neutron/plugin.ini /tmp/out.yml
-
-- name: Fetch output
-  fetch: src=/tmp/out.yml dest=/tmp/out.yml flat=yes
-
-- name: Load configuration variables
-  include_vars: /tmp/out.yml
diff --git a/ansible/gather/roles/nova/tasks/main.yml b/ansible/gather/roles/nova/tasks/main.yml
index 1b09f5846..1611bdcd5 100644
--- a/ansible/gather/roles/nova/tasks/main.yml
+++ b/ansible/gather/roles/nova/tasks/main.yml
@@ -10,7 +10,7 @@
 
 - name: Parse Nova config
   become: true
-  shell: python /tmp/openstack-config-parser.py nova /etc/nova/nova.conf /tmp/out.yml
+  shell: python /tmp/openstack-config-parser.py nova /tmp/out.yml
   when: nova_config.stat.exists
 
 - name: Fetch output
diff --git a/ansible/gather/roles/rabbitmq/tasks/main.yml b/ansible/gather/roles/rabbitmq/tasks/main.yml
index 5d1e26141..25dec670f 100644
--- a/ansible/gather/roles/rabbitmq/tasks/main.yml
+++ b/ansible/gather/roles/rabbitmq/tasks/main.yml
@@ -6,9 +6,20 @@
     shell: rabbitmqctl status | grep file_descriptors | awk -F',' '{print $3}' | sed 's/.$//'
     register: rabbitmq_desc
     ignore_errors: true
+    when: hostvars[inventory_hostname]['containers'] is not defined
+
+  - name : Get rabbitmq file descriptors - containers
+    shell: docker exec rabbitmq rabbitmqctl status | grep total_limit | awk -F',' '{print $2}'| sed 's/.$//'
+    register: rabbitmq_desc_container
+    ignore_errors: true
+    when: hostvars[inventory_hostname]['containers'] is defined
 
   - name: Set rabbitmq file descriptors
     set_fact:
       openstack_rabbitmq_file_descriptors: "{{ rabbitmq_desc.stdout }}"
+    when: hostvars[inventory_hostname]['containers'] is not defined
 
-
+  - name: Set rabbitmq file descriptors - containers
+    set_fact:
+      openstack_rabbitmq_file_descriptors: "{{ rabbitmq_desc_container.stdout }}"
+    when: hostvars[inventory_hostname]['containers'] is defined
diff --git a/ansible/gather/roles/undercloud/tasks/main.yml b/ansible/gather/roles/undercloud/tasks/main.yml
index 194c787e8..e55bce228 100644
--- a/ansible/gather/roles/undercloud/tasks/main.yml
+++ b/ansible/gather/roles/undercloud/tasks/main.yml
@@ -11,7 +11,7 @@
 
   - name: Undercloud.conf
     become: true
-    shell: python /tmp/openstack-config-parser.py undercloud /home/stack/undercloud.conf /tmp/out.yml
+    shell: python /tmp/openstack-config-parser.py undercloud /tmp/out.yml
     when: undercloud_conf.stat.exists
 
   - name: Fetch output
diff --git a/ansible/gather/site.yml b/ansible/gather/site.yml
index 07ed43296..e384dd9fb 100644
--- a/ansible/gather/site.yml
+++ b/ansible/gather/site.yml
@@ -3,6 +3,7 @@
   remote_user: "{{ host_remote_user }}"
   become: true
   roles:
+    - common
     - compute
 
 - hosts: controller