Change filebeat to only execute on hosts

Filebeat has everything it needs to function on the physical host
machines. This change ensures that filebeat is only ever installed and
setup on the physical hosts which will save a lot of IO, and greatly
shorten playbook runtimes.

Change-Id: I5dd9d3ac9dc6871e14568c1591f3de475cba33e2
Signed-off-by: Kevin Carter <kevin.carter@rackspace.com>

elk_metrics_6x/installFilebeat.yml

@@ -1,6 +1,6 @@
 ---
 - name: Install Filebeat
-  hosts: all
+  hosts: hosts
   become: true
   vars:
     haproxy_ssl: false
@@ -112,18 +112,19 @@
         apache_enabled: "{{ (apache2.stat.exists | bool) or (httpd.stat.exists | bool) }}"
         nginx_enabled: "{{ nginx.stat.exists | bool }}"
         auditd_enabled: "{{ audit.stat.exists | bool }}"
-        mysql_enabled: "{{ mysql.stat.exists | bool }}"
+        mysql_enabled: "{{ (mysql.stat.exists | bool) or (inventory_hostname in groups['galera_all'] | default([])) }}"
-        cinder_enabled: "{{ cinder.stat.exists | bool }}"
+        ceph_enabled: "{{ (ceph.stat.exists | bool) or (inventory_hostname in groups['ceph_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*ceph.*') | list | length) > 0) }}"
-        glance_enabled: "{{ glance.stat.exists | bool }}"
+        cinder_enabled: "{{ (cinder.stat.exists | bool) or (inventory_hostname in groups['cinder_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*cinder.*') | list | length) > 0) }}"
-        heat_enabled: "{{ heat.stat.exists | bool }}"
+        glance_enabled: "{{ (glance.stat.exists | bool) or (inventory_hostname in groups['glance_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*glance.*') | list | length) > 0) }}"
-        horizon_enabled: "{{ horizon.stat.exists | bool }}"
+        heat_enabled: "{{ (heat.stat.exists | bool) or (inventory_hostname in groups['heat_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*heat.*') | list | length) > 0) }}"
-        keystone_enabled: "{{ keystone.stat.exists | bool }}"
+        horizon_enabled: "{{ (horizon.stat.exists | bool) or (inventory_hostname in groups['horizon_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*horizon.*') | list | length) > 0) }}"
-        neutron_enabled: "{{ neutron.stat.exists | bool }}"
+        keystone_enabled: "{{ (keystone.stat.exists | bool) or (inventory_hostname in groups['keystone_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*keystone.*') | list | length) > 0) }}"
-        nova_enabled: "{{ nova.stat.exists | bool }}"
+        neutron_enabled: "{{ (neutron.stat.exists | bool) or (inventory_hostname in groups['neutron_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*neutron.*') | list | length) > 0) }}"
-        octavia_enabled: "{{ octavia.stat.exists | bool }}"
+        nova_enabled: "{{ (nova.stat.exists | bool) or (inventory_hostname in groups['nova_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*nova.*') | list | length) > 0) }}"
-        swift_enabled: "{{ swift.stat.exists | bool }}"
+        octavia_enabled: "{{ (octavia.stat.exists | bool) or (inventory_hostname in groups['octavia_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*octavia.*') | list | length) > 0) }}"
-        rabbitmq_enabled: "{{ rabbitmq.stat.exists | bool }}"
+        swift_enabled: "{{ (swift.stat.exists | bool) or (inventory_hostname in groups['swift_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*swift.*') | list | length) > 0) }}"
-        ceph_enabled: "{{ ceph.stat.exists | bool }}"
+        rabbitmq_enabled: "{{ (rabbitmq.stat.exists | bool) or (inventory_hostname in groups['rabbitmq_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*rabbit.*') | list | length) > 0) }}"
+
 
   post_tasks:
     - name: Drop Filebeat conf file
@@ -139,7 +140,7 @@
 
 
 - name: Load Filebeat Dashboards
-  hosts: all[0]
+  hosts: hosts[0]
   become: true
   vars_files:
     - vars/variables.yml

elk_metrics_6x/templates/filebeat.yml.j2

@@ -297,6 +297,7 @@ filebeat.prospectors:
   # Make sure not file is defined twice as this can lead to unexpected behaviour.
   paths:
     - /var/log/cinder/*.log
+    - /openstack/log/*cinder*/*.log
 
   ### Multiline options
 
@@ -326,6 +327,8 @@ filebeat.prospectors:
     - openstack
     - cinder
 
+  symlinks: false
+
 - type: log
 
   # Change to true to enable this prospector configuration.
@@ -338,6 +341,7 @@ filebeat.prospectors:
   # Make sure not file is defined twice as this can lead to unexpected behaviour.
   paths:
     - /var/log/glance/*.log
+    - /openstack/log/*glance*/*.log
 
   ### Multiline options
 
@@ -367,6 +371,8 @@ filebeat.prospectors:
     - openstack
     - glance
 
+  symlinks: false
+
 - type: log
 
   # Change to true to enable this prospector configuration.
@@ -379,6 +385,7 @@ filebeat.prospectors:
   # Make sure not file is defined twice as this can lead to unexpected behaviour.
   paths:
     - /var/log/heat/*.log
+    - /openstack/log/*heat*/*.log
 
   ### Multiline options
 
@@ -408,6 +415,8 @@ filebeat.prospectors:
     - openstack
     - heat
 
+  symlinks: false
+
 - type: log
 
   # Change to true to enable this prospector configuration.
@@ -420,6 +429,7 @@ filebeat.prospectors:
   # Make sure not file is defined twice as this can lead to unexpected behaviour.
   paths:
     - /var/log/horizon/*.log
+    - /openstack/log/*horizon*/*.log
 
   ### Multiline options
 
@@ -449,6 +459,8 @@ filebeat.prospectors:
     - openstack
     - horizon
 
+  symlinks: false
+
 - type: log
 
   # Change to true to enable this prospector configuration.
@@ -461,6 +473,7 @@ filebeat.prospectors:
   # Make sure not file is defined twice as this can lead to unexpected behaviour.
   paths:
     - /var/log/keystone/*.log
+    - /openstack/log/*keystone*/*.log
 
   ### Multiline options
 
@@ -490,6 +503,8 @@ filebeat.prospectors:
     - openstack
     - keystone
 
+  symlinks: false
+
 - type: log
 
   # Change to true to enable this prospector configuration.
@@ -502,6 +517,7 @@ filebeat.prospectors:
   # Make sure not file is defined twice as this can lead to unexpected behaviour.
   paths:
     - /var/log/neutron/*.log
+    - /openstack/log/*neutron*/*.log
 
   ### Multiline options
 
@@ -531,6 +547,8 @@ filebeat.prospectors:
     - openstack
     - neutron
 
+  symlinks: false
+
 - type: log
 
   # Change to true to enable this prospector configuration.
@@ -543,6 +561,7 @@ filebeat.prospectors:
   # Make sure not file is defined twice as this can lead to unexpected behaviour.
   paths:
     - /var/log/nova/*.log
+    - /openstack/log/*nova*/*.log
 
   ### Multiline options
 
@@ -572,6 +591,8 @@ filebeat.prospectors:
     - openstack
     - nova
 
+  symlinks: false
+
 - type: log
 
   # Change to true to enable this prospector configuration.
@@ -584,6 +605,7 @@ filebeat.prospectors:
   # Make sure not file is defined twice as this can lead to unexpected behaviour.
   paths:
     - /var/log/octavia/*.log
+    - /openstack/log/*octavia*/*.log
 
   ### Multiline options
 
@@ -613,6 +635,8 @@ filebeat.prospectors:
     - openstack
     - octavia
 
+  symlinks: false
+
 - type: log
 
   # Change to true to enable this prospector configuration.
@@ -625,6 +649,7 @@ filebeat.prospectors:
   # Make sure not file is defined twice as this can lead to unexpected behaviour.
   paths:
     - /var/log/swift/account*.log
+    - /openstack/log/*swift*/account*.log
 
   ### Multiline options
 
@@ -655,6 +680,8 @@ filebeat.prospectors:
     - swift
     - swift-account
 
+  symlinks: false
+
 - type: log
 
   # Change to true to enable this prospector configuration.
@@ -667,6 +694,7 @@ filebeat.prospectors:
   # Make sure not file is defined twice as this can lead to unexpected behaviour.
   paths:
     - /var/log/swift/container*.log
+    - /openstack/log/*swift*/container*.log
 
   ### Multiline options
 
@@ -697,6 +725,8 @@ filebeat.prospectors:
     - swift
     - swift-container
 
+  symlinks: false
+
 - type: log
 
   # Change to true to enable this prospector configuration.
@@ -709,6 +739,7 @@ filebeat.prospectors:
   # Make sure not file is defined twice as this can lead to unexpected behaviour.
   paths:
     - /var/log/swift/object*.log
+    - /openstack/log/*swift*/object*.log
 
   ### Multiline options
 
@@ -739,6 +770,8 @@ filebeat.prospectors:
     - swift
     - swift-object
 
+  symlinks: false
+
 - type: log
 
   # Change to true to enable this prospector configuration.
@@ -751,6 +784,7 @@ filebeat.prospectors:
   # Make sure not file is defined twice as this can lead to unexpected behaviour.
   paths:
     - /var/log/swift/proxy*.log
+    - /openstack/log/*swift*/proxy*.log
 
   # Optional additional fields. These fields can be freely picked
   # to add additional information to the crawled log files for filtering
@@ -759,6 +793,8 @@ filebeat.prospectors:
     - swift
     - swift-proxy
 
+  symlinks: false
+
 - type: log
 
   # Change to true to enable this prospector configuration.
@@ -770,7 +806,8 @@ filebeat.prospectors:
   # For each file found under this path, a harvester is started.
   # Make sure not file is defined twice as this can lead to unexpected behaviour.
   paths:
-    - /var/log/rabbitmq/*
+    - /var/log/rabbit*/*
+    - /openstack/log/*rabbit*/*
 
   ### Multiline options
 
@@ -799,6 +836,8 @@ filebeat.prospectors:
     - rabbitmq
     - infrastructure
 
+  symlinks: false
+
 - type: log
 
   # Change to true to enable this prospector configuration.
@@ -811,6 +850,7 @@ filebeat.prospectors:
   # Make sure not file is defined twice as this can lead to unexpected behaviour.
   paths:
     - /var/log/ceph/ceph-mon.*.log
+    - /openstack/log/*ceph*/ceph-mon*.log
 
   ### Multiline options
 
@@ -840,6 +880,8 @@ filebeat.prospectors:
     - ceph
     - infrastructure
 
+  symlinks: false
+
 - type: log
 
   # Change to true to enable this prospector configuration.
@@ -852,6 +894,7 @@ filebeat.prospectors:
   # Make sure not file is defined twice as this can lead to unexpected behaviour.
   paths:
     - /var/log/ceph/ceph-osd.*.log
+    - /openstack/log/*ceph*/ceph-osd*.log
 
   # Optional additional fields. These fields can be freely picked
   # to add additional information to the crawled log files for filtering
@@ -860,6 +903,8 @@ filebeat.prospectors:
     - ceph
     - infrastructure
 
+  symlinks: false
+
 - type: log
 
   # Change to true to enable this prospector configuration.
@@ -876,6 +921,7 @@ filebeat.prospectors:
     - /var/log/libvirt/*.log
     - /var/log/libvirt/*/*.log
     - /var/log/lxc/*.log
+    - /openstack/log/ansible-logging/*.log
     #- c:\programdata\elasticsearch\logs\*
 
   # Configure the file encoding for reading files with international characters
@@ -993,7 +1039,7 @@ filebeat.prospectors:
 
   # If symlinks is enabled, symlinks are opened and harvested. The harvester is openening the
   # original for harvesting but will report the symlink name as source.
-  #symlinks: false
+  symlinks: false
 
   # Backoff values define how aggressively filebeat crawls new files for updates
   # The default values can be used in most cases. Backoff defines how long it is waited