openstack/openstack-ansible-ops
Code Issues Proposed changes

Change filebeat to only execute on hosts

Browse Source 
Filebeat has everything it needs to function on the physical host
machines. This change ensures that filebeat is only ever installed and
setup on the physical hosts which will save a lot of IO, and greatly
shorten playbook runtimes.

Change-Id: I5dd9d3ac9dc6871e14568c1591f3de475cba33e2
Signed-off-by: Kevin Carter <kevin.carter@rackspace.com>
This commit is contained in:
Kevin Carter 2018-04-27 12:24:04 -05:00
parent ac286b0ac3
commit 26dd0b7a2e
No known key found for this signature in database
GPG Key ID: 9443251A787B9FB3
2 changed files with 63 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
elk_metrics_6x/installFilebeat.yml
View File

@ -1,6 +1,6 @@
---
- name: Install Filebeat
hosts: all
hosts: hosts
become: true
vars:
haproxy_ssl: false
@ -112,18 +112,19 @@
apache_enabled: "{{ (apache2.stat.exists | bool) or (httpd.stat.exists | bool) }}"
nginx_enabled: "{{ nginx.stat.exists | bool }}"
auditd_enabled: "{{ audit.stat.exists | bool }}"
mysql_enabled: "{{ mysql.stat.exists | bool }}"
cinder_enabled: "{{ cinder.stat.exists | bool }}"
glance_enabled: "{{ glance.stat.exists | bool }}"
heat_enabled: "{{ heat.stat.exists | bool }}"
horizon_enabled: "{{ horizon.stat.exists | bool }}"
keystone_enabled: "{{ keystone.stat.exists | bool }}"
neutron_enabled: "{{ neutron.stat.exists | bool }}"
nova_enabled: "{{ nova.stat.exists | bool }}"
octavia_enabled: "{{ octavia.stat.exists | bool }}"
swift_enabled: "{{ swift.stat.exists | bool }}"
rabbitmq_enabled: "{{ rabbitmq.stat.exists | bool }}"
ceph_enabled: "{{ ceph.stat.exists | bool }}"
mysql_enabled: "{{ (mysql.stat.exists | bool) or (inventory_hostname in groups['galera_all'] | default([])) }}"
ceph_enabled: "{{ (ceph.stat.exists | bool) or (inventory_hostname in groups['ceph_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*ceph.*') | list | length) > 0) }}"
cinder_enabled: "{{ (cinder.stat.exists | bool) or (inventory_hostname in groups['cinder_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*cinder.*') | list | length) > 0) }}"
glance_enabled: "{{ (glance.stat.exists | bool) or (inventory_hostname in groups['glance_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*glance.*') | list | length) > 0) }}"
heat_enabled: "{{ (heat.stat.exists | bool) or (inventory_hostname in groups['heat_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*heat.*') | list | length) > 0) }}"
horizon_enabled: "{{ (horizon.stat.exists | bool) or (inventory_hostname in groups['horizon_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*horizon.*') | list | length) > 0) }}"
keystone_enabled: "{{ (keystone.stat.exists | bool) or (inventory_hostname in groups['keystone_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*keystone.*') | list | length) > 0) }}"
neutron_enabled: "{{ (neutron.stat.exists | bool) or (inventory_hostname in groups['neutron_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*neutron.*') | list | length) > 0) }}"
nova_enabled: "{{ (nova.stat.exists | bool) or (inventory_hostname in groups['nova_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*nova.*') | list | length) > 0) }}"
octavia_enabled: "{{ (octavia.stat.exists | bool) or (inventory_hostname in groups['octavia_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*octavia.*') | list | length) > 0) }}"
swift_enabled: "{{ (swift.stat.exists | bool) or (inventory_hostname in groups['swift_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*swift.*') | list | length) > 0) }}"
rabbitmq_enabled: "{{ (rabbitmq.stat.exists | bool) or (inventory_hostname in groups['rabbitmq_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*rabbit.*') | list | length) > 0) }}"
post_tasks:
- name: Drop Filebeat conf file
@ -139,7 +140,7 @@
- name: Load Filebeat Dashboards
hosts: all[0]
hosts: hosts[0]
become: true
vars_files:
- vars/variables.yml

50
elk_metrics_6x/templates/filebeat.yml.j2
View File

@ -297,6 +297,7 @@ filebeat.prospectors:
# Make sure not file is defined twice as this can lead to unexpected behaviour.
paths:
- /var/log/cinder/*.log
- /openstack/log/*cinder*/*.log
### Multiline options
@ -326,6 +327,8 @@ filebeat.prospectors:
- openstack
- cinder
symlinks: false
- type: log
# Change to true to enable this prospector configuration.
@ -338,6 +341,7 @@ filebeat.prospectors:
# Make sure not file is defined twice as this can lead to unexpected behaviour.
paths:
- /var/log/glance/*.log
- /openstack/log/*glance*/*.log
### Multiline options
@ -367,6 +371,8 @@ filebeat.prospectors:
- openstack
- glance
symlinks: false
- type: log
# Change to true to enable this prospector configuration.
@ -379,6 +385,7 @@ filebeat.prospectors:
# Make sure not file is defined twice as this can lead to unexpected behaviour.
paths:
- /var/log/heat/*.log
- /openstack/log/*heat*/*.log
### Multiline options
@ -408,6 +415,8 @@ filebeat.prospectors:
- openstack
- heat
symlinks: false
- type: log
# Change to true to enable this prospector configuration.
@ -420,6 +429,7 @@ filebeat.prospectors:
# Make sure not file is defined twice as this can lead to unexpected behaviour.
paths:
- /var/log/horizon/*.log
- /openstack/log/*horizon*/*.log
### Multiline options
@ -449,6 +459,8 @@ filebeat.prospectors:
- openstack
- horizon
symlinks: false
- type: log
# Change to true to enable this prospector configuration.
@ -461,6 +473,7 @@ filebeat.prospectors:
# Make sure not file is defined twice as this can lead to unexpected behaviour.
paths:
- /var/log/keystone/*.log
- /openstack/log/*keystone*/*.log
### Multiline options
@ -490,6 +503,8 @@ filebeat.prospectors:
- openstack
- keystone
symlinks: false
- type: log
# Change to true to enable this prospector configuration.
@ -502,6 +517,7 @@ filebeat.prospectors:
# Make sure not file is defined twice as this can lead to unexpected behaviour.
paths:
- /var/log/neutron/*.log
- /openstack/log/*neutron*/*.log
### Multiline options
@ -531,6 +547,8 @@ filebeat.prospectors:
- openstack
- neutron
symlinks: false
- type: log
# Change to true to enable this prospector configuration.
@ -543,6 +561,7 @@ filebeat.prospectors:
# Make sure not file is defined twice as this can lead to unexpected behaviour.
paths:
- /var/log/nova/*.log
- /openstack/log/*nova*/*.log
### Multiline options
@ -572,6 +591,8 @@ filebeat.prospectors:
- openstack
- nova
symlinks: false
- type: log
# Change to true to enable this prospector configuration.
@ -584,6 +605,7 @@ filebeat.prospectors:
# Make sure not file is defined twice as this can lead to unexpected behaviour.
paths:
- /var/log/octavia/*.log
- /openstack/log/*octavia*/*.log
### Multiline options
@ -613,6 +635,8 @@ filebeat.prospectors:
- openstack
- octavia
symlinks: false
- type: log
# Change to true to enable this prospector configuration.
@ -625,6 +649,7 @@ filebeat.prospectors:
# Make sure not file is defined twice as this can lead to unexpected behaviour.
paths:
- /var/log/swift/account*.log
- /openstack/log/*swift*/account*.log
### Multiline options
@ -655,6 +680,8 @@ filebeat.prospectors:
- swift
- swift-account
symlinks: false
- type: log
# Change to true to enable this prospector configuration.
@ -667,6 +694,7 @@ filebeat.prospectors:
# Make sure not file is defined twice as this can lead to unexpected behaviour.
paths:
- /var/log/swift/container*.log
- /openstack/log/*swift*/container*.log
### Multiline options
@ -697,6 +725,8 @@ filebeat.prospectors:
- swift
- swift-container
symlinks: false
- type: log
# Change to true to enable this prospector configuration.
@ -709,6 +739,7 @@ filebeat.prospectors:
# Make sure not file is defined twice as this can lead to unexpected behaviour.
paths:
- /var/log/swift/object*.log
- /openstack/log/*swift*/object*.log
### Multiline options
@ -739,6 +770,8 @@ filebeat.prospectors:
- swift
- swift-object
symlinks: false
- type: log
# Change to true to enable this prospector configuration.
@ -751,6 +784,7 @@ filebeat.prospectors:
# Make sure not file is defined twice as this can lead to unexpected behaviour.
paths:
- /var/log/swift/proxy*.log
- /openstack/log/*swift*/proxy*.log
# Optional additional fields. These fields can be freely picked
# to add additional information to the crawled log files for filtering
@ -759,6 +793,8 @@ filebeat.prospectors:
- swift
- swift-proxy
symlinks: false
- type: log
# Change to true to enable this prospector configuration.
@ -770,7 +806,8 @@ filebeat.prospectors:
# For each file found under this path, a harvester is started.
# Make sure not file is defined twice as this can lead to unexpected behaviour.
paths:
- /var/log/rabbitmq/*
- /var/log/rabbit*/*
- /openstack/log/*rabbit*/*
### Multiline options
@ -799,6 +836,8 @@ filebeat.prospectors:
- rabbitmq
- infrastructure
symlinks: false
- type: log
# Change to true to enable this prospector configuration.
@ -811,6 +850,7 @@ filebeat.prospectors:
# Make sure not file is defined twice as this can lead to unexpected behaviour.
paths:
- /var/log/ceph/ceph-mon.*.log
- /openstack/log/*ceph*/ceph-mon*.log
### Multiline options
@ -840,6 +880,8 @@ filebeat.prospectors:
- ceph
- infrastructure
symlinks: false
- type: log
# Change to true to enable this prospector configuration.
@ -852,6 +894,7 @@ filebeat.prospectors:
# Make sure not file is defined twice as this can lead to unexpected behaviour.
paths:
- /var/log/ceph/ceph-osd.*.log
- /openstack/log/*ceph*/ceph-osd*.log
# Optional additional fields. These fields can be freely picked
# to add additional information to the crawled log files for filtering
@ -860,6 +903,8 @@ filebeat.prospectors:
- ceph
- infrastructure
symlinks: false
- type: log
# Change to true to enable this prospector configuration.
@ -876,6 +921,7 @@ filebeat.prospectors:
- /var/log/libvirt/*.log
- /var/log/libvirt/*/*.log
- /var/log/lxc/*.log
- /openstack/log/ansible-logging/*.log
#- c:\programdata\elasticsearch\logs\*
# Configure the file encoding for reading files with international characters
@ -993,7 +1039,7 @@ filebeat.prospectors:
# If symlinks is enabled, symlinks are opened and harvested. The harvester is openening the
# original for harvesting but will report the symlink name as source.
#symlinks: false
symlinks: false
# Backoff values define how aggressively filebeat crawls new files for updates
# The default values can be used in most cases. Backoff defines how long it is waited