Merge "Update filebeat"

elk_metrics_6x/templates/_include_beat_logging.yml.j2

@@ -0,0 +1,49 @@
+# There are three options for the log output: syslog, file, stderr.
+# Under Windows systems, the log files are per default sent to the file output,
+# under all other system per default to syslog.
+
+# Sets log level. The default log level is info.
+# Available log levels are: critical, error, warning, info, debug
+#logging.level: info
+
+# Enable debug output for selected components. To enable all selectors use ["*"]
+# Other available selectors are "beat", "publish", "service"
+# Multiple selectors can be chained.
+#logging.selectors: [ ]
+
+# Send all logging output to syslog. The default is false.
+#logging.to_syslog: true
+
+# If enabled, apm-server periodically logs its internal metrics that have changed
+# in the last period. For each metric that changed, the delta from the value at
+# the beginning of the period is logged. Also, the total values for
+# all non-zero internal metrics are logged on shutdown. The default is true.
+#logging.metrics.enabled: true
+
+# The period after which to log the internal metrics. The default is 30s.
+#logging.metrics.period: 30s
+
+# Logging to rotating files. Set logging.to_files to false to disable logging to
+# files.
+logging.to_files: true
+logging.files:
+  # Configure the path where the logs are written. The default is the logs directory
+  # under the home path (the binary location).
+  path: /var/log/beats
+
+  # The name of the files where the logs are written to.
+  name: {{ beat_name }}.log
+
+  # Configure log file size limit. If limit is reached, log file will be
+  # automatically rotated
+  #rotateeverybytes: 10485760 # = 10MB
+
+  # Number of rotated log files to keep. Oldest files will be deleted first.
+  keepfiles: 2
+
+  # The permissions mask to apply when rotating log files. The default value is 0600.
+  # Must be a valid Unix-style file permissions mask expressed in octal notation.
+  #permissions: 0600
+
+# Set to true to log messages in json format.
+#logging.json: false

elk_metrics_6x/templates/apm-server.yml.j2

@@ -188,52 +188,6 @@ setup.template.overwrite: true
 {% include 'templates/_include_kibana_setup.yml.j2' %}
 
 #================================ Logging ======================================
-# There are three options for the log output: syslog, file, stderr.
-# Under Windows systems, the log files are per default sent to the file output,
-# under all other system per default to syslog.
-
-# Sets log level. The default log level is info.
-# Available log levels are: critical, error, warning, info, debug
-#logging.level: info
-
-# Enable debug output for selected components. To enable all selectors use ["*"]
-# Other available selectors are "beat", "publish", "service"
-# Multiple selectors can be chained.
-#logging.selectors: [ ]
-
-# Send all logging output to syslog. The default is false.
-#logging.to_syslog: true
-
-# If enabled, apm-server periodically logs its internal metrics that have changed
-# in the last period. For each metric that changed, the delta from the value at
-# the beginning of the period is logged. Also, the total values for
-# all non-zero internal metrics are logged on shutdown. The default is true.
-#logging.metrics.enabled: true
-
-# The period after which to log the internal metrics. The default is 30s.
-#logging.metrics.period: 30s
-
-# Logging to rotating files. Set logging.to_files to false to disable logging to
-# files.
-logging.to_files: true
-logging.files:
-  # Configure the path where the logs are written. The default is the logs directory
-  # under the home path (the binary location).
-  path: /var/log/apm-server
-
-  # The name of the files where the logs are written to.
-  name: apm-server
-
-  # Configure log file size limit. If limit is reached, log file will be
-  # automatically rotated
-  #rotateeverybytes: 10485760 # = 10MB
-
-  # Number of rotated log files to keep. Oldest files will be deleted first.
-  keepfiles: 2
-
-  # The permissions mask to apply when rotating log files. The default value is 0600.
-  # Must be a valid Unix-style file permissions mask expressed in octal notation.
-  #permissions: 0600
-
-# Set to true to log messages in json format.
-#logging.json: false
+{% with beat_name="apm-server" %}
+{%   include 'templates/_include_beat_logging.yml.j2' %}
+{% endwith %}

elk_metrics_6x/templates/auditbeat.yml.j2

@@ -730,58 +730,9 @@ setup.template.overwrite: true
 {% include 'templates/_include_kibana_setup.yml.j2' %}
 
 #================================ Logging ======================================
-# There are four options for the log output: file, stderr, syslog, eventlog
-# The file output is the default.
-
-# Sets log level. The default log level is info.
-# Available log levels are: error, warning, info, debug
-#logging.level: info
-
-# Enable debug output for selected components. To enable all selectors use ["*"]
-# Other available selectors are "beat", "publish", "service"
-# Multiple selectors can be chained.
-#logging.selectors: [ ]
-
-# Send all logging output to syslog. The default is false.
-#logging.to_syslog: false
-
-# Send all logging output to Windows Event Logs. The default is false.
-#logging.to_eventlog: false
-
-# If enabled, auditbeat periodically logs its internal metrics that have changed
-# in the last period. For each metric that changed, the delta from the value at
-# the beginning of the period is logged. Also, the total values for
-# all non-zero internal metrics are logged on shutdown. The default is true.
-#logging.metrics.enabled: true
-
-# The period after which to log the internal metrics. The default is 30s.
-#logging.metrics.period: 30s
-
-# Logging to rotating files. Set logging.to_files to false to disable logging to
-# files.
-logging.to_files: true
-logging.files:
-  # Configure the path where the logs are written. The default is the logs directory
-  # under the home path (the binary location).
-  path: /var/log/auditbeat
-
-  # The name of the files where the logs are written to.
-  name: auditbeat
-
-  # Configure log file size limit. If limit is reached, log file will be
-  # automatically rotated
-  #rotateeverybytes: 10485760 # = 10MB
-
-  # Number of rotated log files to keep. Oldest files will be deleted first.
-  keepfiles: 2
-
-  # The permissions mask to apply when rotating log files. The default value is 0600.
-  # Must be a valid Unix-style file permissions mask expressed in octal notation.
-  #permissions: 0600
-
-# Set to true to log messages in json format.
-#logging.json: false
-
+{% with beat_name="auditbeat" %}
+{%   include 'templates/_include_beat_logging.yml.j2' %}
+{% endwith %}
 
 #============================== Xpack Monitoring =====================================
 # auditbeat can export internal metrics to a central Elasticsearch monitoring cluster.

elk_metrics_6x/templates/filebeat.yml.j2

@@ -120,6 +120,32 @@ filebeat.modules:
     # can be added under this section.
     #prospector:
 
+#--------------------------------- IIS Module --------------------------------
+#- module: iis
+  # Access logs
+  #access:
+    #enabled: true
+
+    # Set custom paths for the log files. If left empty,
+    # Filebeat will choose the paths depending on your OS.
+    #var.paths:
+
+    # Prospector configuration (advanced). Any prospector configuration option
+    # can be added under this section.
+    #prospector:
+
+  # Error logs
+  #error:
+    #enabled: true
+
+    # Set custom paths for the log files. If left empty,
+    # Filebeat will choose the paths depending on your OS.
+    #var.paths:
+
+    # Prospector configuration (advanced). Any prospector configuration option
+    # can be added under this section.
+    #prospector:
+
 #-------------------------------- Kafka Module -------------------------------
 #- module: kafka
   # All logs
@@ -136,22 +162,36 @@ filebeat.modules:
 
 
 #------------------------------ logstash Module ------------------------------
-#- module: logstash
+- module: logstash
   # logs
-  #log:
-    #enabled: true
+  log:
+    enabled: true
 
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.
     # var.paths:
 
   # Slow logs
-  #slowlog:
-   #enabled: true
+  slowlog:
+    enabled: true
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
 
+#------------------------------- mongodb Module ------------------------------
+#- module: mongodb
+  # Logs
+  #log:
+    #enabled: true
+
+    # Set custom paths for the log files. If left empty,
+    # Filebeat will choose the paths depending on your OS.
+    #var.paths:
+
+    # Input configuration (advanced). Any input configuration option
+    # can be added under this section.
+    #input:
+
 #-------------------------------- MySQL Module -------------------------------
 - module: mysql
   # Error logs
@@ -169,8 +209,8 @@ filebeat.modules:
     #prospector:
 
   # Slow logs
-  #slowlog:
-    #enabled: true
+  slowlog:
+    enabled: {{ mysql_enabled | bool }}
 
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.
@@ -959,6 +999,29 @@ filebeat.prospectors:
 
   symlinks: false
 
+- type: log
+
+  # Change to true to enable this prospector configuration.
+  enabled: true
+
+  # Paths that should be crawled and fetched. Glob based paths.
+  # To fetch all ".log" files from a specific level of subdirectories
+  # /var/log/*/*.log can be used.
+  # For each file found under this path, a harvester is started.
+  # Make sure not file is defined twice as this can lead to unexpected behaviour.
+  paths:
+    - /var/log/beats/*.log
+    - /var/log/curator/curator
+    - /var/log/elasticsearch/*.log
+
+  # Optional additional fields. These fields can be freely picked
+  # to add additional information to the crawled log files for filtering
+  tags:
+    - beats
+
+  symlinks: false
+
+
 - type: log
 
   # Change to true to enable this prospector configuration.
@@ -1824,58 +1887,9 @@ setup.template.overwrite: true
 {% include 'templates/_include_kibana_setup.yml.j2' %}
 
 #================================ Logging ======================================
-# There are four options for the log output: file, stderr, syslog, eventlog
-# The file output is the default.
-
-# Sets log level. The default log level is info.
-# Available log levels are: error, warning, info, debug
-#logging.level: info
-
-# Enable debug output for selected components. To enable all selectors use ["*"]
-# Other available selectors are "beat", "publish", "service"
-# Multiple selectors can be chained.
-#logging.selectors: [ ]
-
-# Send all logging output to syslog. The default is false.
-#logging.to_syslog: false
-
-# Send all logging output to Windows Event Logs. The default is false.
-#logging.to_eventlog: false
-
-# If enabled, filebeat periodically logs its internal metrics that have changed
-# in the last period. For each metric that changed, the delta from the value at
-# the beginning of the period is logged. Also, the total values for
-# all non-zero internal metrics are logged on shutdown. The default is true.
-#logging.metrics.enabled: true
-
-# The period after which to log the internal metrics. The default is 30s.
-#logging.metrics.period: 30s
-
-# Logging to rotating files. Set logging.to_files to false to disable logging to
-# files.
-logging.to_files: true
-logging.files:
-  # Configure the path where the logs are written. The default is the logs directory
-  # under the home path (the binary location).
-  path: /var/log/filebeat
-
-  # The name of the files where the logs are written to.
-  name: filebeat
-
-  # Configure log file size limit. If limit is reached, log file will be
-  # automatically rotated
-  #rotateeverybytes: 10485760 # = 10MB
-
-  # Number of rotated log files to keep. Oldest files will be deleted first.
-  keepfiles: 2
-
-  # The permissions mask to apply when rotating log files. The default value is 0600.
-  # Must be a valid Unix-style file permissions mask expressed in octal notation.
-  #permissions: 0600
-
-# Set to true to log messages in json format.
-#logging.json: false
-
+{% with beat_name="filebeat" %}
+{%   include 'templates/_include_beat_logging.yml.j2' %}
+{% endwith %}
 
 #============================== Xpack Monitoring =====================================
 # filebeat can export internal metrics to a central Elasticsearch monitoring cluster.

elk_metrics_6x/templates/heartbeat.yml.j2

@@ -834,58 +834,9 @@ setup.template.overwrite: true
 {% include 'templates/_include_kibana_setup.yml.j2' %}
 
 #================================ Logging ======================================
-# There are four options for the log output: file, stderr, syslog, eventlog
-# The file output is the default.
-
-# Sets log level. The default log level is info.
-# Available log levels are: error, warning, info, debug
-#logging.level: info
-
-# Enable debug output for selected components. To enable all selectors use ["*"]
-# Other available selectors are "beat", "publish", "service"
-# Multiple selectors can be chained.
-#logging.selectors: [ ]
-
-# Send all logging output to syslog. The default is false.
-#logging.to_syslog: false
-
-# Send all logging output to Windows Event Logs. The default is false.
-#logging.to_eventlog: false
-
-# If enabled, heartbeat periodically logs its internal metrics that have changed
-# in the last period. For each metric that changed, the delta from the value at
-# the beginning of the period is logged. Also, the total values for
-# all non-zero internal metrics are logged on shutdown. The default is true.
-#logging.metrics.enabled: true
-
-# The period after which to log the internal metrics. The default is 30s.
-#logging.metrics.period: 30s
-
-# Logging to rotating files. Set logging.to_files to false to disable logging to
-# files.
-logging.to_files: true
-logging.files:
-  # Configure the path where the logs are written. The default is the logs directory
-  # under the home path (the binary location).
-  path: /var/log/heartbeat
-
-  # The name of the files where the logs are written to.
-  name: heartbeat
-
-  # Configure log file size limit. If limit is reached, log file will be
-  # automatically rotated
-  #rotateeverybytes: 10485760 # = 10MB
-
-  # Number of rotated log files to keep. Oldest files will be deleted first.
-  keepfiles: 2
-
-  # The permissions mask to apply when rotating log files. The default value is 0600.
-  # Must be a valid Unix-style file permissions mask expressed in octal notation.
-  #permissions: 0600
-
-# Set to true to log messages in json format.
-#logging.json: false
-
+{% with beat_name="heartbeat" %}
+{%   include 'templates/_include_beat_logging.yml.j2' %}
+{% endwith %}
 
 #============================== Xpack Monitoring =====================================
 # heartbeat can export internal metrics to a central Elasticsearch monitoring cluster.

elk_metrics_6x/templates/journalbeat.yml.j2

@@ -640,54 +640,6 @@ setup.template.overwrite: true
 {% include 'templates/_include_kibana_setup.yml.j2' %}
 
 #================================ Logging ======================================
-# There are four options for the log output: file, stderr, syslog, eventlog
-# The file output is the default.
-
-# Sets log level. The default log level is info.
-# Available log levels are: error, warning, info, debug
-#logging.level: info
-
-# Enable debug output for selected components. To enable all selectors use ["*"]
-# Other available selectors are "beat", "publish", "service"
-# Multiple selectors can be chained.
-#logging.selectors: [ ]
-
-# Send all logging output to syslog. The default is false.
-#logging.to_syslog: false
-
-# Send all logging output to Windows Event Logs. The default is false.
-#logging.to_eventlog: false
-
-# If enabled, journalbeat periodically logs its internal metrics that have changed
-# in the last period. For each metric that changed, the delta from the value at
-# the beginning of the period is logged. Also, the total values for
-# all non-zero internal metrics are logged on shutdown. The default is true.
-#logging.metrics.enabled: true
-
-# The period after which to log the internal metrics. The default is 30s.
-#logging.metrics.period: 30s
-
-# Logging to rotating files. Set logging.to_files to false to disable logging to
-# files.
-logging.to_files: true
-logging.files:
-  # Configure the path where the logs are written. The default is the logs directory
-  # under the home path (the binary location).
-  path: /var/log/journalbeat
-
-  # The name of the files where the logs are written to.
-  name: journalbeat
-
-  # Configure log file size limit. If limit is reached, log file will be
-  # automatically rotated
-  #rotateeverybytes: 10485760 # = 10MB
-
-  # Number of rotated log files to keep. Oldest files will be deleted first.
-  keepfiles: 2
-
-  # The permissions mask to apply when rotating log files. The default value is 0600.
-  # Must be a valid Unix-style file permissions mask expressed in octal notation.
-  #permissions: 0600
-
-# Set to true to log messages in json format.
-#logging.json: false
+{% with beat_name="journalbeat" %}
+{%   include 'templates/_include_beat_logging.yml.j2' %}
+{% endwith %}

elk_metrics_6x/templates/metricbeat.yml.j2

@@ -1118,58 +1118,9 @@ setup.template.overwrite: true
 {% include 'templates/_include_kibana_setup.yml.j2' %}
 
 #================================ Logging ======================================
-# There are four options for the log output: file, stderr, syslog, eventlog
-# The file output is the default.
-
-# Sets log level. The default log level is info.
-# Available log levels are: error, warning, info, debug
-#logging.level: info
-
-# Enable debug output for selected components. To enable all selectors use ["*"]
-# Other available selectors are "beat", "publish", "service"
-# Multiple selectors can be chained.
-#logging.selectors: [ ]
-
-# Send all logging output to syslog. The default is false.
-#logging.to_syslog: false
-
-# Send all logging output to Windows Event Logs. The default is false.
-#logging.to_eventlog: false
-
-# If enabled, metricbeat periodically logs its internal metrics that have changed
-# in the last period. For each metric that changed, the delta from the value at
-# the beginning of the period is logged. Also, the total values for
-# all non-zero internal metrics are logged on shutdown. The default is true.
-#logging.metrics.enabled: true
-
-# The period after which to log the internal metrics. The default is 30s.
-#logging.metrics.period: 30s
-
-# Logging to rotating files. Set logging.to_files to false to disable logging to
-# files.
-logging.to_files: true
-logging.files:
-  # Configure the path where the logs are written. The default is the logs directory
-  # under the home path (the binary location).
-  path: /var/log/metricbeat
-
-  # The name of the files where the logs are written to.
-  name: metricbeat
-
-  # Configure log file size limit. If limit is reached, log file will be
-  # automatically rotated
-  #rotateeverybytes: 10485760 # = 10MB
-
-  # Number of rotated log files to keep. Oldest files will be deleted first.
-  keepfiles: 2
-
-  # The permissions mask to apply when rotating log files. The default value is 0600.
-  # Must be a valid Unix-style file permissions mask expressed in octal notation.
-  #permissions: 0600
-
-# Set to true to log messages in json format.
-#logging.json: false
-
+{% with beat_name="metricbeat" %}
+{%   include 'templates/_include_beat_logging.yml.j2' %}
+{% endwith %}
 
 #============================== Xpack Monitoring =====================================
 # metricbeat can export internal metrics to a central Elasticsearch monitoring cluster.

elk_metrics_6x/templates/packetbeat.yml.j2

@@ -1110,58 +1110,9 @@ setup.template.overwrite: true
 {% include 'templates/_include_kibana_setup.yml.j2' %}
 
 #================================ Logging ======================================
-# There are four options for the log output: file, stderr, syslog, eventlog
-# The file output is the default.
-
-# Sets log level. The default log level is info.
-# Available log levels are: error, warning, info, debug
-#logging.level: info
-
-# Enable debug output for selected components. To enable all selectors use ["*"]
-# Other available selectors are "beat", "publish", "service"
-# Multiple selectors can be chained.
-#logging.selectors: [ ]
-
-# Send all logging output to syslog. The default is false.
-#logging.to_syslog: false
-
-# Send all logging output to Windows Event Logs. The default is false.
-#logging.to_eventlog: false
-
-# If enabled, packetbeat periodically logs its internal metrics that have changed
-# in the last period. For each metric that changed, the delta from the value at
-# the beginning of the period is logged. Also, the total values for
-# all non-zero internal metrics are logged on shutdown. The default is true.
-#logging.metrics.enabled: true
-
-# The period after which to log the internal metrics. The default is 30s.
-#logging.metrics.period: 30s
-
-# Logging to rotating files. Set logging.to_files to false to disable logging to
-# files.
-logging.to_files: true
-logging.files:
-  # Configure the path where the logs are written. The default is the logs directory
-  # under the home path (the binary location).
-  path: /var/log/packetbeat
-
-  # The name of the files where the logs are written to.
-  name: packetbeat
-
-  # Configure log file size limit. If limit is reached, log file will be
-  # automatically rotated
-  #rotateeverybytes: 10485760 # = 10MB
-
-  # Number of rotated log files to keep. Oldest files will be deleted first.
-  keepfiles: 2
-
-  # The permissions mask to apply when rotating log files. The default value is 0600.
-  # Must be a valid Unix-style file permissions mask expressed in octal notation.
-  #permissions: 0600
-
-# Set to true to log messages in json format.
-#logging.json: false
-
+{% with beat_name="packetbeat" %}
+{%   include 'templates/_include_beat_logging.yml.j2' %}
+{% endwith %}
 
 #============================== Xpack Monitoring =====================================
 # packetbeat can export internal metrics to a central Elasticsearch monitoring cluster.