diff --git a/elk_metrics_6x/templates/02-general.conf b/elk_metrics_6x/files/02-general.conf similarity index 100% rename from elk_metrics_6x/templates/02-general.conf rename to elk_metrics_6x/files/02-general.conf diff --git a/elk_metrics_6x/files/02-journald.conf b/elk_metrics_6x/files/02-journald.conf new file mode 100644 index 00000000..6822f565 --- /dev/null +++ b/elk_metrics_6x/files/02-journald.conf @@ -0,0 +1,22 @@ +filter { + if "journald" in [tags] { + if [systemd_slice] { + mutate { + copy => { "systemd_slice" => "systemd_slice_tag" } + } + mutate { + gsub => [ "systemd_slice_tag", ".slice", "" ] + } + if [systemd_slice_tag] != "-" { + mutate { + add_tag => [ + "%{systemd_slice_tag}" + ] + } + } + mutate { + remove_field => [ "%{systemd_slice_tag}" ] + } + } + } +} diff --git a/elk_metrics_6x/templates/03-nova.conf b/elk_metrics_6x/files/03-nova.conf similarity index 100% rename from elk_metrics_6x/templates/03-nova.conf rename to elk_metrics_6x/files/03-nova.conf diff --git a/elk_metrics_6x/templates/04-neutron.conf b/elk_metrics_6x/files/04-neutron.conf similarity index 100% rename from elk_metrics_6x/templates/04-neutron.conf rename to elk_metrics_6x/files/04-neutron.conf diff --git a/elk_metrics_6x/templates/05-glance.conf b/elk_metrics_6x/files/05-glance.conf similarity index 100% rename from elk_metrics_6x/templates/05-glance.conf rename to elk_metrics_6x/files/05-glance.conf diff --git a/elk_metrics_6x/templates/06-cinder.conf b/elk_metrics_6x/files/06-cinder.conf similarity index 100% rename from elk_metrics_6x/templates/06-cinder.conf rename to elk_metrics_6x/files/06-cinder.conf diff --git a/elk_metrics_6x/templates/07-libvirt.conf b/elk_metrics_6x/files/07-libvirt.conf similarity index 100% rename from elk_metrics_6x/templates/07-libvirt.conf rename to elk_metrics_6x/files/07-libvirt.conf diff --git a/elk_metrics_6x/templates/08-apache.conf b/elk_metrics_6x/files/08-apache.conf similarity index 100% rename from elk_metrics_6x/templates/08-apache.conf rename to elk_metrics_6x/files/08-apache.conf diff --git a/elk_metrics_6x/templates/09-heat.conf b/elk_metrics_6x/files/09-heat.conf similarity index 100% rename from elk_metrics_6x/templates/09-heat.conf rename to elk_metrics_6x/files/09-heat.conf diff --git a/elk_metrics_6x/templates/10-mysql.conf b/elk_metrics_6x/files/10-mysql.conf similarity index 100% rename from elk_metrics_6x/templates/10-mysql.conf rename to elk_metrics_6x/files/10-mysql.conf diff --git a/elk_metrics_6x/templates/10-syslog-filter.conf.j2 b/elk_metrics_6x/files/10-syslog-filter.conf similarity index 100% rename from elk_metrics_6x/templates/10-syslog-filter.conf.j2 rename to elk_metrics_6x/files/10-syslog-filter.conf diff --git a/elk_metrics_6x/templates/11-auth.conf b/elk_metrics_6x/files/11-auth.conf similarity index 100% rename from elk_metrics_6x/templates/11-auth.conf rename to elk_metrics_6x/files/11-auth.conf diff --git a/elk_metrics_6x/templates/12-logstash.conf b/elk_metrics_6x/files/12-logstash.conf similarity index 100% rename from elk_metrics_6x/templates/12-logstash.conf rename to elk_metrics_6x/files/12-logstash.conf diff --git a/elk_metrics_6x/templates/13-swift.conf b/elk_metrics_6x/files/13-swift.conf similarity index 100% rename from elk_metrics_6x/templates/13-swift.conf rename to elk_metrics_6x/files/13-swift.conf diff --git a/elk_metrics_6x/templates/14-keystone.conf b/elk_metrics_6x/files/14-keystone.conf similarity index 100% rename from elk_metrics_6x/templates/14-keystone.conf rename to elk_metrics_6x/files/14-keystone.conf diff --git a/elk_metrics_6x/templates/16-elasticsearch.conf b/elk_metrics_6x/files/16-elasticsearch.conf similarity index 100% rename from elk_metrics_6x/templates/16-elasticsearch.conf rename to elk_metrics_6x/files/16-elasticsearch.conf diff --git a/elk_metrics_6x/templates/17-rabbitmq.conf b/elk_metrics_6x/files/17-rabbitmq.conf similarity index 100% rename from elk_metrics_6x/templates/17-rabbitmq.conf rename to elk_metrics_6x/files/17-rabbitmq.conf diff --git a/elk_metrics_6x/templates/18-ceph.conf b/elk_metrics_6x/files/18-ceph.conf similarity index 100% rename from elk_metrics_6x/templates/18-ceph.conf rename to elk_metrics_6x/files/18-ceph.conf diff --git a/elk_metrics_6x/templates/19-nginx.conf b/elk_metrics_6x/files/19-nginx.conf similarity index 100% rename from elk_metrics_6x/templates/19-nginx.conf rename to elk_metrics_6x/files/19-nginx.conf diff --git a/elk_metrics_6x/templates/20-magnum.conf b/elk_metrics_6x/files/20-magnum.conf similarity index 100% rename from elk_metrics_6x/templates/20-magnum.conf rename to elk_metrics_6x/files/20-magnum.conf diff --git a/elk_metrics_6x/templates/21-octavia.conf b/elk_metrics_6x/files/21-octavia.conf similarity index 100% rename from elk_metrics_6x/templates/21-octavia.conf rename to elk_metrics_6x/files/21-octavia.conf diff --git a/elk_metrics_6x/templates/98-traceback.conf b/elk_metrics_6x/files/98-traceback.conf similarity index 100% rename from elk_metrics_6x/templates/98-traceback.conf rename to elk_metrics_6x/files/98-traceback.conf diff --git a/elk_metrics_6x/installLogstash.yml b/elk_metrics_6x/installLogstash.yml index c6462f27..7bceec68 100644 --- a/elk_metrics_6x/installLogstash.yml +++ b/elk_metrics_6x/installLogstash.yml @@ -15,12 +15,14 @@ - name: Set quarter memory fact set_fact: q_mem: "{{ (ansible_memtotal_mb | int) // 4 }}" + q_storage: "{{ ansible_processor_cores }}" tags: - always - name: Set logstash facts set_fact: elastic_heap_size: "{{ ((q_mem | int) > 30720) | ternary(30720, q_mem) }}" + logstash_queue_size: "{{ (((q_storage | int) > 16) | ternary(16, q_storage) | int) * 1024 }}" tags: - always @@ -79,21 +81,6 @@ tags: - package_install - - name: Drop Logstash conf for beats input - template: - src: templates/02-beats-input.conf.j2 - dest: /etc/logstash/conf.d/02-beats-input.conf - - - name: Drop Logstash conf for beats input - template: - src: templates/10-syslog-filter.conf.j2 - dest: /etc/logstash/conf.d/10-syslog-filter.conf - - - name: Drop Logstash conf for elasticsearch output - template: - src: templates/99-elasticsearch-output.conf.j2 - dest: /etc/logstash/conf.d/99-elasticsearch-output.conf - - name: Drop elasticsearch conf file template: src: "{{ item.src }}" @@ -101,6 +88,14 @@ with_items: - src: templates/jvm.options.j2 dest: /etc/logstash/jvm.options + - src: templates/99-elasticsearch-output.conf.j2 + dest: /etc/logstash/conf.d/99-elasticsearch-output.conf + - src: templates/02-beats-input.conf.j2 + dest: /etc/logstash/conf.d/02-beats-input.conf + - src: templates/logstash.yml.j2 + dest: /etc/logstash/logstash.yml + notify: + - Enable and restart logstash tags: - config @@ -123,16 +118,19 @@ - extras when: - logstash_deploy_filters + notify: + - Enable and restart logstash tags: - logstash-filters - config - name: Deploy Logstash configuration files - template: - src: "{{ item }}" + copy: + src: "files/{{ item }}" dest: "/etc/logstash/conf.d/{{ item }}" with_items: - 02-general.conf + - 02-journald.conf - 03-nova.conf - 04-neutron.conf - 05-glance.conf @@ -141,6 +139,7 @@ - 08-apache.conf - 09-heat.conf - 10-mysql.conf + - 10-syslog-filter.conf - 11-auth.conf - 12-logstash.conf - 13-swift.conf @@ -154,6 +153,8 @@ - 98-traceback.conf when: - logstash_deploy_filters + notify: + - Enable and restart logstash tags: - logstash-filters - config @@ -178,6 +179,8 @@ template: src: "templates/99-kafka-output.conf" dest: "/etc/logstash/conf.d/99-kafka-output.conf" + notify: + - Enable and restart logstash when: - logstash_kafka_options is defined @@ -187,7 +190,12 @@ owner: logstash group: logstash recurse: true + register: l_perms + until: l_perms is success + retries: 3 + delay: 1 + handlers: - name: Enable and restart logstash systemd: name: "logstash" diff --git a/elk_metrics_6x/templates/journalbeat.yml.j2 b/elk_metrics_6x/templates/journalbeat.yml.j2 index 808d7cb9..d1e12ccf 100644 --- a/elk_metrics_6x/templates/journalbeat.yml.j2 +++ b/elk_metrics_6x/templates/journalbeat.yml.j2 @@ -82,7 +82,8 @@ name: journalbeat # The tags of the shipper are included in their own field with each # transaction published. Tags make it easy to group servers by different # logical properties. -#tags: ["service-X", "web-tier"] +tags: + - journald # Optional fields that you can specify to add additional information to the # output. Fields can be scalar values, arrays, dictionaries, or any nested diff --git a/elk_metrics_6x/templates/logstash.yml.j2 b/elk_metrics_6x/templates/logstash.yml.j2 new file mode 100644 index 00000000..b0c007a2 --- /dev/null +++ b/elk_metrics_6x/templates/logstash.yml.j2 @@ -0,0 +1,246 @@ +# Settings file in YAML +# +# Settings can be specified either in hierarchical form, e.g.: +# +# pipeline: +# batch: +# size: 125 +# delay: 5 +# +# Or as flat keys: +# +# pipeline.batch.size: 125 +# pipeline.batch.delay: 5 +# +# ------------ Node identity ------------ +# +# Use a descriptive name for the node: +# +# node.name: test +# +# If omitted the node name will default to the machine's host name +# +# ------------ Data path ------------------ +# +# Which directory should be used by logstash and its plugins +# for any persistent needs. Defaults to LOGSTASH_HOME/data +# +path.data: /var/lib/logstash +# +# ------------ Pipeline Settings -------------- +# +# The ID of the pipeline. +# +# pipeline.id: main +# +# Set the number of workers that will, in parallel, execute the filters+outputs +# stage of the pipeline. +# +# This defaults to the number of the host's CPU cores. +# +# pipeline.workers: 2 +# +# How many events to retrieve from inputs before sending to filters+workers +# +# pipeline.batch.size: 125 +# +# How long to wait in milliseconds while polling for the next event +# before dispatching an undersized batch to filters+outputs +# +# pipeline.batch.delay: 50 +# +# Force Logstash to exit during shutdown even if there are still inflight +# events in memory. By default, logstash will refuse to quit until all +# received events have been pushed to the outputs. +# +# WARNING: enabling this can lead to data loss during shutdown +# +# pipeline.unsafe_shutdown: false +# +# ------------ Pipeline Configuration Settings -------------- +# +# Where to fetch the pipeline configuration for the main pipeline +# +# path.config: +# +# Pipeline configuration string for the main pipeline +# +# config.string: +# +# At startup, test if the configuration is valid and exit (dry run) +# +# config.test_and_exit: false +# +# Periodically check if the configuration has changed and reload the pipeline +# This can also be triggered manually through the SIGHUP signal +# +# config.reload.automatic: false +# +# How often to check if the pipeline configuration has changed (in seconds) +# +# config.reload.interval: 3s +# +# Show fully compiled configuration as debug log message +# NOTE: --log.level must be 'debug' +# +# config.debug: false +# +# When enabled, process escaped characters such as \n and \" in strings in the +# pipeline configuration files. +# +# config.support_escapes: false +# +# ------------ Module Settings --------------- +# Define modules here. Modules definitions must be defined as an array. +# The simple way to see this is to prepend each `name` with a `-`, and keep +# all associated variables under the `name` they are associated with, and +# above the next, like this: +# +# modules: +# - name: MODULE_NAME +# var.PLUGINTYPE1.PLUGINNAME1.KEY1: VALUE +# var.PLUGINTYPE1.PLUGINNAME1.KEY2: VALUE +# var.PLUGINTYPE2.PLUGINNAME1.KEY1: VALUE +# var.PLUGINTYPE3.PLUGINNAME3.KEY1: VALUE +# +# Module variable names must be in the format of +# +# var.PLUGIN_TYPE.PLUGIN_NAME.KEY +# +# modules: +# +# ------------ Cloud Settings --------------- +# Define Elastic Cloud settings here. +# Format of cloud.id is a base64 value e.g. dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyRub3RhcmVhbCRpZGVudGlmaWVy +# and it may have an label prefix e.g. staging:dXMtZ... +# This will overwrite 'var.elasticsearch.hosts' and 'var.kibana.host' +# cloud.id: +# +# Format of cloud.auth is: : +# This is optional +# If supplied this will overwrite 'var.elasticsearch.username' and 'var.elasticsearch.password' +# If supplied this will overwrite 'var.kibana.username' and 'var.kibana.password' +# cloud.auth: elastic: +# +# ------------ Queuing Settings -------------- +# +# Internal queuing model, "memory" for legacy in-memory based queuing and +# "persisted" for disk-based acked queueing. Defaults is memory +# +queue.type: persisted +# +# If using queue.type: persisted, the directory path where the data files will be stored. +# Default is path.data/queue +# +# path.queue: +# +# If using queue.type: persisted, the page data files size. The queue data consists of +# append-only data files separated into pages. Default is 64mb +# +# queue.page_capacity: 64mb +# +# If using queue.type: persisted, the maximum number of unread events in the queue. +# Default is 0 (unlimited) +# +# queue.max_events: 0 +# +# If using queue.type: persisted, the total capacity of the queue in number of bytes. +# If you would like more unacked events to be buffered in Logstash, you can increase the +# capacity using this setting. Please make sure your disk drive has capacity greater than +# the size specified here. If both max_bytes and max_events are specified, Logstash will pick +# whichever criteria is reached first +# Default is 1024mb or 1gb +# +queue.max_bytes: {{ logstash_queue_size }}mb +# +# If using queue.type: persisted, the maximum number of acked events before forcing a checkpoint +# Default is 1024, 0 for unlimited +# +# queue.checkpoint.acks: 1024 +# +# If using queue.type: persisted, the maximum number of written events before forcing a checkpoint +# Default is 1024, 0 for unlimited +# +# queue.checkpoint.writes: 1024 +# +# If using queue.type: persisted, the interval in milliseconds when a checkpoint is forced on the head page +# Default is 1000, 0 for no periodic checkpoint. +# +# queue.checkpoint.interval: 1000 +# +# ------------ Dead-Letter Queue Settings -------------- +# Flag to turn on dead-letter queue. +# +# dead_letter_queue.enable: false + +# If using dead_letter_queue.enable: true, the maximum size of each dead letter queue. Entries +# will be dropped if they would increase the size of the dead letter queue beyond this setting. +# Default is 1024mb +# dead_letter_queue.max_bytes: 1024mb + +# If using dead_letter_queue.enable: true, the directory path where the data files will be stored. +# Default is path.data/dead_letter_queue +# +# path.dead_letter_queue: +# +# ------------ Metrics Settings -------------- +# +# Bind address for the metrics REST endpoint +# +# http.host: "127.0.0.1" +# +# Bind port for the metrics REST endpoint, this option also accept a range +# (9600-9700) and logstash will pick up the first available ports. +# +# http.port: 9600-9700 +# +# ------------ Debugging Settings -------------- +# +# Options for log.level: +# * fatal +# * error +# * warn +# * info (default) +# * debug +# * trace +# +# log.level: info +path.logs: /var/log/logstash +# +# ------------ Other Settings -------------- +# +# Where to find custom plugins +# path.plugins: [] +# +# ------------ X-Pack Settings (not applicable for OSS build)-------------- +# +# X-Pack Monitoring +# https://www.elastic.co/guide/en/logstash/current/monitoring-logstash.html +#xpack.monitoring.enabled: false +#xpack.monitoring.elasticsearch.username: logstash_system +#xpack.monitoring.elasticsearch.password: password +#xpack.monitoring.elasticsearch.url: ["https://es1:9200", "https://es2:9200"] +#xpack.monitoring.elasticsearch.ssl.ca: [ "/path/to/ca.crt" ] +#xpack.monitoring.elasticsearch.ssl.truststore.path: path/to/file +#xpack.monitoring.elasticsearch.ssl.truststore.password: password +#xpack.monitoring.elasticsearch.ssl.keystore.path: /path/to/file +#xpack.monitoring.elasticsearch.ssl.keystore.password: password +#xpack.monitoring.elasticsearch.ssl.verification_mode: certificate +#xpack.monitoring.elasticsearch.sniffing: false +#xpack.monitoring.collection.interval: 10s +#xpack.monitoring.collection.pipeline.details.enabled: true +# +# X-Pack Management +# https://www.elastic.co/guide/en/logstash/current/logstash-centralized-pipeline-management.html +#xpack.management.enabled: false +#xpack.management.pipeline.id: ["main", "apache_logs"] +#xpack.management.elasticsearch.username: logstash_admin_user +#xpack.management.elasticsearch.password: password +#xpack.management.elasticsearch.url: ["https://es1:9200", "https://es2:9200"] +#xpack.management.elasticsearch.ssl.ca: [ "/path/to/ca.crt" ] +#xpack.management.elasticsearch.ssl.truststore.path: /path/to/file +#xpack.management.elasticsearch.ssl.truststore.password: password +#xpack.management.elasticsearch.ssl.keystore.path: /path/to/file +#xpack.management.elasticsearch.ssl.keystore.password: password +#xpack.management.elasticsearch.sniffing: false +#xpack.management.logstash.poll_interval: 5s