Merge "Thread pools should be based on processor counts"

elk_metrics_6x/roles/elastic_data_hosts/vars/data-node-variables.yml

@@ -96,7 +96,10 @@ logstash_data_node_details: >-
 # based on the assignment of roles to hosts, set per host booleans
 master_node: "{{ (inventory_hostname in master_nodes) | ternary(true, false) }}"
 data_node: "{{ (inventory_hostname in data_nodes) | ternary(true, false) }}"
-elastic_thread_pool_size: "{{ ((ansible_processor_cores | int) >= 24) | ternary(24, ansible_processor_cores) }}"
+
+elastic_processors_floor: "{{ ((ansible_processor_count | int) - 1) }}"
+elastic_processors_floor_set: "{{ ((elastic_processors_floor | int) > 0) | ternary(elastic_processors_floor, 1) }}"
+elastic_thread_pool_size: "{{ ((ansible_processor_count | int) >= 24) | ternary(23, elastic_processors_floor_set) }}"
 
 # Set a data node facts. The data nodes, in the case of elasticsearch are also
 # ingest nodes.

elk_metrics_6x/roles/elastic_logstash/defaults/main.yml

@@ -17,7 +17,7 @@ temp_dir: /var/lib/logstash/tmp
 logstash_pipelines: "{{lookup('template', 'logstash-pipelines.yml.j2') }}"
 
 # Set processor cores fact
-q_storage: "{{ (ansible_processor_cores | int) * (ansible_processor_threads_per_core | int) * 2 }}"
+q_storage: "{{ (ansible_processor_count | int) * (ansible_processor_threads_per_core | int) * 2 }}"
 
 # Set logstash facts
 logstash_queue_size: "{{ ((((q_storage | int) >= 2) | ternary(q_storage, 2) | int) * 1024) // ((logstash_pipelines | from_yaml) | length) }}"

elk_metrics_6x/roles/elastic_logstash/templates/logstash.yml.j2

@@ -38,7 +38,10 @@ path.data: /var/lib/logstash
 #
 # This defaults to the number of the host's CPU cores.
 #
-{% set processors = ((elastic_thread_pool_size | int) > 0) | ternary(elastic_thread_pool_size, 1) %}
+
+{% set _h_processors = ((ansible_processor_count | int) // 2) %}
+{% set _processors = ((_h_processors | int) > 0) | ternary(_h_processors, 1) %}
+{% set processors = ((_processors | int) > 8) | ternary(8, _processors) %}
 pipeline.workers: {{ processors | int }}
 #
 # How many events to retrieve from inputs before sending to filters+workers

elk_metrics_6x/roles/elasticsearch/vars/vars_kibana.yml

@@ -19,4 +19,9 @@ elasticsearch_node_data: false
 elasticsearch_node_ingest: false
 elastic_coordination_node: true
 elastic_heap_size: "{{ (elastic_heap_size_default | int) // 3 }}"
-elastic_thread_pool_size: "{{ ((ansible_processor_cores | int) > 4) | ternary(4, (ansible_processor_cores // 2)) }}"
+
+# This variable is redefined because kibana runs elasticsearch but only in a
+# load balancer capacity.
+elastic_processors_half: "{{ ((ansible_processor_count | int) // 2) }}"
+elastic_processors_half_set: "{{ ((elastic_processors_half | int) > 0) | ternary(elastic_processors_half, 1) }}"
+elastic_thread_pool_size: "{{ ((elastic_processors_half_set | int) > 4) | ternary(4, elastic_processors_half_set) }}"

elk_metrics_6x/templates/_include_log_stash_output.yml.j2

@@ -43,7 +43,7 @@ output.logstash:
   # less than or equal to 0 disables the splitting of batches. When splitting
   # is disabled, the queue decides on the number of events to be contained in a
   # batch.
-  bulk_max_size: {{ (elastic_thread_pool_size | int) * 256 }}
+  bulk_max_size: {{ (ansible_processor_count | int) * 256 }}
 
 {% if named_index is defined %}
   # Optional index name. The default index name is set to {{ named_index }}

elk_metrics_6x/templates/_include_xpack_elasticsearch_output.yml.j2

@@ -45,7 +45,7 @@ xpack.monitoring.elasticsearch:
 
   # The maximum number of events to bulk in a single Elasticsearch bulk API index request.
   # The default is 50.
-  bulk_max_size: {{ (elastic_thread_pool_size | int) * 64 }}
+  bulk_max_size: {{ (ansible_processor_count | int) * 64 }}
 
   # Configure http request timeout before failing an request to Elasticsearch.
   timeout: 120

elk_metrics_6x/templates/logstash-pipelines.yml.j2

@@ -547,12 +547,6 @@
             add_tag => ["apimetrics"]
           }
         }
-      } else if "swift-container" in [tags] {
-        grok {
-          match => {
-            "message" => "%{CISCOTIMESTAMP}%{SPACE}%{S3_REQUEST_LINE}%{SPACE}%{CISCOTIMESTAMP}%{SPACE}%{HOSTNAME}%{SPACE}%{PROG}%{SPACE}%{USER}%{SPACE}%{USERNAME}%{SPACE}%{NOTSPACE}%{SPACE}%{S3_REQUEST_LINE}%{SPACE}%{HTTPDUSER}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE}%{SPACE}%{INT}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE}%{SPACE}%{SECOND}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE}"
-          }
-        }
       } else if "swift-account" in [tags] {
         grok {
           match => {