From 0b0efcb841f4d7ec3c7bd2c3b210b0da34d70922 Mon Sep 17 00:00:00 2001 From: Kevin Carter Date: Mon, 17 Sep 2018 21:17:20 -0500 Subject: [PATCH] Add capability to set node role Presently the node role assignment is only automatic. Auto selection makes the assumption every node is identical however in many deployments a deployer may want to assign node roles to specific hardware thereby optimizing resources and improving general performance. This change adds and documents the ability to set the node roles within an ansible inventory. Change-Id: I22a2b636cb1441f17e575439b55ca64f9c7b0336 Signed-off-by: Kevin Carter --- elk_metrics_6x/README.rst | 50 +++++++++++ elk_metrics_6x/bootstrap-embedded-ansible.sh | 2 +- .../vars/data-node-variables.yml | 88 ++++++++++++++++--- .../templates/jvm.options.j2 | 3 +- .../roles/elastic_logstash/handlers/main.yml | 4 +- .../templates/logstash.yml.j2 | 4 +- .../systemd.elasticsearch-overrides.conf.j2 | 2 +- 7 files changed, 134 insertions(+), 19 deletions(-) diff --git a/elk_metrics_6x/README.rst b/elk_metrics_6x/README.rst index e408b14a..d02c3e2e 100644 --- a/elk_metrics_6x/README.rst +++ b/elk_metrics_6x/README.rst @@ -508,6 +508,56 @@ Overview of kibana custom dashboard :align: center +Optional | Customize Elasticsearch cluster configuration +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Cluster configuration can be augmented using several variables which will force +a node to use a given role. + +Available roles are *data*, *ingest*, and *master*. + +* ``elasticsearch_node_data``: This variable will override the automatic node + determination and set a given node to be an "data" node. +* ``elasticsearch_node_ingest``: This variable will override the automatic node + determination and set a given node to be an "ingest" node. +* ``elasticsearch_node_master``: This variable will override the automatic node + determination and set a given node to be an "master" node. + +Example setting override options within inventory. + +.. code-block:: yaml + + hosts: + children: + elastic-logstash: + hosts: + elk1: + ansible_host: 10.0.0.1 + ansible_user: root + elasticsearch_node_master: true + elasticsearch_node_data: false + elasticsearch_node_ingest: false + elk2: + ansible_host: 10.0.0.2 + ansible_user: root + elasticsearch_node_master: false + elasticsearch_node_data: true + elasticsearch_node_ingest: false + elk3: + ansible_host: 10.0.0.3 + ansible_user: root + elasticsearch_node_master: false + elasticsearch_node_data: false + elasticsearch_node_ingest: true + elk4: + ansible_host: 10.0.0.4 + ansible_user: root + +With the following inventory settings **elk1** would be a master node, **elk2** +would be a data, **elk3** would be an ingest node, and **elk4** would auto +select a role. + + Upgrading the cluster --------------------- diff --git a/elk_metrics_6x/bootstrap-embedded-ansible.sh b/elk_metrics_6x/bootstrap-embedded-ansible.sh index f40f8640..f0a4c14e 100755 --- a/elk_metrics_6x/bootstrap-embedded-ansible.sh +++ b/elk_metrics_6x/bootstrap-embedded-ansible.sh @@ -52,7 +52,7 @@ if [[ ! -d "${ANSIBLE_EMBED_HOME}/repositories/ansible-config_template" ]]; then popd fi -if [[ ! -d "${ANSIBLE_EMBED_HOME}/repositories/openstack_ansible_plugins" ]]; then +if [[ ! -d "${ANSIBLE_EMBED_HOME}/repositories/openstack-ansible-plugins" ]]; then mkdir -p "${ANSIBLE_EMBED_HOME}/repositories" git clone https://git.openstack.org/openstack/openstack-ansible-plugins "${ANSIBLE_EMBED_HOME}/repositories/openstack-ansible-plugins" pushd "${ANSIBLE_EMBED_HOME}/repositories/openstack-ansible-plugins" diff --git a/elk_metrics_6x/roles/elastic_data_hosts/vars/data-node-variables.yml b/elk_metrics_6x/roles/elastic_data_hosts/vars/data-node-variables.yml index b0e25026..a161df10 100644 --- a/elk_metrics_6x/roles/elastic_data_hosts/vars/data-node-variables.yml +++ b/elk_metrics_6x/roles/elastic_data_hosts/vars/data-node-variables.yml @@ -21,7 +21,7 @@ storage_node_count: "{{ groups['elastic-logstash'] | length }}" # 2 node clusters have 1 master-eligable nodes to avoid split-brain # 3 node clusters have 3 master-eligable nodes # >3 node clusters have (nodes // 2) eligable masters rounded up to the next odd number -master_node_count: |- +elastic_master_node_count: |- {% set masters = 0 %} {% if (storage_node_count | int) < 3 %} {% set masters = 1 %} @@ -36,10 +36,76 @@ master_node_count: |- {{ masters }} # Assign node roles -# the first 'master_node_count' hosts in groups['elastic-logstash'] become master-eligible nodes -# the first 'master_node_count' and subsequent alternate hosts in groups['elastic-logstash'] becomes data nodes -data_nodes: "{{ (groups['elastic-logstash'][:master_node_count | int] + groups['elastic-logstash'][master_node_count | int::2]) }}" -master_nodes: "{{ groups['elastic-logstash'][:master_node_count | int] }}" +# the first 'elastic_master_node_count' hosts in groups['elastic-logstash'] become master-eligible nodes +# the first 'elastic_master_node_count' and subsequent alternate hosts in groups['elastic-logstash'] becomes data nodes +## While the data node group is dynamically chosen the override +## `elasticsearch_node_data` can be used to override the node type. +## Dynamic node inclusion will still work for all other nodes in the group. +_data_nodes: "{{ (groups['elastic-logstash'][:elastic_master_node_count | int] | union(groups['elastic-logstash'][elastic_master_node_count | int::2])) }}" +data_nodes: |- + {% set nodes = [] %} + {% for node in groups['elastic-logstash'] %} + {% if (hostvars[node]['elasticsearch_node_data'] is defined) and (hostvars[node]['elasticsearch_node_data'] | bool) %} + {% set _ = nodes.append(node) %} + {% elif (node in _data_nodes) %} + {% set _ = nodes.append(node) %} + {% endif %} + {% endfor %} + {{ nodes }} + +## While the logstash node group is dynamically chosen the override +## `elasticsearch_node_ingest` can be used to override the node type. +## Dynamic node inclusion will still work for all other nodes in the group. +_logstash_nodes: "{{ data_nodes }}" +logstash_nodes: |- + {% set nodes = [] %} + {% for node in groups['elastic-logstash'] %} + {% if (hostvars[node]['elasticsearch_node_ingest'] is defined) and (hostvars[node]['elasticsearch_node_ingest'] | bool) %} + {% set _ = nodes.append(node) %} + {% elif (node in _logstash_nodes) %} + {% set _ = nodes.append(node) %} + {% endif %} + {% endfor %} + {{ nodes }} + +## While the logstash node group is dynamically chosen the override +## `elasticsearch_node_ingest` can be used to override the node type. +## Dynamic node inclusion will still work for all other nodes in the group. +_ingest_nodes: "{{ data_nodes }}" +ingest_nodes: |- + {% set nodes = [] %} + {% for node in groups['elastic-logstash'] %} + {% if (hostvars[node]['elasticsearch_node_ingest'] is defined) and (hostvars[node]['elasticsearch_node_ingest'] | bool) %} + {% set _ = nodes.append(node) %} + {% elif (node in _ingest_nodes) %} + {% set _ = nodes.append(node) %} + {% endif %} + {% endfor %} + {{ nodes }} + +## While the master node group is dynamically chosen the override +## `elasticsearch_node_master` can be used to override the node type. +## Dynamic node inclusion will still work for all other nodes in the group. +_master_nodes: "{{ groups['elastic-logstash'][:elastic_master_node_count | int] }}" +master_nodes: |- + {% set nodes = [] %} + {% for node in groups['elastic-logstash'] %} + {% if (nodes | length) <= (elastic_master_node_count | int) %} + {% if (hostvars[node]['elasticsearch_node_master'] is defined) and (hostvars[node]['elasticsearch_node_master'] | bool) %} + {% set _ = nodes.append(node) %} + {% endif %} + {% endif %} + {% endfor %} + {% for node in groups['elastic-logstash'] %} + {% if (nodes | length) <= (elastic_master_node_count | int) %} + {% if (node in _master_nodes) %} + {% set _ = nodes.append(node) %} + {% endif %} + {% endif %} + {% endfor %} + {{ nodes }} +master_node_count: "{{ master_nodes | length }}" + coordination_nodes: >- {{ (groups['kibana'] | map('extract', hostvars, 'ansible_host') | list) @@ -90,7 +156,7 @@ elasticsearch_data_node_details: >- }} logstash_data_node_details: >- {{ - (data_nodes | map('extract', hostvars, 'ansible_host') | list) | map('regex_replace', '(.*)' ,'\1:' ~ logstash_beat_input_port) | list + (logstash_nodes | map('extract', hostvars, 'ansible_host') | list) | map('regex_replace', '(.*)' ,'\1:' ~ logstash_beat_input_port) | list }} # based on the assignment of roles to hosts, set per host booleans @@ -105,16 +171,14 @@ elastic_thread_pool_size: "{{ ((ansible_processor_count | int) >= 24) | ternary( # ingest nodes. elasticsearch_number_of_replicas: "{{ ((data_nodes | length) > 2) | ternary('2', ((data_nodes | length) > 1) | ternary('1', '0')) }}" elasticsearch_data_hosts: |- - {% set nodes = elasticsearch_data_node_details %} - {% set data_hosts = nodes | shuffle(seed=inventory_hostname) %} + {% set data_hosts = elasticsearch_data_node_details | shuffle(seed=inventory_hostname) %} {% if inventory_hostname in data_nodes %} - {% set _ = nodes.insert(0, '127.0.0.1:' ~ elastic_port) %} + {% set _ = data_hosts.insert(0, '127.0.0.1:' ~ elastic_port) %} {% endif %} {{ data_hosts }} logstash_data_hosts: |- - {% set nodes = logstash_data_node_details %} - {% set data_hosts = nodes | shuffle(seed=inventory_hostname) %} + {% set data_hosts = logstash_data_node_details | shuffle(seed=inventory_hostname) %} {% if inventory_hostname in data_nodes %} - {% set _ = nodes.insert(0, '127.0.0.1:' ~ logstash_beat_input_port) %} + {% set _ = data_hosts.insert(0, '127.0.0.1:' ~ logstash_beat_input_port) %} {% endif %} {{ data_hosts }} diff --git a/elk_metrics_6x/roles/elastic_dependencies/templates/jvm.options.j2 b/elk_metrics_6x/roles/elastic_dependencies/templates/jvm.options.j2 index e4e4e921..6ec918d1 100644 --- a/elk_metrics_6x/roles/elastic_dependencies/templates/jvm.options.j2 +++ b/elk_metrics_6x/roles/elastic_dependencies/templates/jvm.options.j2 @@ -8,7 +8,8 @@ -Xms{{ heap_size }}m # Xmx represents the maximum size of total heap space -Xmx{{ heap_size }}m - +# Sets the thread stack size +-Xss1m ################################################################ ## Expert settings diff --git a/elk_metrics_6x/roles/elastic_logstash/handlers/main.yml b/elk_metrics_6x/roles/elastic_logstash/handlers/main.yml index c8e3f6b3..3a1c3e52 100644 --- a/elk_metrics_6x/roles/elastic_logstash/handlers/main.yml +++ b/elk_metrics_6x/roles/elastic_logstash/handlers/main.yml @@ -17,7 +17,7 @@ systemd: name: "logstash" enabled: true - state: restarted + state: "{{ (inventory_hostname in logstash_nodes) | ternary('restarted', 'stopped') }}" daemon_reload: true when: - ansible_service_mgr == 'systemd' @@ -26,7 +26,7 @@ - name: Enable and restart logstash (upstart) service: name: "logstash" - state: restarted + state: "{{ (inventory_hostname in logstash_nodes) | ternary('restarted', 'stopped') }}" enabled: yes when: - ansible_service_mgr == 'upstart' diff --git a/elk_metrics_6x/roles/elastic_logstash/templates/logstash.yml.j2 b/elk_metrics_6x/roles/elastic_logstash/templates/logstash.yml.j2 index 3b200875..b3f5de2e 100644 --- a/elk_metrics_6x/roles/elastic_logstash/templates/logstash.yml.j2 +++ b/elk_metrics_6x/roles/elastic_logstash/templates/logstash.yml.j2 @@ -231,14 +231,14 @@ path.logs: /var/log/logstash xpack.monitoring.enabled: true #xpack.monitoring.elasticsearch.username: logstash_system #xpack.monitoring.elasticsearch.password: password -#xpack.monitoring.elasticsearch.url: ["https://es1:9200", "https://es2:9200"] +xpack.monitoring.elasticsearch.url: ["127.0.0.1:9200"] #xpack.monitoring.elasticsearch.ssl.ca: [ "/path/to/ca.crt" ] #xpack.monitoring.elasticsearch.ssl.truststore.path: path/to/file #xpack.monitoring.elasticsearch.ssl.truststore.password: password #xpack.monitoring.elasticsearch.ssl.keystore.path: /path/to/file #xpack.monitoring.elasticsearch.ssl.keystore.password: password #xpack.monitoring.elasticsearch.ssl.verification_mode: certificate -#xpack.monitoring.elasticsearch.sniffing: false +xpack.monitoring.elasticsearch.sniffing: false xpack.monitoring.collection.interval: 30s xpack.monitoring.collection.pipeline.details.enabled: true # diff --git a/elk_metrics_6x/roles/elasticsearch/templates/systemd.elasticsearch-overrides.conf.j2 b/elk_metrics_6x/roles/elasticsearch/templates/systemd.elasticsearch-overrides.conf.j2 index 0fe93012..64dcf40a 100644 --- a/elk_metrics_6x/roles/elasticsearch/templates/systemd.elasticsearch-overrides.conf.j2 +++ b/elk_metrics_6x/roles/elasticsearch/templates/systemd.elasticsearch-overrides.conf.j2 @@ -11,4 +11,4 @@ LimitMEMLOCK=infinity {% endif %} # Number of File Descriptors -LimitNOFILE=131070 +LimitNOFILE=65536