From 443539b7fa82898c17512be2eadbeb92abe3e212 Mon Sep 17 00:00:00 2001 From: Clark Boylan Date: Sat, 22 Jun 2013 10:39:04 -0700 Subject: [PATCH] Better elasticsearch cluster settings. Use mutliple discover nodes to determine elasticsearch cluster membership. Put a timeout on recovery starting instead of the default to recovery immediately. Describe cluster topology in elasticsearch yaml config so that it can make smarter decisions. Round robin kibana requests across each discover node. Change-Id: I08ef9dd158ddf6a6ce01dfb2050626f543d45b10 Reviewed-on: https://review.openstack.org/34106 Reviewed-by: Jeremy Stanley Reviewed-by: James E. Blair Approved: James E. Blair Tested-by: Jenkins --- manifests/site.pp | 12 ++++++++++-- modules/kibana/manifests/init.pp | 2 +- modules/kibana/templates/config.rb.erb | 2 +- modules/logstash/files/elasticsearch.default | 2 +- modules/logstash/manifests/elasticsearch.pp | 2 +- modules/logstash/manifests/web.pp | 4 ++-- modules/logstash/templates/elasticsearch.yml.erb | 12 +++++++----- modules/logstash/templates/kibana.vhost.erb | 4 ++-- modules/openstack_project/manifests/elasticsearch.pp | 4 ++-- modules/openstack_project/manifests/logstash.pp | 4 ++-- 10 files changed, 29 insertions(+), 19 deletions(-) diff --git a/manifests/site.pp b/manifests/site.pp index e745f41d58..1eabc5705a 100644 --- a/manifests/site.pp +++ b/manifests/site.pp @@ -214,7 +214,11 @@ node 'logstash.openstack.org' { 'logstash-worker4.openstack.org', 'logstash-worker5.openstack.org', ], - discover_node => 'elasticsearch.openstack.org', + discover_nodes => [ + 'elasticsearch.openstack.org:9200', + 'elasticsearch2.openstack.org:9200', + 'elasticsearch3.openstack.org:9200', + ], } } @@ -246,7 +250,11 @@ node /^elasticsearch\d*\.openstack\.org$/ { 'logstash-worker4.openstack.org', 'logstash-worker5.openstack.org', ], - discover_node => 'elasticsearch.openstack.org', + discover_nodes => [ + 'elasticsearch.openstack.org', + 'elasticsearch2.openstack.org', + 'elasticsearch3.openstack.org', + ], } } diff --git a/modules/kibana/manifests/init.pp b/modules/kibana/manifests/init.pp index 0fcfd3c756..eb091ab592 100644 --- a/modules/kibana/manifests/init.pp +++ b/modules/kibana/manifests/init.pp @@ -15,7 +15,7 @@ # Class to install kibana frontend to logstash. # class kibana ( - $discover_node = 'localhost' + $discover_nodes = ['localhost:9200'] ) { group { 'kibana': diff --git a/modules/kibana/templates/config.rb.erb b/modules/kibana/templates/config.rb.erb index 65384ec999..b59843e28a 100755 --- a/modules/kibana/templates/config.rb.erb +++ b/modules/kibana/templates/config.rb.erb @@ -7,7 +7,7 @@ module KibanaConfig # Your elastic search server(s). This may be set as an array for round robin # load balancing # Elasticsearch = ["elasticsearch1:9200","elasticsearch2:9200"] - Elasticsearch = "<%= scope.lookupvar("::kibana::discover_node") %>:9200" + Elasticsearch = ["<%= scope.lookupvar("::kibana::discover_nodes").join("\",\"") %>"] #Set the Net::HTTP read/open timeouts for the connection to the ES backend ElasticsearchTimeout = 500 diff --git a/modules/logstash/files/elasticsearch.default b/modules/logstash/files/elasticsearch.default index 774b8e23f0..1d1aac6438 100644 --- a/modules/logstash/files/elasticsearch.default +++ b/modules/logstash/files/elasticsearch.default @@ -17,7 +17,7 @@ ES_HEAP_SIZE=16g # Maximum locked memory size. Set to "unlimited" if you use the # bootstrap.mlockall option in elasticsearch.yml. You must also set # ES_HEAP_SIZE. -#MAX_LOCKED_MEMORY=unlimited +MAX_LOCKED_MEMORY=unlimited # ElasticSearch log directory #LOG_DIR=/var/log/elasticsearch diff --git a/modules/logstash/manifests/elasticsearch.pp b/modules/logstash/manifests/elasticsearch.pp index 68018b9b50..106a25478a 100644 --- a/modules/logstash/manifests/elasticsearch.pp +++ b/modules/logstash/manifests/elasticsearch.pp @@ -15,7 +15,7 @@ # Class to install elasticsearch. # class logstash::elasticsearch ( - discover_node = 'localhost' + discover_nodes = ['localhost'] ) { # install java runtime package { 'java7-runtime-headless': diff --git a/modules/logstash/manifests/web.pp b/modules/logstash/manifests/web.pp index a60fe4c304..9919301f9a 100644 --- a/modules/logstash/manifests/web.pp +++ b/modules/logstash/manifests/web.pp @@ -18,7 +18,7 @@ class logstash::web ( $vhost_name = $::fqdn, $serveradmin = "webmaster@${::fqdn}", $frontend = 'internal', - $discover_node = 'localhost', + $discover_nodes = ['localhost:9200'], $proxy_elasticsearch = false ) { include apache @@ -57,7 +57,7 @@ class logstash::web ( 'kibana': { class { 'kibana': - discover_node => $discover_node, + discover_nodes => $discover_nodes, } $vhost = 'logstash/kibana.vhost.erb' } diff --git a/modules/logstash/templates/elasticsearch.yml.erb b/modules/logstash/templates/elasticsearch.yml.erb index 1803562ebd..6e102ec75b 100644 --- a/modules/logstash/templates/elasticsearch.yml.erb +++ b/modules/logstash/templates/elasticsearch.yml.erb @@ -255,18 +255,18 @@ bootstrap.mlockall: true # Allow recovery process after N nodes in a cluster are up: # -# gateway.recover_after_nodes: 1 +gateway.recover_after_nodes: 2 # Set the timeout to initiate the recovery process, once the N nodes # from previous setting are up (accepts time value): # -# gateway.recover_after_time: 5m +gateway.recover_after_time: 5m # Set how many nodes are expected in this cluster. Once these N nodes # are up (and recover_after_nodes is met), begin recovery process immediately # (without waiting for recover_after_time to expire): # -# gateway.expected_nodes: 2 +gateway.expected_nodes: 3 ############################# Recovery Throttling ############################# @@ -304,7 +304,7 @@ bootstrap.mlockall: true # operational within the cluster. Set this option to a higher value (2-4) # for large clusters (>3 nodes): # -# discovery.zen.minimum_master_nodes: 1 +discovery.zen.minimum_master_nodes: 2 # Set the time to wait for ping responses from other nodes when discovering. # Set this option to a higher value on a slow or congested network @@ -327,7 +327,9 @@ discovery.zen.ping.multicast.enabled: false # to perform discovery when new nodes (master or data) are started: # # discovery.zen.ping.unicast.hosts: ["host1", "host2:port", "host3[portX-portY]"] -discovery.zen.ping.unicast.hosts: ["<%= discover_node %>"] +# +# Make all master eligible nodes discover nodes. +discovery.zen.ping.unicast.hosts: ["<%= discover_nodes.join("\", \"") %>"] # EC2 discovery allows to use AWS EC2 API in order to perform discovery. # diff --git a/modules/logstash/templates/kibana.vhost.erb b/modules/logstash/templates/kibana.vhost.erb index 341c65fbca..59576ee53c 100644 --- a/modules/logstash/templates/kibana.vhost.erb +++ b/modules/logstash/templates/kibana.vhost.erb @@ -11,9 +11,9 @@ <% if proxy_elasticsearch == true %> # Proxy for elasticsearch _aliases, .*/_status, and .*/_search. - ProxyPassMatch http://<%= scope.lookupvar("::logstash::web::discover_node") %>:9200/$1 + ProxyPassMatch http://<%= scope.lookupvar("::logstash::web::discover_nodes")[0] %>/$1 - ProxyPassReverse /elasticsearch/ http://<%= scope.lookupvar("::logstash::web::discover_node") %>:9200/ + ProxyPassReverse /elasticsearch/ http://<%= scope.lookupvar("::logstash::web::discover_nodes")[0] %>/ <% end %> ProxyPass / http://127.0.0.1:5601/ retry=0 diff --git a/modules/openstack_project/manifests/elasticsearch.pp b/modules/openstack_project/manifests/elasticsearch.pp index f6648df75a..5bf0c4bf5b 100644 --- a/modules/openstack_project/manifests/elasticsearch.pp +++ b/modules/openstack_project/manifests/elasticsearch.pp @@ -17,7 +17,7 @@ class openstack_project::elasticsearch ( $elasticsearch_nodes = [], $elasticsearch_clients = [], - $discover_node = 'localhost', + $discover_nodes = ['localhost'], $sysadmins = [] ) { $iptables_nodes_rule = regsubst ($elasticsearch_nodes, '^(.*)$', '-m state --state NEW -m tcp -p tcp --dport 9200:9400 -s \1 -j ACCEPT') @@ -31,7 +31,7 @@ class openstack_project::elasticsearch ( } class { 'logstash::elasticsearch': - discover_node => $discover_node, + discover_nodes => $discover_nodes, } cron { 'delete_old_es_indices': diff --git a/modules/openstack_project/manifests/logstash.pp b/modules/openstack_project/manifests/logstash.pp index 7cb74beb5d..c341b3b2e3 100644 --- a/modules/openstack_project/manifests/logstash.pp +++ b/modules/openstack_project/manifests/logstash.pp @@ -17,7 +17,7 @@ class openstack_project::logstash ( $elasticsearch_nodes = [], $gearman_workers = [], - $discover_node = 'elasticsearch.openstack.org', + $discover_nodes = ['elasticsearch.openstack.org:9200'], $sysadmins = [] ) { $iptables_es_rule = regsubst ($elasticsearch_nodes, '^(.*)$', '-m state --state NEW -m tcp -p tcp --dport 9200:9400 -s \1 -j ACCEPT') @@ -32,7 +32,7 @@ class openstack_project::logstash ( class { 'logstash::web': frontend => 'kibana', - discover_node => $discover_node, + discover_nodes => $discover_nodes, proxy_elasticsearch => true, }