From 38b53a4beeac7c762cdd1cb7248ac2272b6b8d66 Mon Sep 17 00:00:00 2001 From: Clark Boylan Date: Wed, 6 Aug 2014 17:56:16 -0700 Subject: [PATCH] Cleanup elasticsearch related config Remove elasticsearch01 as it has been turned off. Allow all elasticsearch running nodes to talk to each other over ports 9200 to 9400. This includes the logstash-worker nodes running non data elasticsearch clients. This appears to be necessary to keep logs from filling up with gigs of NoRouteToHost exceptions on the worker nodes. Change-Id: I3e3365a717d22d881654206692d17dd05ed32c98 --- manifests/site.pp | 84 +++++++------------ .../manifests/logstash_worker.pp | 3 +- 2 files changed, 34 insertions(+), 53 deletions(-) diff --git a/manifests/site.pp b/manifests/site.pp index d0411ee8d2..5f202ad899 100644 --- a/manifests/site.pp +++ b/manifests/site.pp @@ -6,7 +6,6 @@ # passed around in test.sh # $elasticsearch_nodes = [ - 'elasticsearch01.openstack.org', 'elasticsearch02.openstack.org', 'elasticsearch03.openstack.org', 'elasticsearch04.openstack.org', @@ -14,6 +13,29 @@ $elasticsearch_nodes = [ 'elasticsearch06.openstack.org', 'elasticsearch07.openstack.org', ] +$elasticsearch_clients = [ + 'logstash.openstack.org', + 'logstash-worker01.openstack.org', + 'logstash-worker02.openstack.org', + 'logstash-worker03.openstack.org', + 'logstash-worker04.openstack.org', + 'logstash-worker05.openstack.org', + 'logstash-worker06.openstack.org', + 'logstash-worker07.openstack.org', + 'logstash-worker08.openstack.org', + 'logstash-worker09.openstack.org', + 'logstash-worker10.openstack.org', + 'logstash-worker11.openstack.org', + 'logstash-worker12.openstack.org', + 'logstash-worker13.openstack.org', + 'logstash-worker14.openstack.org', + 'logstash-worker15.openstack.org', + 'logstash-worker16.openstack.org', + 'logstash-worker17.openstack.org', + 'logstash-worker18.openstack.org', + 'logstash-worker19.openstack.org', + 'logstash-worker20.openstack.org', +] # # Default: should at least behave like an openstack server @@ -290,31 +312,10 @@ node 'wiki.openstack.org' { # Node-OS: precise node 'logstash.openstack.org' { class { 'openstack_project::logstash': - sysadmins => hiera('sysadmins', []), - elasticsearch_nodes => $elasticsearch_nodes, - gearman_workers => [ - 'logstash-worker01.openstack.org', - 'logstash-worker02.openstack.org', - 'logstash-worker03.openstack.org', - 'logstash-worker04.openstack.org', - 'logstash-worker05.openstack.org', - 'logstash-worker06.openstack.org', - 'logstash-worker07.openstack.org', - 'logstash-worker08.openstack.org', - 'logstash-worker09.openstack.org', - 'logstash-worker10.openstack.org', - 'logstash-worker11.openstack.org', - 'logstash-worker12.openstack.org', - 'logstash-worker13.openstack.org', - 'logstash-worker14.openstack.org', - 'logstash-worker15.openstack.org', - 'logstash-worker16.openstack.org', - 'logstash-worker17.openstack.org', - 'logstash-worker18.openstack.org', - 'logstash-worker19.openstack.org', - 'logstash-worker20.openstack.org', - ], - discover_nodes => [ + sysadmins => hiera('sysadmins', []), + elasticsearch_nodes => $elasticsearch_nodes, + gearman_workers => $elasticsearch_clients, + discover_nodes => [ 'elasticsearch02.openstack.org:9200', 'elasticsearch03.openstack.org:9200', 'elasticsearch04.openstack.org:9200', @@ -328,9 +329,10 @@ node 'logstash.openstack.org' { # Node-OS: precise node /^logstash-worker\d+\.openstack\.org$/ { class { 'openstack_project::logstash_worker': - sysadmins => hiera('sysadmins', []), - elasticsearch_nodes => $elasticsearch_nodes, - discover_node => 'elasticsearch02.openstack.org', + sysadmins => hiera('sysadmins', []), + elasticsearch_nodes => $elasticsearch_nodes, + elasticsearch_clients => $elasticsearch_clients, + discover_node => 'elasticsearch02.openstack.org', } } @@ -339,29 +341,7 @@ node /^elasticsearch0[1-7]\.openstack\.org$/ { class { 'openstack_project::elasticsearch_node': sysadmins => hiera('sysadmins', []), elasticsearch_nodes => $elasticsearch_nodes, - elasticsearch_clients => [ - 'logstash.openstack.org', - 'logstash-worker01.openstack.org', - 'logstash-worker02.openstack.org', - 'logstash-worker03.openstack.org', - 'logstash-worker04.openstack.org', - 'logstash-worker05.openstack.org', - 'logstash-worker06.openstack.org', - 'logstash-worker07.openstack.org', - 'logstash-worker08.openstack.org', - 'logstash-worker09.openstack.org', - 'logstash-worker10.openstack.org', - 'logstash-worker11.openstack.org', - 'logstash-worker12.openstack.org', - 'logstash-worker13.openstack.org', - 'logstash-worker14.openstack.org', - 'logstash-worker15.openstack.org', - 'logstash-worker16.openstack.org', - 'logstash-worker17.openstack.org', - 'logstash-worker18.openstack.org', - 'logstash-worker19.openstack.org', - 'logstash-worker20.openstack.org', - ], + elasticsearch_clients => $elasticsearch_clients, discover_nodes => $elasticsearch_nodes, } } diff --git a/modules/openstack_project/manifests/logstash_worker.pp b/modules/openstack_project/manifests/logstash_worker.pp index 9c7b415f78..6d30b5f274 100644 --- a/modules/openstack_project/manifests/logstash_worker.pp +++ b/modules/openstack_project/manifests/logstash_worker.pp @@ -16,10 +16,11 @@ # class openstack_project::logstash_worker ( $elasticsearch_nodes = [], + $elasticsearch_clients = [], $discover_node = 'elasticsearch01.openstack.org', $sysadmins = [] ) { - $iptables_rule = regsubst ($elasticsearch_nodes, '^(.*)$', '-m state --state NEW -m tcp -p tcp --dport 9200:9400 -s \1 -j ACCEPT') + $iptables_rule = regsubst (flatten([$elasticsearch_nodes, $elasticsearch_clients]) , '^(.*)$', '-m state --state NEW -m tcp -p tcp --dport 9200:9400 -s \1 -j ACCEPT') class { 'openstack_project::server': iptables_public_tcp_ports => [22], iptables_rules6 => $iptables_rule,