From 2820835473444bb1388a97e3175586212ca3fe62 Mon Sep 17 00:00:00 2001
From: Clark Boylan <clark.boylan@gmail.com>
Date: Fri, 2 Aug 2013 13:22:47 -0700
Subject: [PATCH] Add sixth elasticsearch node.

* manifests/site.pp: Put elasticsearch nodes in a list variable that is
reused.

* modules/logstash/templates/elasticsearch.yml.erb: Update cluster
settings to be more appropriate for the larger cluster size, including
changing the minimum number of master eligible nodes that must be seen
before becoming operational in a cluster. Should help prevent split
brained clusters.

* modules/openstack_project/manifests/cacti.pp: Add new node to cacti
node list.

Change-Id: I9a18422aa56a56b9ad030892a312772d9b6b2a79
---
 manifests/site.pp                             | 49 ++++++-------------
 .../logstash/templates/elasticsearch.yml.erb  |  6 +--
 modules/openstack_project/manifests/cacti.pp  |  1 +
 3 files changed, 18 insertions(+), 38 deletions(-)

diff --git a/manifests/site.pp b/manifests/site.pp
index 07b28adfd5..b4a4b3f271 100644
--- a/manifests/site.pp
+++ b/manifests/site.pp
@@ -224,16 +224,19 @@ node 'puppet-dashboard.openstack.org' {
   }
 }
 
+$elasticsearch_nodes = [
+  'elasticsearch.openstack.org',
+  'elasticsearch2.openstack.org',
+  'elasticsearch3.openstack.org',
+  'elasticsearch4.openstack.org',
+  'elasticsearch5.openstack.org',
+  'elasticsearch6.openstack.org',
+]
+
 node 'logstash.openstack.org' {
   class { 'openstack_project::logstash':
     sysadmins           => hiera('sysadmins'),
-    elasticsearch_nodes => [
-      'elasticsearch.openstack.org',
-      'elasticsearch2.openstack.org',
-      'elasticsearch3.openstack.org',
-      'elasticsearch4.openstack.org',
-      'elasticsearch5.openstack.org',
-    ],
+    elasticsearch_nodes => $elasticsearch_nodes,
     gearman_workers     => [
       'logstash-worker1.openstack.org',
       'logstash-worker2.openstack.org',
@@ -244,26 +247,14 @@ node 'logstash.openstack.org' {
       'logstash-worker7.openstack.org',
       'logstash-worker8.openstack.org',
     ],
-    discover_nodes      => [
-      'elasticsearch.openstack.org:9200',
-      'elasticsearch2.openstack.org:9200',
-      'elasticsearch3.openstack.org:9200',
-      'elasticsearch4.openstack.org:9200',
-      'elasticsearch5.openstack.org:9200',
-    ],
+    discover_nodes      => $elasticsearch_nodes,
   }
 }
 
 node /^logstash-worker\d+\.openstack\.org$/ {
   class { 'openstack_project::logstash_worker':
     sysadmins           => hiera('sysadmins'),
-    elasticsearch_nodes => [
-      'elasticsearch.openstack.org',
-      'elasticsearch2.openstack.org',
-      'elasticsearch3.openstack.org',
-      'elasticsearch4.openstack.org',
-      'elasticsearch5.openstack.org',
-    ],
+    elasticsearch_nodes => $elasticsearch_nodes,
     discover_node       => 'elasticsearch.openstack.org',
   }
 }
@@ -271,13 +262,7 @@ node /^logstash-worker\d+\.openstack\.org$/ {
 node /^elasticsearch\d*\.openstack\.org$/ {
   class { 'openstack_project::elasticsearch':
     sysadmins             => hiera('sysadmins'),
-    elasticsearch_nodes   => [
-      'elasticsearch.openstack.org',
-      'elasticsearch2.openstack.org',
-      'elasticsearch3.openstack.org',
-      'elasticsearch4.openstack.org',
-      'elasticsearch5.openstack.org',
-    ],
+    elasticsearch_nodes   => $elasticsearch_nodes,
     elasticsearch_clients => [
       'logstash.openstack.org',
       'logstash-worker1.openstack.org',
@@ -289,13 +274,7 @@ node /^elasticsearch\d*\.openstack\.org$/ {
       'logstash-worker7.openstack.org',
       'logstash-worker8.openstack.org',
     ],
-    discover_nodes        => [
-      'elasticsearch.openstack.org',
-      'elasticsearch2.openstack.org',
-      'elasticsearch3.openstack.org',
-      'elasticsearch4.openstack.org',
-      'elasticsearch5.openstack.org',
-    ],
+    discover_nodes        => $elasticsearch_nodes,
   }
 }
 
diff --git a/modules/logstash/templates/elasticsearch.yml.erb b/modules/logstash/templates/elasticsearch.yml.erb
index f07cf9327f..ed89730c23 100644
--- a/modules/logstash/templates/elasticsearch.yml.erb
+++ b/modules/logstash/templates/elasticsearch.yml.erb
@@ -255,7 +255,7 @@ bootstrap.mlockall: true
 
 # Allow recovery process after N nodes in a cluster are up:
 #
-gateway.recover_after_nodes: 4
+gateway.recover_after_nodes: 5
 
 # Set the timeout to initiate the recovery process, once the N nodes
 # from previous setting are up (accepts time value):
@@ -266,7 +266,7 @@ gateway.recover_after_time: 5m
 # are up (and recover_after_nodes is met), begin recovery process immediately
 # (without waiting for recover_after_time to expire):
 #
-gateway.expected_nodes: 5
+gateway.expected_nodes: 6
 
 
 ############################# Recovery Throttling #############################
@@ -304,7 +304,7 @@ gateway.expected_nodes: 5
 # operational within the cluster. Set this option to a higher value (2-4)
 # for large clusters (>3 nodes):
 #
-discovery.zen.minimum_master_nodes: 2
+discovery.zen.minimum_master_nodes: 4
 
 # Set the time to wait for ping responses from other nodes when discovering.
 # Set this option to a higher value on a slow or congested network
diff --git a/modules/openstack_project/manifests/cacti.pp b/modules/openstack_project/manifests/cacti.pp
index b92b668594..db0c040394 100644
--- a/modules/openstack_project/manifests/cacti.pp
+++ b/modules/openstack_project/manifests/cacti.pp
@@ -64,6 +64,7 @@ class openstack_project::cacti (
     'elasticsearch3.openstack.org',
     'elasticsearch4.openstack.org',
     'elasticsearch5.openstack.org',
+    'elasticsearch6.openstack.org',
     'etherpad.openstack.org',
     'graphite.openstack.org',
     'jenkins.openstack.org',