Switch to dedicated elasticsearch node.

Switch to a large dedicated elasticsearch node as sharing resources
between logstash, kibana, jenkins-log-pusher, and elasticsearch results
in a constrained environment.

Change-Id: I39e6210f2c577429be2cb38aca09111a0f56f9be
Reviewed-on: https://review.openstack.org/30344
Approved: Jeremy Stanley <fungi@yuggoth.org>
Reviewed-by: Jeremy Stanley <fungi@yuggoth.org>
Tested-by: Jenkins

manifests/site.pp

@@ -203,6 +203,12 @@ node 'logstash.openstack.org' {
   }
 }
 
+node 'elasticsearch.openstack.org' {
+  class { 'openstack_project::elasticsearch':
+    sysadmins => hiera('sysadmins'),
+  }
+}
+
 # A machine to run ODSREG in preparation for summits.
 node 'summit.openstack.org' {
   class { 'openstack_project::summit':

modules/logstash/files/elasticsearch.default

@@ -3,7 +3,7 @@
 #ES_GROUP=elasticsearch
 
 # Heap Size (defaults to 256m min, 1g max)
-ES_HEAP_SIZE=4g
+ES_HEAP_SIZE=16g
 
 # Heap new generation
 #ES_HEAP_NEWSIZE=

modules/openstack_project/manifests/elasticsearch.pp

@@ -0,0 +1,45 @@
+# Copyright 2013 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# Elasticsearch server glue class.
+#
+class openstack_project::elasticsearch (
+  $sysadmins = []
+) {
+  $iptables_rule = '-m state --state NEW -m tcp -p tcp --dport 9200:9400 -s logstash.openstack.org -j ACCEPT'
+  class { 'openstack_project::server':
+    iptables_public_tcp_ports => [22],
+    iptables_rules6           => $iptables_rule,
+    iptables_rules4           => $iptables_rule,
+    sysadmins                 => $sysadmins,
+  }
+
+  include logstash::elasticsearch
+
+  cron { 'delete_old_es_indices':
+    user        => 'root',
+    hour        => '5',
+    minute      => '0',
+    command     => 'curl -sS -XDELETE "http://localhost:9200/logstash-`date -d \'last week\' +\%Y.\%m.\%d`/" > /dev/null',
+    environment => 'PATH=/usr/bin:/bin:/usr/sbin:/sbin',
+  }
+
+  cron { 'optimize_old_es_indices':
+    user        => 'root',
+    hour        => '5',
+    minute      => '0',
+    command     => 'curl -sS -XPOST "http://localhost:9200/logstash-`date -d yesterday +\%Y.\%m.\%d`/_optimize?max_num_segments=2" > /dev/null',
+    environment => 'PATH=/usr/bin:/bin:/usr/sbin:/sbin',
+  }
+}

modules/openstack_project/manifests/logstash.pp

@@ -28,11 +28,6 @@ class openstack_project::logstash (
   class { 'logstash::web':
     frontend => 'kibana',
   }
-  include logstash::elasticsearch
-
-  package { 'redis-server':
-    ensure => 'absent',
-  }
 
   package { 'python3':
     ensure => 'present',
@@ -82,20 +77,4 @@ class openstack_project::logstash (
     subscribe  => File['/etc/logstash/jenkins-log-pusher.yaml'],
     require    => File['/etc/init.d/jenkins-log-pusher'],
   }
-
-  cron { 'delete_old_es_indices':
-    user        => 'root',
-    hour        => '5',
-    minute      => '0',
-    command     => 'curl -sS -XDELETE "http://localhost:9200/logstash-`date -d \'last week\' +\%Y.\%m.\%d`/"',
-    environment => 'PATH=/usr/bin:/bin:/usr/sbin:/sbin',
-  }
-
-  cron { 'optimize_old_es_indices':
-    user        => 'root',
-    hour        => '5',
-    minute      => '0',
-    command     => 'curl -sS -XPOST "http://localhost:9200/logstash-`date -d yesterday +\%Y.\%m.\%d`/_optimize?max_num_segments=2"',
-    environment => 'PATH=/usr/bin:/bin:/usr/sbin:/sbin',
-  }
 }

modules/openstack_project/templates/logstash/indexer.conf.erb

@@ -51,6 +51,7 @@ filter {
 
 output {
   elasticsearch {
-    host => "127.0.0.1"
+    host => "elasticsearch.openstack.org"
+    node_name => "logstash-indexer"
   }
 }