diff --git a/manifests/site.pp b/manifests/site.pp index a7cf074791..30c769548e 100644 --- a/manifests/site.pp +++ b/manifests/site.pp @@ -201,32 +201,48 @@ node 'puppet-dashboard.openstack.org' { node 'logstash.openstack.org' { class { 'openstack_project::logstash': - sysadmins => hiera('sysadmins'), - elasticsearch_masters => ['elasticsearch.openstack.org'], - gearman_workers => [ + sysadmins => hiera('sysadmins'), + elasticsearch_nodes => [ + 'elasticsearch.openstack.org', + 'elasticsearch2.openstack.org', + 'elasticsearch3.openstack.org', + ], + gearman_workers => [ 'logstash-worker1.openstack.org', 'logstash-worker2.openstack.org', 'logstash-worker3.openstack.org', ], + discover_node => 'elasticsearch.openstack.org', } } node /^logstash-worker\d+\.openstack\.org$/ { class { 'openstack_project::logstash_worker': - sysadmins => hiera('sysadmins'), - elasticsearch_masters => ['elasticsearch.openstack.org'], + sysadmins => hiera('sysadmins'), + elasticsearch_nodes => [ + 'elasticsearch.openstack.org', + 'elasticsearch2.openstack.org', + 'elasticsearch3.openstack.org', + ], + discover_node => 'elasticsearch.openstack.org', } } -node 'elasticsearch.openstack.org' { +node /^elasticsearch\d*\.openstack\.org$/ { class { 'openstack_project::elasticsearch': - sysadmins => hiera('sysadmins'), - logstash_workers => [ + sysadmins => hiera('sysadmins'), + elasticsearch_nodes => [ + 'elasticsearch.openstack.org', + 'elasticsearch2.openstack.org', + 'elasticsearch3.openstack.org', + ], + elasticsearch_clients => [ 'logstash.openstack.org', 'logstash-worker1.openstack.org', 'logstash-worker2.openstack.org', 'logstash-worker3.openstack.org', ], + discover_node => 'elasticsearch.openstack.org', } } diff --git a/modules/kibana/manifests/init.pp b/modules/kibana/manifests/init.pp index 2987eaf115..0fcfd3c756 100644 --- a/modules/kibana/manifests/init.pp +++ b/modules/kibana/manifests/init.pp @@ -15,7 +15,7 @@ # Class to install kibana frontend to logstash. # class kibana ( - $elasticsearch_host = 'localhost' + $discover_node = 'localhost' ) { group { 'kibana': diff --git a/modules/kibana/templates/config.rb.erb b/modules/kibana/templates/config.rb.erb index 98f0d9fc12..65384ec999 100755 --- a/modules/kibana/templates/config.rb.erb +++ b/modules/kibana/templates/config.rb.erb @@ -7,7 +7,7 @@ module KibanaConfig # Your elastic search server(s). This may be set as an array for round robin # load balancing # Elasticsearch = ["elasticsearch1:9200","elasticsearch2:9200"] - Elasticsearch = "<%= scope.lookupvar("::kibana::elasticsearch_host") %>:9200" + Elasticsearch = "<%= scope.lookupvar("::kibana::discover_node") %>:9200" #Set the Net::HTTP read/open timeouts for the connection to the ES backend ElasticsearchTimeout = 500 diff --git a/modules/logstash/manifests/elasticsearch.pp b/modules/logstash/manifests/elasticsearch.pp index c16d86ed39..68018b9b50 100644 --- a/modules/logstash/manifests/elasticsearch.pp +++ b/modules/logstash/manifests/elasticsearch.pp @@ -14,7 +14,9 @@ # # Class to install elasticsearch. # -class logstash::elasticsearch { +class logstash::elasticsearch ( + discover_node = 'localhost' +) { # install java runtime package { 'java7-runtime-headless': ensure => present, @@ -40,7 +42,7 @@ class logstash::elasticsearch { file { '/etc/elasticsearch/elasticsearch.yml': ensure => present, - source => 'puppet:///modules/logstash/elasticsearch.yml', + content => template('logstash/elasticsearch.yml.erb'), replace => true, owner => 'root', group => 'root', diff --git a/modules/logstash/manifests/web.pp b/modules/logstash/manifests/web.pp index 1e3172e094..a60fe4c304 100644 --- a/modules/logstash/manifests/web.pp +++ b/modules/logstash/manifests/web.pp @@ -18,7 +18,7 @@ class logstash::web ( $vhost_name = $::fqdn, $serveradmin = "webmaster@${::fqdn}", $frontend = 'internal', - $elasticsearch_host = 'localhost', + $discover_node = 'localhost', $proxy_elasticsearch = false ) { include apache @@ -57,7 +57,7 @@ class logstash::web ( 'kibana': { class { 'kibana': - elasticsearch_host => $elasticsearch_host, + discover_node => $discover_node, } $vhost = 'logstash/kibana.vhost.erb' } diff --git a/modules/logstash/files/elasticsearch.yml b/modules/logstash/templates/elasticsearch.yml.erb similarity index 98% rename from modules/logstash/files/elasticsearch.yml rename to modules/logstash/templates/elasticsearch.yml.erb index 8bc5d1a120..1803562ebd 100644 --- a/modules/logstash/files/elasticsearch.yml +++ b/modules/logstash/templates/elasticsearch.yml.erb @@ -37,7 +37,7 @@ # Node names are generated dynamically on startup, so you're relieved # from configuring them manually. You can tie this node to a specific name: # -# node.name: "Franz Kafka" +node.name: "<%= scope.lookupvar("::hostname") %>" # Every node can be configured to allow or deny being eligible as the master, # and to allow or deny to store the data. @@ -139,7 +139,7 @@ index.store.compress.stored: true index.store.compress.tv: true -indices.memory.index_buffer_size: "40%" +indices.memory.index_buffer_size: "33%" #################################### Paths #################################### @@ -184,7 +184,7 @@ indices.memory.index_buffer_size: "40%" # # Set this property to true to lock the memory: # -# bootstrap.mlockall: true +bootstrap.mlockall: true # Make sure that the ES_MIN_MEM and ES_MAX_MEM environment variables are set # to the same value, and that the machine has enough memory to allocate @@ -327,7 +327,7 @@ discovery.zen.ping.multicast.enabled: false # to perform discovery when new nodes (master or data) are started: # # discovery.zen.ping.unicast.hosts: ["host1", "host2:port", "host3[portX-portY]"] -discovery.zen.ping.unicast.hosts: ["localhost"] +discovery.zen.ping.unicast.hosts: ["<%= discover_node %>"] # EC2 discovery allows to use AWS EC2 API in order to perform discovery. # diff --git a/modules/logstash/templates/kibana.vhost.erb b/modules/logstash/templates/kibana.vhost.erb index 63c5e2629d..341c65fbca 100644 --- a/modules/logstash/templates/kibana.vhost.erb +++ b/modules/logstash/templates/kibana.vhost.erb @@ -11,9 +11,9 @@ <% if proxy_elasticsearch == true %> # Proxy for elasticsearch _aliases, .*/_status, and .*/_search. - ProxyPassMatch http://<%= scope.lookupvar("::logstash::web::elasticsearch_host") %>:9200/$1 + ProxyPassMatch http://<%= scope.lookupvar("::logstash::web::discover_node") %>:9200/$1 - ProxyPassReverse /elasticsearch/ http://<%= scope.lookupvar("::logstash::web::elasticsearch_host") %>:9200/ + ProxyPassReverse /elasticsearch/ http://<%= scope.lookupvar("::logstash::web::discover_node") %>:9200/ <% end %> ProxyPass / http://127.0.0.1:5601/ retry=0 diff --git a/modules/openstack_project/manifests/elasticsearch.pp b/modules/openstack_project/manifests/elasticsearch.pp index 420eba6253..f6648df75a 100644 --- a/modules/openstack_project/manifests/elasticsearch.pp +++ b/modules/openstack_project/manifests/elasticsearch.pp @@ -15,10 +15,14 @@ # Elasticsearch server glue class. # class openstack_project::elasticsearch ( - $logstash_workers = [], + $elasticsearch_nodes = [], + $elasticsearch_clients = [], + $discover_node = 'localhost', $sysadmins = [] ) { - $iptables_rule = regsubst ($logstash_workers, '^(.*)$', '-m state --state NEW -m tcp -p tcp --dport 9200:9400 -s \1 -j ACCEPT') + $iptables_nodes_rule = regsubst ($elasticsearch_nodes, '^(.*)$', '-m state --state NEW -m tcp -p tcp --dport 9200:9400 -s \1 -j ACCEPT') + $iptables_clients_rule = regsubst ($elasticsearch_clients, '^(.*)$', '-m state --state NEW -m tcp -p tcp --dport 9200:9400 -s \1 -j ACCEPT') + $iptables_rule = flatten([$iptables_nodes_rule, $iptables_clients_rule]) class { 'openstack_project::server': iptables_public_tcp_ports => [22], iptables_rules6 => $iptables_rule, @@ -26,7 +30,9 @@ class openstack_project::elasticsearch ( sysadmins => $sysadmins, } - include logstash::elasticsearch + class { 'logstash::elasticsearch': + discover_node => $discover_node, + } cron { 'delete_old_es_indices': user => 'root', diff --git a/modules/openstack_project/manifests/logstash.pp b/modules/openstack_project/manifests/logstash.pp index d19b629e0b..7cb74beb5d 100644 --- a/modules/openstack_project/manifests/logstash.pp +++ b/modules/openstack_project/manifests/logstash.pp @@ -15,11 +15,12 @@ # Logstash web frontend glue class. # class openstack_project::logstash ( - $elasticsearch_masters = [], + $elasticsearch_nodes = [], $gearman_workers = [], + $discover_node = 'elasticsearch.openstack.org', $sysadmins = [] ) { - $iptables_es_rule = regsubst ($elasticsearch_masters, '^(.*)$', '-m state --state NEW -m tcp -p tcp --dport 9200:9400 -s \1 -j ACCEPT') + $iptables_es_rule = regsubst ($elasticsearch_nodes, '^(.*)$', '-m state --state NEW -m tcp -p tcp --dport 9200:9400 -s \1 -j ACCEPT') $iptables_gm_rule = regsubst ($gearman_workers, '^(.*)$', '-m state --state NEW -m tcp -p tcp --dport 4730 -s \1 -j ACCEPT') $iptables_rule = flatten([$iptables_es_rule, $iptables_gm_rule]) class { 'openstack_project::server': @@ -31,7 +32,7 @@ class openstack_project::logstash ( class { 'logstash::web': frontend => 'kibana', - elasticsearch_host => 'elasticsearch.openstack.org', + discover_node => $discover_node, proxy_elasticsearch => true, } diff --git a/modules/openstack_project/manifests/logstash_worker.pp b/modules/openstack_project/manifests/logstash_worker.pp index 1b1473e264..9f26289648 100644 --- a/modules/openstack_project/manifests/logstash_worker.pp +++ b/modules/openstack_project/manifests/logstash_worker.pp @@ -15,10 +15,11 @@ # Logstash indexer worker glue class. # class openstack_project::logstash_worker ( - $elasticsearch_masters = [], + $elasticsearch_nodes = [], + $discover_node = 'elasticsearch.openstack.org', $sysadmins = [] ) { - $iptables_rule = regsubst ($elasticsearch_masters, '^(.*)$', '-m state --state NEW -m tcp -p tcp --dport 9200:9400 -s \1 -j ACCEPT') + $iptables_rule = regsubst ($elasticsearch_nodes, '^(.*)$', '-m state --state NEW -m tcp -p tcp --dport 9200:9400 -s \1 -j ACCEPT') class { 'openstack_project::server': iptables_public_tcp_ports => [22], iptables_rules6 => $iptables_rule, diff --git a/modules/openstack_project/templates/logstash/indexer.conf.erb b/modules/openstack_project/templates/logstash/indexer.conf.erb index 14b7686041..d55505923b 100644 --- a/modules/openstack_project/templates/logstash/indexer.conf.erb +++ b/modules/openstack_project/templates/logstash/indexer.conf.erb @@ -127,7 +127,7 @@ filter { output { elasticsearch { - host => "elasticsearch.openstack.org" + host => "<%= scope.lookupvar("::openstack_project::logstash_worker::discover_node") %>" node_name => "<%= scope.lookupvar("::hostname") %>" max_inflight_requests => 512 }