Better elasticsearch cluster settings.
Use mutliple discover nodes to determine elasticsearch cluster membership. Put a timeout on recovery starting instead of the default to recovery immediately. Describe cluster topology in elasticsearch yaml config so that it can make smarter decisions. Round robin kibana requests across each discover node. Change-Id: I08ef9dd158ddf6a6ce01dfb2050626f543d45b10 Reviewed-on: https://review.openstack.org/34106 Reviewed-by: Jeremy Stanley <fungi@yuggoth.org> Reviewed-by: James E. Blair <corvus@inaugust.com> Approved: James E. Blair <corvus@inaugust.com> Tested-by: Jenkins
This commit is contained in:
Clark Boylan
Jenkins
parent
cfd1df90d9
commit
443539b7fa
@ -214,7 +214,11 @@ node 'logstash.openstack.org' {
|
||||
'logstash-worker4.openstack.org',
|
||||
'logstash-worker5.openstack.org',
|
||||
],
|
||||
discover_node => 'elasticsearch.openstack.org',
|
||||
discover_nodes => [
|
||||
'elasticsearch.openstack.org:9200',
|
||||
'elasticsearch2.openstack.org:9200',
|
||||
'elasticsearch3.openstack.org:9200',
|
||||
],
|
||||
}
|
||||
}
|
||||
|
||||
@ -246,7 +250,11 @@ node /^elasticsearch\d*\.openstack\.org$/ {
|
||||
'logstash-worker4.openstack.org',
|
||||
'logstash-worker5.openstack.org',
|
||||
],
|
||||
discover_node => 'elasticsearch.openstack.org',
|
||||
discover_nodes => [
|
||||
'elasticsearch.openstack.org',
|
||||
'elasticsearch2.openstack.org',
|
||||
'elasticsearch3.openstack.org',
|
||||
],
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -15,7 +15,7 @@
|
||||
# Class to install kibana frontend to logstash.
|
||||
#
|
||||
class kibana (
|
||||
$discover_node = 'localhost'
|
||||
$discover_nodes = ['localhost:9200']
|
||||
) {
|
||||
|
||||
group { 'kibana':
|
||||
|
||||
@ -7,7 +7,7 @@ module KibanaConfig
|
||||
# Your elastic search server(s). This may be set as an array for round robin
|
||||
# load balancing
|
||||
# Elasticsearch = ["elasticsearch1:9200","elasticsearch2:9200"]
|
||||
Elasticsearch = "<%= scope.lookupvar("::kibana::discover_node") %>:9200"
|
||||
Elasticsearch = ["<%= scope.lookupvar("::kibana::discover_nodes").join("\",\"") %>"]
|
||||
|
||||
#Set the Net::HTTP read/open timeouts for the connection to the ES backend
|
||||
ElasticsearchTimeout = 500
|
||||
|
||||
@ -17,7 +17,7 @@ ES_HEAP_SIZE=16g
|
||||
# Maximum locked memory size. Set to "unlimited" if you use the
|
||||
# bootstrap.mlockall option in elasticsearch.yml. You must also set
|
||||
# ES_HEAP_SIZE.
|
||||
#MAX_LOCKED_MEMORY=unlimited
|
||||
MAX_LOCKED_MEMORY=unlimited
|
||||
|
||||
# ElasticSearch log directory
|
||||
#LOG_DIR=/var/log/elasticsearch
|
||||
|
||||
@ -15,7 +15,7 @@
|
||||
# Class to install elasticsearch.
|
||||
#
|
||||
class logstash::elasticsearch (
|
||||
discover_node = 'localhost'
|
||||
discover_nodes = ['localhost']
|
||||
) {
|
||||
# install java runtime
|
||||
package { 'java7-runtime-headless':
|
||||
|
||||
@ -18,7 +18,7 @@ class logstash::web (
|
||||
$vhost_name = $::fqdn,
|
||||
$serveradmin = "webmaster@${::fqdn}",
|
||||
$frontend = 'internal',
|
||||
$discover_node = 'localhost',
|
||||
$discover_nodes = ['localhost:9200'],
|
||||
$proxy_elasticsearch = false
|
||||
) {
|
||||
include apache
|
||||
@ -57,7 +57,7 @@ class logstash::web (
|
||||
|
||||
'kibana': {
|
||||
class { 'kibana':
|
||||
discover_node => $discover_node,
|
||||
discover_nodes => $discover_nodes,
|
||||
}
|
||||
$vhost = 'logstash/kibana.vhost.erb'
|
||||
}
|
||||
|
||||
@ -255,18 +255,18 @@ bootstrap.mlockall: true
|
||||
|
||||
# Allow recovery process after N nodes in a cluster are up:
|
||||
#
|
||||
# gateway.recover_after_nodes: 1
|
||||
gateway.recover_after_nodes: 2
|
||||
|
||||
# Set the timeout to initiate the recovery process, once the N nodes
|
||||
# from previous setting are up (accepts time value):
|
||||
#
|
||||
# gateway.recover_after_time: 5m
|
||||
gateway.recover_after_time: 5m
|
||||
|
||||
# Set how many nodes are expected in this cluster. Once these N nodes
|
||||
# are up (and recover_after_nodes is met), begin recovery process immediately
|
||||
# (without waiting for recover_after_time to expire):
|
||||
#
|
||||
# gateway.expected_nodes: 2
|
||||
gateway.expected_nodes: 3
|
||||
|
||||
|
||||
############################# Recovery Throttling #############################
|
||||
@ -304,7 +304,7 @@ bootstrap.mlockall: true
|
||||
# operational within the cluster. Set this option to a higher value (2-4)
|
||||
# for large clusters (>3 nodes):
|
||||
#
|
||||
# discovery.zen.minimum_master_nodes: 1
|
||||
discovery.zen.minimum_master_nodes: 2
|
||||
|
||||
# Set the time to wait for ping responses from other nodes when discovering.
|
||||
# Set this option to a higher value on a slow or congested network
|
||||
@ -327,7 +327,9 @@ discovery.zen.ping.multicast.enabled: false
|
||||
# to perform discovery when new nodes (master or data) are started:
|
||||
#
|
||||
# discovery.zen.ping.unicast.hosts: ["host1", "host2:port", "host3[portX-portY]"]
|
||||
discovery.zen.ping.unicast.hosts: ["<%= discover_node %>"]
|
||||
#
|
||||
# Make all master eligible nodes discover nodes.
|
||||
discovery.zen.ping.unicast.hosts: ["<%= discover_nodes.join("\", \"") %>"]
|
||||
|
||||
# EC2 discovery allows to use AWS EC2 API in order to perform discovery.
|
||||
#
|
||||
|
||||
@ -11,9 +11,9 @@
|
||||
<% if proxy_elasticsearch == true %>
|
||||
# Proxy for elasticsearch _aliases, .*/_status, and .*/_search.
|
||||
<LocationMatch "^/elasticsearch/(_aliases|.*/_status|.*/_search)$">
|
||||
ProxyPassMatch http://<%= scope.lookupvar("::logstash::web::discover_node") %>:9200/$1
|
||||
ProxyPassMatch http://<%= scope.lookupvar("::logstash::web::discover_nodes")[0] %>/$1
|
||||
</LocationMatch>
|
||||
ProxyPassReverse /elasticsearch/ http://<%= scope.lookupvar("::logstash::web::discover_node") %>:9200/
|
||||
ProxyPassReverse /elasticsearch/ http://<%= scope.lookupvar("::logstash::web::discover_nodes")[0] %>/
|
||||
<% end %>
|
||||
|
||||
ProxyPass / http://127.0.0.1:5601/ retry=0
|
||||
|
||||
@ -17,7 +17,7 @@
|
||||
class openstack_project::elasticsearch (
|
||||
$elasticsearch_nodes = [],
|
||||
$elasticsearch_clients = [],
|
||||
$discover_node = 'localhost',
|
||||
$discover_nodes = ['localhost'],
|
||||
$sysadmins = []
|
||||
) {
|
||||
$iptables_nodes_rule = regsubst ($elasticsearch_nodes, '^(.*)$', '-m state --state NEW -m tcp -p tcp --dport 9200:9400 -s \1 -j ACCEPT')
|
||||
@ -31,7 +31,7 @@ class openstack_project::elasticsearch (
|
||||
}
|
||||
|
||||
class { 'logstash::elasticsearch':
|
||||
discover_node => $discover_node,
|
||||
discover_nodes => $discover_nodes,
|
||||
}
|
||||
|
||||
cron { 'delete_old_es_indices':
|
||||
|
||||
@ -17,7 +17,7 @@
|
||||
class openstack_project::logstash (
|
||||
$elasticsearch_nodes = [],
|
||||
$gearman_workers = [],
|
||||
$discover_node = 'elasticsearch.openstack.org',
|
||||
$discover_nodes = ['elasticsearch.openstack.org:9200'],
|
||||
$sysadmins = []
|
||||
) {
|
||||
$iptables_es_rule = regsubst ($elasticsearch_nodes, '^(.*)$', '-m state --state NEW -m tcp -p tcp --dport 9200:9400 -s \1 -j ACCEPT')
|
||||
@ -32,7 +32,7 @@ class openstack_project::logstash (
|
||||
|
||||
class { 'logstash::web':
|
||||
frontend => 'kibana',
|
||||
discover_node => $discover_node,
|
||||
discover_nodes => $discover_nodes,
|
||||
proxy_elasticsearch => true,
|
||||
}
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user