diff --git a/modules/logstash/files/logstash-watchdog.sh b/modules/logstash/files/logstash-watchdog.sh new file mode 100644 index 0000000000..c60a288689 --- /dev/null +++ b/modules/logstash/files/logstash-watchdog.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# +# This is a work around for https://logstash.jira.com/browse/LOGSTASH-1951 +# Logstash disconnects from the cluster and will not rejoin under +# its own power. + +ES_ADDRESS=$1 + +JSON_OUT=$(curl -sf "http://${ES_ADDRESS}:9200/_cluster/nodes/${HOSTNAME}") +CURL_RET=$? +RESULT=$(echo $JSON_OUT | jq '.nodes == {}') + +if [ "$CURL_RET" == "0" ] && [ "$RESULT" == "true" ] ; +then + stop --quiet logstash-indexer + start --quiet logstash-indexer +fi diff --git a/modules/logstash/manifests/watchdog.pp b/modules/logstash/manifests/watchdog.pp new file mode 100644 index 0000000000..993b7e5925 --- /dev/null +++ b/modules/logstash/manifests/watchdog.pp @@ -0,0 +1,41 @@ +# Copyright 2013 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# Class to install a simple watchdog for the logstash-indexer service. +# es_api_node is the address to access the elasticsearch api at (should +# be a 'host:port' string). + +class logstash::watchdog ( + $es_api_node = 'localhost' +) { + package { 'jq': + ensure => present, + } + + file { '/usr/local/bin/logstash-watchdog': + ensure => present, + source => 'puppet:///modules/logstash/logstash-watchdog.sh', + replace => true, + owner => 'root', + group => 'root', + mode => '0555', + } + + cron { 'logstash-watchdog': + minute => '*/10', + environment => 'PATH=/bin:/usr/bin:/usr/local/bin', + command => "sleep $((RANDOM\%60)) && /usr/local/bin/logstash-watchdog ${es_api_node}", + require => Service['logstash-indexer'] + } +} diff --git a/modules/openstack_project/manifests/logstash_worker.pp b/modules/openstack_project/manifests/logstash_worker.pp index e707a87a62..e14ea8f09b 100644 --- a/modules/openstack_project/manifests/logstash_worker.pp +++ b/modules/openstack_project/manifests/logstash_worker.pp @@ -39,6 +39,10 @@ class openstack_project::logstash_worker ( conf_template => 'openstack_project/logstash/indexer.conf.erb', } + class { 'logstash::watchdog': + es_api_node => $discover_node, + } + include log_processor log_processor::worker { 'A': config_file => 'puppet:///modules/openstack_project/logstash/jenkins-log-worker.yaml',