From 920fc0c42b01aaac549eff1e7612f5dd4a4c1eee Mon Sep 17 00:00:00 2001 From: "James E. Blair" Date: Tue, 17 Dec 2013 15:40:38 -0800 Subject: [PATCH] Refactor logstash_worker into log_processor module Separate the jenkins log client and worker bits into a new module called log_processor with ::client and ::worker classes. Instantiate two workers on each logstash worker node. Change-Id: I7cfec410983c25633e6b555f22a85e9435884cfb --- .../files}/classify-log.crm | 0 .../files}/jenkins-log-client.init | 0 .../files}/log-gearman-client.py | 0 .../files}/log-gearman-worker.py | 0 modules/log_processor/manifests/client.pp | 63 +++++++++++ modules/log_processor/manifests/init.pp | 88 +++++++++++++++ modules/log_processor/manifests/worker.pp | 67 ++++++++++++ .../templates/jenkins-log-worker.init.erb} | 4 +- .../openstack_project/manifests/logstash.pp | 74 +------------ .../manifests/logstash_worker.pp | 101 +----------------- 10 files changed, 228 insertions(+), 169 deletions(-) rename modules/{openstack_project/files/logstash => log_processor/files}/classify-log.crm (100%) rename modules/{openstack_project/files/logstash => log_processor/files}/jenkins-log-client.init (100%) rename modules/{openstack_project/files/logstash => log_processor/files}/log-gearman-client.py (100%) rename modules/{openstack_project/files/logstash => log_processor/files}/log-gearman-worker.py (100%) create mode 100644 modules/log_processor/manifests/client.pp create mode 100644 modules/log_processor/manifests/init.pp create mode 100644 modules/log_processor/manifests/worker.pp rename modules/{openstack_project/files/logstash/jenkins-log-worker.init => log_processor/templates/jenkins-log-worker.init.erb} (95%) diff --git a/modules/openstack_project/files/logstash/classify-log.crm b/modules/log_processor/files/classify-log.crm similarity index 100% rename from modules/openstack_project/files/logstash/classify-log.crm rename to modules/log_processor/files/classify-log.crm diff --git a/modules/openstack_project/files/logstash/jenkins-log-client.init b/modules/log_processor/files/jenkins-log-client.init similarity index 100% rename from modules/openstack_project/files/logstash/jenkins-log-client.init rename to modules/log_processor/files/jenkins-log-client.init diff --git a/modules/openstack_project/files/logstash/log-gearman-client.py b/modules/log_processor/files/log-gearman-client.py similarity index 100% rename from modules/openstack_project/files/logstash/log-gearman-client.py rename to modules/log_processor/files/log-gearman-client.py diff --git a/modules/openstack_project/files/logstash/log-gearman-worker.py b/modules/log_processor/files/log-gearman-worker.py similarity index 100% rename from modules/openstack_project/files/logstash/log-gearman-worker.py rename to modules/log_processor/files/log-gearman-worker.py diff --git a/modules/log_processor/manifests/client.pp b/modules/log_processor/manifests/client.pp new file mode 100644 index 0000000000..15d340fc8e --- /dev/null +++ b/modules/log_processor/manifests/client.pp @@ -0,0 +1,63 @@ +# Copyright 2012-2013 Hewlett-Packard Development Company, L.P. +# Copyright 2013 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# == Class: log_processor::client +# +class log_processor::client ( + $config_file, +) { + + file { '/etc/logstash/jenkins-log-client.yaml': + ensure => present, + owner => 'root', + group => 'root', + mode => '0555', + source => $config_file, + require => File['/etc/logstash'], + } + + file { '/etc/init.d/jenkins-log-client': + ensure => present, + owner => 'root', + group => 'root', + mode => '0555', + source => 'puppet:///modules/log_processor/jenkins-log-client.init', + require => [ + File['/usr/local/bin/log-gearman-client.py'], + File['/etc/logstash/jenkins-log-client.yaml'], + ], + } + + service { 'jenkins-log-client': + enable => true, + hasrestart => true, + subscribe => File['/etc/logstash/jenkins-log-client.yaml'], + require => File['/etc/init.d/jenkins-log-client'], + } + + include logrotate + logrotate::file { 'log-client-debug.log': + log => '/var/log/logstash/log-client-debug.log', + options => [ + 'compress', + 'copytruncate', + 'missingok', + 'rotate 7', + 'daily', + 'notifempty', + ], + require => Service['jenkins-log-client'], + } +} diff --git a/modules/log_processor/manifests/init.pp b/modules/log_processor/manifests/init.pp new file mode 100644 index 0000000000..e69f9d010d --- /dev/null +++ b/modules/log_processor/manifests/init.pp @@ -0,0 +1,88 @@ +# Copyright 2012-2013 Hewlett-Packard Development Company, L.P. +# Copyright 2013 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# == Class: log_processor +# +class log_processor ( +) { + package { 'python-daemon': + ensure => present, + } + + package { 'python-zmq': + ensure => present, + } + + package { 'python-yaml': + ensure => present, + } + + package { 'crm114': + ensure => present, + } + + include pip + package { 'gear': + ensure => latest, + provider => 'pip', + require => Class['pip'], + } + + file { '/var/lib/crm114': + ensure => directory, + owner => 'logstash', + group => 'logstash', + require => User['logstash'], + } + + file { '/usr/local/bin/classify-log.crm': + ensure => present, + owner => 'root', + group => 'root', + mode => '0755', + source => 'puppet:///modules/log_processor/classify-log.crm', + require => [ + Package['crm114'], + ], + } + + file { '/usr/local/bin/log-gearman-client.py': + ensure => present, + owner => 'root', + group => 'root', + mode => '0755', + source => 'puppet:///modules/log_processor/log-gearman-client.py', + require => [ + Package['python-daemon'], + Package['python-zmq'], + Package['python-yaml'], + Package['gear'], + ], + } + + file { '/usr/local/bin/log-gearman-worker.py': + ensure => present, + owner => 'root', + group => 'root', + mode => '0755', + source => 'puppet:///modules/log_processor/log-gearman-worker.py', + require => [ + Package['python-daemon'], + Package['python-zmq'], + Package['python-yaml'], + Package['gear'], + ], + } +} diff --git a/modules/log_processor/manifests/worker.pp b/modules/log_processor/manifests/worker.pp new file mode 100644 index 0000000000..ec5653bdac --- /dev/null +++ b/modules/log_processor/manifests/worker.pp @@ -0,0 +1,67 @@ +# Copyright 2012-2013 Hewlett-Packard Development Company, L.P. +# Copyright 2013 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# == Class: log_processor::worker +# +define log_processor::worker ( + $config_file, +) { + $suffix = "-${name}" + + file { "/etc/logstash/jenkins-log-worker${suffix}.yaml": + ensure => present, + owner => 'root', + group => 'root', + mode => '0555', + source => $config_file, + require => Class['logstash::indexer'], + } + + file { "/etc/init.d/jenkins-log-worker${suffix}": + ensure => present, + owner => 'root', + group => 'root', + mode => '0555', + content => template('log_processor/jenkins-log-worker.init.erb'), + require => [ + File['/usr/local/bin/log-gearman-worker.py'], + File["/etc/logstash/jenkins-log-worker${suffix}.yaml"], + ], + } + + service { "jenkins-log-worker${suffix}": + enable => true, + hasrestart => true, + subscribe => File["/etc/logstash/jenkins-log-worker${suffix}.yaml"], + require => [ + Class['logstash::indexer'], + File["/etc/init.d/jenkins-log-worker${suffix}"], + ], + } + + include logrotate + logrotate::file { "log-worker${suffix}-debug.log": + log => "/var/log/logstash/log-worker${suffix}-debug.log", + options => [ + 'compress', + 'copytruncate', + 'missingok', + 'rotate 7', + 'daily', + 'notifempty', + ], + require => Service["jenkins-log-worker${suffix}"], + } +} diff --git a/modules/openstack_project/files/logstash/jenkins-log-worker.init b/modules/log_processor/templates/jenkins-log-worker.init.erb similarity index 95% rename from modules/openstack_project/files/logstash/jenkins-log-worker.init rename to modules/log_processor/templates/jenkins-log-worker.init.erb index c8ef3eb361..152a97b1f8 100755 --- a/modules/openstack_project/files/logstash/jenkins-log-worker.init +++ b/modules/log_processor/templates/jenkins-log-worker.init.erb @@ -14,10 +14,10 @@ # PATH should only include /usr/* if it runs after the mountnfs.sh script PATH=/sbin:/usr/sbin:/bin:/usr/bin DESC="Jenkins Log Worker" -NAME=jenkins-log-worker +NAME=jenkins-log-worker<%= suffix %> DAEMON=/usr/local/bin/log-gearman-worker.py PIDFILE=/var/run/$NAME/$NAME.pid -DAEMON_ARGS="-c /etc/logstash/jenkins-log-worker.yaml -d /var/log/logstash/log-worker-debug.log -p $PIDFILE" +DAEMON_ARGS="-c /etc/logstash/jenkins-log-worker<%= suffix %>.yaml -d /var/log/logstash/log-worker<%= suffix %>-debug.log -p $PIDFILE" SCRIPTNAME=/etc/init.d/$NAME USER=logstash diff --git a/modules/openstack_project/manifests/logstash.pp b/modules/openstack_project/manifests/logstash.pp index e9956fa17b..950e19522b 100644 --- a/modules/openstack_project/manifests/logstash.pp +++ b/modules/openstack_project/manifests/logstash.pp @@ -43,78 +43,10 @@ class openstack_project::logstash ( proxy_elasticsearch => true, } - package { 'python-daemon': - ensure => present, - } + class { 'log_processor': } - package { 'python-zmq': - ensure => present, - } - - package { 'python-yaml': - ensure => present, - } - - include pip - package { 'gear': - ensure => latest, - provider => 'pip', - require => Class['pip'], - } - - file { '/usr/local/bin/log-gearman-client.py': - ensure => present, - owner => 'root', - group => 'root', - mode => '0755', - source => 'puppet:///modules/openstack_project/logstash/log-gearman-client.py', - require => [ - Package['python-daemon'], - Package['python-zmq'], - Package['python-yaml'], - Package['gear'], - ], - } - - file { '/etc/logstash/jenkins-log-client.yaml': - ensure => present, - owner => 'root', - group => 'root', - mode => '0555', - source => 'puppet:///modules/openstack_project/logstash/jenkins-log-client.yaml', - } - - file { '/etc/init.d/jenkins-log-client': - ensure => present, - owner => 'root', - group => 'root', - mode => '0555', - source => 'puppet:///modules/openstack_project/logstash/jenkins-log-client.init', - require => [ - File['/usr/local/bin/log-gearman-client.py'], - File['/etc/logstash/jenkins-log-client.yaml'], - ], - } - - service { 'jenkins-log-client': - enable => true, - hasrestart => true, - subscribe => File['/etc/logstash/jenkins-log-client.yaml'], - require => File['/etc/init.d/jenkins-log-client'], - } - - include logrotate - logrotate::file { 'log-client-debug.log': - log => '/var/log/logstash/log-client-debug.log', - options => [ - 'compress', - 'copytruncate', - 'missingok', - 'rotate 7', - 'daily', - 'notifempty', - ], - require => Service['jenkins-log-client'], + class { 'log_processor::client': + config_file => 'puppet:///modules/openstack_project/logstash/jenkins-log-client.yaml', } class { 'elastic_recheck::bot': diff --git a/modules/openstack_project/manifests/logstash_worker.pp b/modules/openstack_project/manifests/logstash_worker.pp index bf10f8f93e..37b35d6df1 100644 --- a/modules/openstack_project/manifests/logstash_worker.pp +++ b/modules/openstack_project/manifests/logstash_worker.pp @@ -31,102 +31,11 @@ class openstack_project::logstash_worker ( conf_template => 'openstack_project/logstash/indexer.conf.erb', } - package { 'python-daemon': - ensure => present, + include log_processor + log_processor::worker { 'A': + config_file => 'puppet:///modules/openstack_project/logstash/jenkins-log-worker.yaml', } - - package { 'python-zmq': - ensure => present, - } - - package { 'python-yaml': - ensure => present, - } - - package { 'crm114': - ensure => present, - } - - include pip - package { 'gear': - ensure => latest, - provider => 'pip', - require => Class['pip'], - } - - file { '/var/lib/crm114': - ensure => directory, - owner => 'logstash', - group => 'logstash', - require => User['logstash'], - } - - file { '/usr/local/bin/classify-log.crm': - ensure => present, - owner => 'root', - group => 'root', - mode => '0755', - source => 'puppet:///modules/openstack_project/logstash/classify-log.crm', - require => [ - Package['crm114'], - ], - } - - file { '/usr/local/bin/log-gearman-worker.py': - ensure => present, - owner => 'root', - group => 'root', - mode => '0755', - source => 'puppet:///modules/openstack_project/logstash/log-gearman-worker.py', - require => [ - Package['python-daemon'], - Package['python-zmq'], - Package['python-yaml'], - Package['gear'], - ], - } - - file { '/etc/logstash/jenkins-log-worker.yaml': - ensure => present, - owner => 'root', - group => 'root', - mode => '0555', - source => 'puppet:///modules/openstack_project/logstash/jenkins-log-worker.yaml', - } - - file { '/etc/init.d/jenkins-log-worker': - ensure => present, - owner => 'root', - group => 'root', - mode => '0555', - source => 'puppet:///modules/openstack_project/logstash/jenkins-log-worker.init', - require => [ - File['/usr/local/bin/log-gearman-worker.py'], - File['/etc/logstash/jenkins-log-worker.yaml'], - ], - } - - service { 'jenkins-log-worker': - enable => true, - hasrestart => true, - subscribe => File['/etc/logstash/jenkins-log-worker.yaml'], - require => [ - Class['logstash::indexer'], - File['/etc/init.d/jenkins-log-worker'], - ], - } - - include logrotate - logrotate::file { 'log-worker-debug.log': - log => '/var/log/logstash/log-worker-debug.log', - options => [ - 'compress', - 'copytruncate', - 'missingok', - 'rotate 7', - 'daily', - 'notifempty', - ], - require => Service['jenkins-log-worker'], + log_processor::worker { 'B': + config_file => 'puppet:///modules/openstack_project/logstash/jenkins-log-worker.yaml', } }