Merge "Restart jenkins masters safely once a week"

manifests/site.pp

@@ -217,7 +217,9 @@ node 'puppetmaster.openstack.org' {
     pin_puppet                => '3.6.',
   }
   class { 'openstack_project::puppetmaster':
-    root_rsa_key => hiera('puppetmaster_root_rsa_key', 'XXX'),
+    root_rsa_key     => hiera('puppetmaster_root_rsa_key', 'XXX'),
+    jenkins_api_user => hiera('jenkins_api_user', 'username'),
+    jenkins_api_key  => hiera('jenkins_api_key', 'XXX'),
   }
 }
 

modules/openstack_project/files/ansible/playbooks/restart_jenkins_masters.yaml

@@ -0,0 +1,12 @@
+---
+- hosts: 'jenkins0*.openstack.org'
+  # Treat any errors as fatal so that we don't stop all the jenkins
+  # masters.
+  any_errors_fatal: true
+  tasks:
+    - shell: '/usr/local/jenkins/bin/safe_jenkins_shutdown --url https://{{ ansible_fqdn }}/ --user {{ user }} --password {{ password }}'
+    - service: name=jenkins state=stopped
+      # This is necessary because stopping Jenkins is not reliable.
+      # We allow return code 1 which means no processes found.
+    - shell: 'pkill -9 -U jenkins || [ $? -eq "1" ]'
+    - service: name=jenkins state=started

modules/openstack_project/manifests/puppetmaster.pp

@@ -1,6 +1,8 @@
 # == Class: openstack_project::puppetmaster
 #
 class openstack_project::puppetmaster (
+  $jenkins_api_key,
+  $jenkins_api_user = 'hudson-openstack',
   $root_rsa_key = 'xxx',
   $puppetdb = true,
   $puppetdb_server = 'puppetdb.openstack.org',
@@ -157,6 +159,30 @@ class openstack_project::puppetmaster (
     }
   }
 
+# Jenkins master management
+  cron { 'restartjenkinsmasters':
+    user        => 'root',
+    # Run through all masters onces a week.
+    weekday     => '6',
+    hour        => '0',
+    minute      => '15',
+    command     => "flock -n /var/run/puppet/restart_jenkins_masters.lock ansible-playbook -f 1 /etc/ansible/playbooks/restart_jenkins_masters.yaml --extra-vars 'user=${jenkins_api_user} password=${jenkins_api_key}' >> /var/log/restart_jenkins_masters.log 2>&1",
+  }
+
+  logrotate::file { 'restartjenkinsmasters':
+    ensure  => present,
+    log     => '/var/log/restart_jenkins_masters.log',
+    options => ['compress',
+      'copytruncate',
+      'delaycompress',
+      'missingok',
+      'rotate 7',
+      'daily',
+      'notifempty',
+    ],
+    require => Cron['restartjenkinsmasters'],
+  }
+
 # Playbooks
 #
   file { '/etc/ansible/playbooks':