Make Jenkins Jobs more modular

Each job has a manifest in the jenkins_jobs/manifests/jobs directory
Jobs are grouped in python_jobs and generic_jobs
The job groups or individual jobs can be added to a site manifest

Also fixes bad XML output, python27 job and bug 987428

Change-Id: I1ebc889cee08b080deb6ba46aee69ab2ae0bb6b6

manifests/site.pp

@@ -150,8 +150,13 @@ node "jenkins.openstack.org" {
   }
   class { "jenkins_jobs":
     site => "openstack",
-    projects => ["python-glanceclient"]
   }
+  jenkins_jobs::python_jobs { "python-glanceclient":
+    site => "openstack",
+    project => "python-glanceclient",
+    node_group => "oneiric"
+  }
+  
 }
 
 node "jenkins-dev.openstack.org" {

manifests/stackforge.pp

@@ -64,7 +64,18 @@ node "jenkins.stackforge.org" {
 
   class { "jenkins_jobs":
     site => "stackforge",
-    projects => ["reddwarf"]
+  }
+
+  jenkins_jobs::python_jobs { "reddwarf-natty":
+    site => "stackforge",
+    project => "reddwarf",
+    node_group => "natty"
+  }
+
+  jenkins_jobs::generic_jobs { "reddwarf":
+    site => "stackforge",
+    project => "reddwarf",
+    node_group => "oneiric"
   }
 }
 

modules/jenkins_jobs/lib/puppet/parser/functions/builder.rb

@@ -0,0 +1,13 @@
+module Puppet::Parser::Functions
+  newfunction(:builder, :type => :rvalue) do |args|
+    file = "jenkins_jobs/builder_" + args[0] + ".xml.erb"
+    wrapper = Puppet::Parser::TemplateWrapper.new(self)
+    wrapper.file = file
+      begin
+        wrapper.result
+      rescue => detail
+        raise Puppet::ParseError,
+          "Failed to parse template #{file}: #{detail}"
+      end
+  end
+end

modules/jenkins_jobs/lib/puppet/parser/functions/misc.rb

@@ -0,0 +1,13 @@
+module Puppet::Parser::Functions
+  newfunction(:misc, :type => :rvalue) do |args|
+    file = "jenkins_jobs/" + args[0] + ".xml.erb"
+    wrapper = Puppet::Parser::TemplateWrapper.new(self)
+    wrapper.file = file
+      begin
+        wrapper.result
+      rescue => detail
+        raise Puppet::ParseError,
+          "Failed to parse template #{file}: #{detail}"
+      end
+  end
+end

modules/jenkins_jobs/lib/puppet/parser/functions/publisher.rb

@@ -0,0 +1,13 @@
+module Puppet::Parser::Functions
+  newfunction(:publisher, :type => :rvalue) do |args|
+    file = "jenkins_jobs/publisher_" + args[0] + ".xml.erb"
+    wrapper = Puppet::Parser::TemplateWrapper.new(self)
+    wrapper.file = file
+      begin
+        wrapper.result
+      rescue => detail
+        raise Puppet::ParseError,
+          "Failed to parse template #{file}: #{detail}"
+      end
+  end
+end

modules/jenkins_jobs/lib/puppet/parser/functions/scm.rb

@@ -0,0 +1,13 @@
+module Puppet::Parser::Functions
+  newfunction(:scm, :type => :rvalue) do |args|
+    file = "jenkins_jobs/scm_" + args[0] + ".xml.erb"
+    wrapper = Puppet::Parser::TemplateWrapper.new(self)
+    wrapper.file = file
+      begin
+        wrapper.result
+      rescue => detail
+        raise Puppet::ParseError,
+          "Failed to parse template #{file}: #{detail}"
+      end
+  end
+end

modules/jenkins_jobs/lib/puppet/parser/functions/trigger.rb

@@ -0,0 +1,13 @@
+module Puppet::Parser::Functions
+  newfunction(:trigger, :type => :rvalue) do |args|
+    file = "jenkins_jobs/trigger_" + args[0] + ".xml.erb"
+    wrapper = Puppet::Parser::TemplateWrapper.new(self)
+    wrapper.file = file
+      begin
+        wrapper.result
+      rescue => detail
+        raise Puppet::ParseError,
+          "Failed to parse template #{file}: #{detail}"
+      end
+  end
+end

modules/jenkins_jobs/manifests/add_jobs.pp

@@ -1,85 +0,0 @@
-define jenkins_jobs::add_jobs($site) {
-  $project = $name
-
-  jenkins_jobs::job { "${name}-coverage":
-    site => "${site}",
-    project => "${name}",
-    job => "coverage",
-    logrotate => template("jenkins_jobs/logrotate.xml.erb"),
-    builders => [template("jenkins_jobs/builder_copy_bundle.xml.erb"), template("jenkins_jobs/builder_coverage.xml.erb")],
-    publishers => template("jenkins_jobs/publisher_coverage.xml.erb"),
-    triggers => template("jenkins_jobs/trigger_timed_15mins.xml.erb"),
-    scm => template("jenkins_jobs/scm_git.xml.erb")
-  }
-
-  jenkins_jobs::job { "${name}-docs":
-    site => "${site}",
-    project => "${name}",
-    job => "docs",
-    triggers => template("jenkins_jobs/trigger_timed_15mins.xml.erb"),
-    builders => [template("jenkins_jobs/builder_copy_bundle.xml.erb"), template("jenkins_jobs/builder_docs.xml.erb")],
-    publishers => template("jenkins_jobs/publisher_docs.xml.erb"),
-    scm => template("jenkins_jobs/scm_git.xml.erb")
-  }
-
-  jenkins_jobs::job { "gate-${name}-merge":
-    site => "${site}",
-    project => "${name}",
-    job => "merge",
-    triggers => template("jenkins_jobs/trigger_gerrit_comment.xml.erb"),
-    builders => template("jenkins_jobs/builder_gerrit_git_prep.xml.erb")
-  }
-
-  jenkins_jobs::job { "gate-${name}-pep8":
-    site => "${site}",
-    project => "${name}",
-    job => "pep8",
-    triggers => template("jenkins_jobs/trigger_gerrit_comment.xml.erb"),
-    builders => [template("jenkins_jobs/builder_gerrit_git_prep.xml.erb"), template("jenkins_jobs/builder_copy_bundle.xml.erb"), template("jenkins_jobs/builder_pep8.xml.erb")],
-    publishers => template("jenkins_jobs/publisher_pep8.xml.erb")
-  }
-
-  jenkins_jobs::job { "${name}-ppa":
-    site => "${site}",
-    project => "${name}",
-    job => "ppa",
-    builders => template("jenkins_jobs/builder_ppa.xml.erb"),
-    publishers => template("jenkins_jobs/publisher_ppa.xml.erb")
-  }
-
-  jenkins_jobs::job { "gate-${name}-python26":
-    site => "${site}",
-    project => "${name}",
-    job => "python26",
-    triggers => template("jenkins_jobs/trigger_gerrit_comment.xml.erb"),
-    builders => [template("jenkins_jobs/builder_gerrit_git_prep.xml.erb"), template("jenkins_jobs/builder_copy_bundle.xml.erb"), template("jenkins_jobs/builder_python26.xml.erb")],
-  }
-
-  jenkins_jobs::job { "gate-${name}-python27":
-    site => "${site}",
-    project => "${name}",
-    job => "python27",
-    triggers => template("jenkins_jobs/trigger_gerrit_comment.xml.erb"),
-    builders => [template("jenkins_jobs/builder_gerrit_git_prep.xml.erb"), template("jenkins_jobs/builder_copy_bundle.xml.erb"), template("jenkins_jobs/builder_python27.xml.erb")],
-  }
-
-  jenkins_jobs::job { "${name}-tarball":
-    site => "${site}",
-    project => "${name}",
-    job => "tarball",
-    triggers => template("jenkins_jobs/trigger_gerrit_ref_updated.xml.erb"),
-    builders => [template("jenkins_jobs/builder_gerrit_git_prep.xml.erb"), template("jenkins_jobs/builder_copy_bundle.xml.erb"), template("jenkins_jobs/builder_tarball.xml.erb")],
-    publishers => template("jenkins_jobs/publisher_tarball.xml.erb")
-  }
-
-  jenkins_jobs::job { "${name}-venv":
-    site => "${site}",
-    project => "${name}",
-    job => "venv",
-    triggers => template("jenkins_jobs/trigger_timed_midnight.xml.erb"),
-    builders => template("jenkins_jobs/builder_venv.xml.erb"),
-    publishers => template("jenkins_jobs/publisher_venv.xml.erb"),
-    scm => template("jenkins_jobs/scm_git.xml.erb")
-  }
-
-}

modules/jenkins_jobs/manifests/build_job.pp

@@ -0,0 +1,36 @@
+define jenkins_jobs::build_job($site, $project, $job, $node_group, $triggers="", $builders, $publishers="", $logrotate="", $scm="", $trigger_branches="") {
+
+  file { "/var/lib/jenkins/jobs/${name}":
+    ensure => directory,
+    owner => 'jenkins',
+  }
+
+  file { "/var/lib/jenkins/jobs/${name}/builds":
+    ensure => directory,
+    owner => 'jenkins',
+    require => File["/var/lib/jenkins/jobs/${name}"]
+  }
+
+  file { "/var/lib/jenkins/jobs/${name}/config-history":
+    ensure => directory,
+    owner => 'jenkins',
+    require => File["/var/lib/jenkins/jobs/${name}"]
+  }
+
+  file { "/var/lib/jenkins/jobs/${name}/config.xml":
+    ensure => present,
+    content => template("jenkins_jobs/body.xml.erb"),
+    owner => 'jenkins',
+    require => File["/var/lib/jenkins/jobs/${name}"],
+    notify => Exec["jenkins"]
+  }
+
+  file { "/var/lib/jenkins/jobs/${name}/nextBuildNumber":
+    ensure => present,
+    content => "1",
+    owner => 'jenkins',
+    replace => false,
+    require => File["/var/lib/jenkins/jobs/${name}"]
+  }
+
+}

modules/jenkins_jobs/manifests/generic_jobs.pp

@@ -0,0 +1,28 @@
+define jenkins_jobs::generic_jobs($site, $project, $node_group) {
+  jenkins_jobs::jobs::docs { $name:
+    site => $site,
+    project => $project,
+    node_group => $node_group
+  }
+
+  jenkins_jobs::jobs::merge_gate { $name:
+    site => $site,
+    project => $project,
+    node_group => $node_group,
+    trigger_branches => [[$project, '**']]
+  }
+
+  jenkins_jobs::jobs::ppa { $name:
+    site => $site,
+    project => $project,
+    node_group => $node_group
+  }
+
+  jenkins_jobs::jobs::tarball { $name:
+    site => $site,
+    project => $project,
+    node_group => $node_group,
+    trigger_branches => [[$project, '^(?!refs/).*$']]
+  }
+
+}

modules/jenkins_jobs/manifests/init.pp

@@ -1,9 +1,4 @@
-class jenkins_jobs($site, $projects) {
-
-  jenkins_jobs::add_jobs { $projects:
-    site => "${site}"
-  }
-
+class jenkins_jobs($site) {
   exec { "jenkins":
     command => "/usr/bin/curl https://jenkins.${site}.org/reload",
     refreshonly => true

modules/jenkins_jobs/manifests/job.pp

@@ -1,36 +0,0 @@
-define jenkins_jobs::job($site, $project, $job, $triggers="", $builders, $publishers="", $logrotate="", $scm="") {
-
-  file { "/var/lib/jenkins/jobs/${project}-${job}":
-    ensure => directory,
-    owner => 'jenkins',
-  }
-
-  file { "/var/lib/jenkins/jobs/${project}-${job}/builds":
-    ensure => directory,
-    owner => 'jenkins',
-    require => File["/var/lib/jenkins/jobs/${project}-${job}"]
-  }
-
-  file { "/var/lib/jenkins/jobs/${project}-${job}/config-history":
-    ensure => directory,
-    owner => 'jenkins',
-    require => File["/var/lib/jenkins/jobs/${project}-${job}"]
-  }
-
-  file { "/var/lib/jenkins/jobs/${project}-${job}/config.xml":
-    ensure => present,
-    content => template("jenkins_jobs/body.xml.erb"),
-    owner => 'jenkins',
-    require => File["/var/lib/jenkins/jobs/${project}-${job}"],
-    notify => Exec["jenkins"]
-  }
-
-  file { "/var/lib/jenkins/jobs/${project}-${job}/nextBuildNumber":
-    ensure => present,
-    content => "1",
-    owner => 'jenkins',
-    replace => false,
-    require => File["/var/lib/jenkins/jobs/${project}-${job}"]
-  }
-
-}

modules/jenkins_jobs/manifests/jobs/coverage.pp

@@ -0,0 +1,13 @@
+define jenkins_jobs::jobs::coverage($site, $project, $node_group) {
+  jenkins_jobs::build_job { "${name}-coverage":
+    site => $site,
+    project => $project,
+    job => "coverage",
+    node_group => $node_group,
+    logrotate => misc("logrotate"),
+    builders => [builder("copy_bundle"), builder("coverage")],
+    publishers => publisher("coverage"),
+    triggers => trigger("timed_15mins"),
+    scm => scm("git")
+  }
+}

modules/jenkins_jobs/manifests/jobs/docs.pp

@@ -0,0 +1,12 @@
+define jenkins_jobs::jobs::docs($site, $project, $node_group) {
+  jenkins_jobs::build_job { "${name}-docs":
+    site => $site,
+    project => $project,
+    job => "docs",
+    node_group => $node_group,
+    triggers => trigger("timed_15mins"),
+    builders => [builder("copy_bundle"), builder("docs")],
+    publishers => publisher("docs"),
+    scm => scm("git")
+  }
+}

modules/jenkins_jobs/manifests/jobs/merge_gate.pp

@@ -0,0 +1,11 @@
+define jenkins_jobs::jobs::merge_gate($site, $project, $node_group, $trigger_branches) {
+  jenkins_jobs::build_job { "gate-${name}-merge":
+    site => $site,
+    project => $project,
+    job => "merge",
+    node_group => $node_group,
+    triggers => trigger("gerrit_comment"),
+    builders => builder("gerrit_git_prep"),
+    trigger_branches => $trigger_branches
+  }
+}

modules/jenkins_jobs/manifests/jobs/pep8_gate.pp

@@ -0,0 +1,12 @@
+define jenkins_jobs::jobs::pep8_gate($site, $project, $node_group, $trigger_branches) {
+  jenkins_jobs::build_job { "gate-${name}-pep8":
+    site => $site,
+    project => $project,
+    job => "pep8",
+    node_group => $node_group,
+    triggers => trigger("gerrit_comment"),
+    builders => [builder("gerrit_git_prep"), builder("copy_bundle"), builder("pep8")],
+    publishers => publisher("pep8"),
+    trigger_branches => $trigger_branches
+  }
+}

modules/jenkins_jobs/manifests/jobs/ppa.pp

@@ -0,0 +1,10 @@
+define jenkins_jobs::jobs::ppa($site, $project, $node_group) {
+  jenkins_jobs::build_job { "${name}-ppa":
+    site => $site,
+    project => $project,
+    job => "ppa",
+    node_group => $node_group,
+    builders => builder("ppa"),
+    publishers => publisher("ppa")
+  }
+}

modules/jenkins_jobs/manifests/jobs/python26_gate.pp

@@ -0,0 +1,11 @@
+define jenkins_jobs::jobs::python26_gate($site, $project, $node_group, $trigger_branches) {
+  jenkins_jobs::build_job { "gate-${name}-python26":
+    site => $site,
+    project => $project,
+    job => "python26",
+    node_group => $node_group,
+    triggers => trigger("gerrit_comment"),
+    builders => [builder("gerrit_git_prep"), builder("copy_bundle"), builder("python26")],
+    trigger_branches => $trigger_branches
+  }
+}

modules/jenkins_jobs/manifests/jobs/python27_gate.pp

@@ -0,0 +1,11 @@
+define jenkins_jobs::jobs::python27_gate($site, $project, $node_group, $trigger_branches) {
+  jenkins_jobs::build_job { "gate-${name}-python27":
+    site => $site,
+    project => $project,
+    job => "python27",
+    node_group => $node_group,
+    triggers => trigger("gerrit_comment"),
+    builders => [builder("gerrit_git_prep"), builder("copy_bundle"), builder("python27")],
+    trigger_branches => $trigger_branches
+  }
+}

modules/jenkins_jobs/manifests/jobs/tarball.pp

@@ -0,0 +1,12 @@
+define jenkins_jobs::jobs::tarball($site, $project, $node_group, $trigger_branches) {
+  jenkins_jobs::build_job { "${name}-tarball":
+    site => $site,
+    project => $project,
+    job => "tarball",
+    node_group => $node_group,
+    triggers => trigger("gerrit_ref_updated"),
+    builders => [builder("gerrit_git_prep"), builder("copy_bundle"), builder("tarball")],
+    publishers => publisher("tarball"),
+    trigger_branches => $trigger_branches
+  }
+}

modules/jenkins_jobs/manifests/jobs/venv.pp

@@ -0,0 +1,12 @@
+define jenkins_jobs::jobs::venv($site, $project, $node_group) {
+  jenkins_jobs::build_job { "${name}-venv":
+    site => $site,
+    project => $project,
+    job => "venv",
+    node_group => $node_group,
+    triggers => trigger("timed_midnight"),
+    builders => builder("venv"),
+    publishers => publisher("venv"),
+    scm => scm("git")
+  }
+}

modules/jenkins_jobs/manifests/python_jobs.pp

@@ -0,0 +1,30 @@
+define jenkins_jobs::python_jobs($site, $project, $node_group) {
+  jenkins_jobs::jobs::coverage { $name:
+    site => $site,
+    project => $project,
+    node_group => $node_group
+  }
+  jenkins_jobs::jobs::pep8_gate { $name:
+    site => $site,
+    project => $project,
+    node_group => $node_group,
+    trigger_branches => [[$project, '**']]
+  }
+  jenkins_jobs::jobs::python26_gate { $name:
+    site => $site,
+    project => $project,
+    node_group => $node_group,
+    trigger_branches => [[$project, '**']]
+  }
+  jenkins_jobs::jobs::python27_gate { $name:
+    site => $site,
+    project => $project,
+    node_group => $node_group,
+    trigger_branches => [[$project, '**']]
+  }
+  jenkins_jobs::jobs::venv { $name:
+    site => $site,
+    project => $project,
+    node_group => $node_group
+  }
+}

modules/jenkins_jobs/templates/body.xml.erb

@@ -45,26 +45,20 @@ In modules/jenkins_jobs
   <% else %>
   <%= scm %>
   <% end %>
-  <assignedNode>oneiric</assignedNode>
+  <assignedNode><%= node_group %></assignedNode>
   <canRoam>false</canRoam>
   <disabled>false</disabled>
   <blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
   <blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
-  <% if triggers != "" %>
   <triggers class="vector">
   <%= triggers %>
   </triggers>
-  <% end %>
   <concurrentBuild>false</concurrentBuild>
-  <% if builders != "" %>
   <builders>
   <%= builders %>
   </builders>
-  <% end %>
   <publishers>
-  <% if publishers != "" %>
   <%= publishers %>
-  <% end %>
   </publishers>
   <buildWrappers/>
 </project>

modules/jenkins_jobs/templates/builder_copy_bundle.xml.erb

@@ -1,4 +1,3 @@
-  <builders>
     <hudson.plugins.copyartifact.CopyArtifact>
       <projectName>$PROJECT-venv</projectName>
       <filter></filter>
@@ -8,4 +7,3 @@
     <hudson.tasks.Shell>
       <command>/usr/local/jenkins/slave_scripts/copy-bundle.sh</command>
     </hudson.tasks.Shell>
-  </builders>

modules/jenkins_jobs/templates/builder_python27.xml.erb

@@ -1,3 +1,3 @@
     <hudson.tasks.Shell>
-      <command>/usr/local/jenkins/slave_scripts/run-tox.sh 26</command>
+      <command>/usr/local/jenkins/slave_scripts/run-tox.sh 27</command>
     </hudson.tasks.Shell>

modules/jenkins_jobs/templates/trigger_gerrit_comment.xml.erb

@@ -1,16 +1,18 @@
     <com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.GerritTrigger>
       <spec></spec>
       <gerritProjects>
+        <% trigger_branches.each do |trigger| %>
         <com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.data.GerritProject>
           <compareType>PLAIN</compareType>
-          <pattern><%= site %>/<%= project %></pattern>
+          <pattern><%= site %>/<%= trigger[0] %></pattern>
           <branches>
             <com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.data.Branch>
               <compareType>ANT</compareType>
-              <pattern>**</pattern>
+              <pattern><%= trigger[1] %></pattern>
             </com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.data.Branch>
           </branches>
         </com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.data.GerritProject>
+        <% end %>
       </gerritProjects>
       <silentMode>false</silentMode>
       <escapeQuotes>true</escapeQuotes>

modules/jenkins_jobs/templates/trigger_gerrit_ref_updated.xml.erb

@@ -1,13 +1,18 @@
+    <com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.GerritTrigger>
+      <spec></spec>
+      <gerritProjects>
+        <% trigger_branches.each do |trigger| %>
         <com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.data.GerritProject>
           <compareType>PLAIN</compareType>
-          <pattern><%= site %>/<%= project %></pattern>
+          <pattern><%= site %>/<%= trigger[0] %></pattern>
           <branches>
             <com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.data.Branch>
               <compareType>REG_EXP</compareType>
-              <pattern>^(?!refs/).*$</pattern>
+              <pattern><%= trigger[1] %></pattern>
             </com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.data.Branch>
           </branches>
         </com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.data.GerritProject>
+        <% end %>
       </gerritProjects>
       <silentMode>false</silentMode>
       <escapeQuotes>true</escapeQuotes>