Trim down some un-needed repetition.

Adding a yml file to the project but not listing it in the .pp file
only really ever caused bugs.
Moved the full openstack project list to a variable for re-use.

Change-Id: I211b5cf54d5a84c6136a4b04ea819c688cc1e2b6

manifests/openstack.pp

@@ -1,4 +1,122 @@
 import "users"
+
+$openstack_project_list = [ {
+     name => 'openstack/keystone',
+     close_pull => 'true'
+     }, {
+     name => 'openstack/glance',
+     close_pull => 'true'
+     }, {
+     name => 'openstack/swift',
+     close_pull => 'true'
+     }, {
+     name => 'openstack/nova',
+     close_pull => 'true'
+     }, {
+     name => 'openstack/horizon',
+     close_pull => 'true'
+     }, {
+     name => 'openstack/quantum',
+     close_pull => 'true'
+     }, {
+     name => 'openstack/melange',
+     close_pull => 'true'
+     }, {
+     name => 'openstack/tempest',
+     close_pull => 'true'
+     }, {
+     name => 'openstack/openstack-ci',
+     close_pull => 'true'
+     }, {
+     name => 'openstack/openstack-ci-puppet',
+     close_pull => 'true'
+     }, {
+     name => 'openstack/openstack-puppet',
+     close_pull => 'true'
+     }, {
+     name => 'openstack/openstack-chef',
+     close_pull => 'true'
+     }, {
+     name => 'openstack/openstack-manuals',
+     close_pull => 'true'
+     }, {
+     name => 'openstack/compute-api',
+     close_pull => 'true'
+     }, {
+     name => 'openstack/image-api',
+     close_pull => 'true'
+     }, {
+     name => 'openstack/identity-api',
+     close_pull => 'true'
+     }, {
+     name => 'openstack/object-api',
+     close_pull => 'true'
+     }, {
+     name => 'openstack/netconn-api',
+     close_pull => 'true'
+     }, {
+     name => 'openstack-dev/devstack',
+     close_pull => 'true'
+     }, {
+     name => 'openstack-dev/openstack-qa',
+     close_pull => 'true'
+     }, {
+     name => 'openstack-dev/pbr',
+     close_pull => 'true'
+     }, {
+     name => 'openstack/python-novaclient',
+     close_pull => 'true'
+     }, {
+     name => 'openstack/python-glanceclient',
+     close_pull => 'true'
+     }, {
+     name => 'openstack-ci/git-review',
+     close_pull => 'true'
+     }, {
+     name => 'openstack-ci/lodgeit',
+     close_pull => 'true'
+     }, {
+     name => 'openstack-ci/meetbot',
+     close_pull => 'true'
+     }, {
+     name => 'openstack-ci/zuul',
+     close_pull => 'true'
+     }, {
+     name => 'openstack-ci/pypi-mirror',
+     close_pull => 'true'
+     }, {
+     name => 'openstack/openstack-common',
+     close_pull => 'true'
+     }, {
+     name => 'openstack/cinder',
+     close_pull => 'true'
+     }, {
+     name => 'openstack/python-openstackclient',
+     close_pull => 'true'
+     }, {
+     name => 'openstack-dev/openstack-nose',
+     close_pull => 'true'
+     }, {
+     name => 'openstack/python-cinderclient',
+     close_pull => 'true'
+     }, {
+     name => 'openstack/python-swiftclient',
+     close_pull => 'true'
+     }, {
+     name => 'stackforge/MRaaS',
+     close_pull => 'true'
+     }, {
+     name => 'stackforge/reddwarf',
+     close_pull => 'true'
+     }, {
+     name => 'stackforge/ceilometer',
+     close_pull => 'true'
+     }, {
+     name => 'heat-api/heat',
+     close_pull => 'true'
+     }
+  ]
+
 #
 # Abstract classes:
 #

manifests/site.pp

@@ -135,121 +135,7 @@ node "review.openstack.org" {
     core_packedgitwindowsize => '16k',
     sshd_threads => '100',
     httpd_maxwait => '5000min',
-    github_projects => [ {
+    github_projects => $openstack_project_list,
-                         name => 'openstack/keystone',
-                         close_pull => 'true'
-                         }, {
-                         name => 'openstack/glance',
-                         close_pull => 'true'
-                         }, {
-                         name => 'openstack/swift',
-                         close_pull => 'true'
-                         }, {
-                         name => 'openstack/nova',
-                         close_pull => 'true'
-                         }, {
-                         name => 'openstack/horizon',
-                         close_pull => 'true'
-                         }, {
-                         name => 'openstack/quantum',
-                         close_pull => 'true'
-                         }, {
-                         name => 'openstack/melange',
-                         close_pull => 'true'
-                         }, {
-                         name => 'openstack/tempest',
-                         close_pull => 'true'
-                         }, {
-                         name => 'openstack/openstack-ci',
-                         close_pull => 'true'
-                         }, {
-                         name => 'openstack/openstack-ci-puppet',
-                         close_pull => 'true'
-                         }, {
-                         name => 'openstack/openstack-puppet',
-                         close_pull => 'true'
-                         }, {
-                         name => 'openstack/openstack-chef',
-                         close_pull => 'true'
-                         }, {
-                         name => 'openstack/openstack-manuals',
-                         close_pull => 'true'
-                         }, {
-                         name => 'openstack/compute-api',
-                         close_pull => 'true'
-                         }, {
-                         name => 'openstack/image-api',
-                         close_pull => 'true'
-                         }, {
-                         name => 'openstack/identity-api',
-                         close_pull => 'true'
-                         }, {
-                         name => 'openstack/object-api',
-                         close_pull => 'true'
-                         }, {
-                         name => 'openstack/netconn-api',
-                         close_pull => 'true'
-                         }, {
-                         name => 'openstack-dev/devstack',
-                         close_pull => 'true'
-                         }, {
-                         name => 'openstack-dev/openstack-qa',
-                         close_pull => 'true'
-                         }, {
-                         name => 'openstack-dev/pbr',
-                         close_pull => 'true'
-                         }, {
-                         name => 'openstack/python-novaclient',
-                         close_pull => 'true'
-                         }, {
-                         name => 'openstack/python-glanceclient',
-                         close_pull => 'true'
-                         }, {
-                         name => 'openstack-ci/git-review',
-                         close_pull => 'true'
-                         }, {
-                         name => 'openstack-ci/lodgeit',
-                         close_pull => 'true'
-                         }, {
-                         name => 'openstack-ci/meetbot',
-                         close_pull => 'true'
-                         }, {
-                         name => 'openstack-ci/zuul',
-                         close_pull => 'true'
-                         }, {
-                         name => 'openstack-ci/pypi-mirror',
-                         close_pull => 'true'
-                         }, {
-                         name => 'openstack/openstack-common',
-                         close_pull => 'true'
-                         }, {
-                         name => 'openstack/cinder',
-                         close_pull => 'true'
-                         }, {
-                         name => 'openstack/python-openstackclient',
-                         close_pull => 'true'
-                         }, {
-                         name => 'openstack-dev/openstack-nose',
-                         close_pull => 'true'
-                         }, {
-                         name => 'openstack/python-cinderclient',
-                         close_pull => 'true'
-                         }, {
-                         name => 'openstack/python-swiftclient',
-                         close_pull => 'true'
-                         }, {
-                         name => 'stackforge/MRaaS',
-                         close_pull => 'true'
-                         }, {
-                         name => 'stackforge/reddwarf',
-                         close_pull => 'true'
-                         }, {
-                         name => 'stackforge/ceilometer',
-                         close_pull => 'true'
-                         }, {
-                         name => 'heat-api/heat',
-                         close_pull => 'true'
-                         } ],
     upstream_projects => [ {
                          name => 'openstack-ci/gerrit',
                          remote => 'https://gerrit.googlesource.com/gerrit'
@@ -325,37 +211,6 @@ node "jenkins.openstack.org" {
     username => "gerrig",
     password => hiera('jenkins_jobs_password'),
     site => "openstack",
-    projects => [
-      'cinder',
-      'devstack',
-      'devstack-gate',
-      'gerrit',
-      'gerritbot',
-      'gerritlib',
-      'gerrit-verification-status-plugin',
-      'glance',
-      'horizon',
-      'keystone',
-      'nova',
-      'openstack-ci-puppet',
-      'openstack-common',
-      'pbr',
-      'pypi-mirror',
-      'python-cinderclient',
-      'python-glanceclient',
-      'python-keystoneclient',
-      'python-novaclient',
-      'python-openstackclient',
-      'python-quantumclient',
-      'python-swiftclient',
-      'quantum',
-      'swift',
-      'tempest',
-      'zuul',
-      'reddwarf',
-      'ceilometer',
-      'heat'
-    ]
   }
   class { 'zuul': }
   file { "/etc/zuul/layout.yaml":
@@ -516,23 +371,7 @@ node "pypi.openstack.org" {
 
   class { "pypimirror":
     base_url => "http://pypi.openstack.org",
-    projects => [
+    projects => $openstack_project_list,
-      'cinder',
-      'glance',
-      'horizon',
-      'keystone',
-      'nova',
-      'openstack-common',
-      'python-cinderclient',
-      'python-glanceclient',
-      'python-keystoneclient',
-      'python-novaclient',
-      'python-openstackclient',
-      'python-quantumclient',
-      'python-swiftclient',
-      'quantum',
-      'swift'
-      ]
   }
 }
 

modules/jenkins_jobs/files/jenkins_jobs.py

@@ -34,9 +34,9 @@ class JenkinsJobsException(Exception): pass
 parser = argparse.ArgumentParser()
 subparser = parser.add_subparsers(help='update, test or delete job', dest='command')
 parser_update = subparser.add_parser('update')
-parser_update.add_argument('file', help='YAML file for update', type=file)
+parser_update.add_argument('file', help='YAML file for update')
 parser_update = subparser.add_parser('test')
-parser_update.add_argument('file', help='YAML file for test', type=file)
+parser_update.add_argument('file', help='YAML file for test')
 parser_delete = subparser.add_parser('delete')
 parser_delete.add_argument('name', help='name of job')
 parser.add_argument('--conf', dest='conf', help='Configuration file')
@@ -242,10 +242,16 @@ def delete_job():
     remote_jenkins.delete_job(options.name)
 
 def update_job(test = False):
-    yparse = YamlParser(options.file)
+    if os.path.isdir(options.file):
+        files_to_process = [os.path.join(options.file, f)
+                            for f in os.listdir(options.file)]
+    else:
+        files_to_process = [options.file]
     cache = CacheStorage()
     if not test:
         remote_jenkins = Jenkins(config.get('jenkins','url'), config.get('jenkins','user'), config.get('jenkins','password'))
+    for in_file in files_to_process:
+        yparse = YamlParser(open(in_file, 'r'))
         while True:
             try:
                 xml = yparse.get_next_xml()

modules/jenkins_jobs/manifests/init.pp

@@ -1,4 +1,4 @@
-class jenkins_jobs($url, $username, $password, $site, $projects) {
+class jenkins_jobs($url, $username, $password, $site) {
 
   package { 'python-yaml':
     ensure => 'present'
@@ -24,8 +24,10 @@ class jenkins_jobs($url, $username, $password, $site, $projects) {
     require => File['/usr/local/jenkins_jobs']
   }
 
-  process_projects { $projects:
+  exec { "jenkins_job_${site}":
-    site => $site,
+    command => "python /usr/local/jenkins_jobs/jenkins_jobs.py update /usr/local/jenkins_jobs/projects/${site}",
+    cwd => '/usr/local/jenkins_jobs/',
+    path => '/bin:/usr/bin',
     require => [
       File['/usr/local/jenkins_jobs/jenkins_jobs.ini'],
       Package['python-jenkins']

modules/jenkins_jobs/manifests/process_projects.pp

@@ -1,8 +0,0 @@
-define process_projects($site) {
-  exec { "jenkins_job_${name}":
-    command => "python /usr/local/jenkins_jobs/jenkins_jobs.py update /usr/local/jenkins_jobs/projects/${site}/${name}.yml",
-    cwd => '/usr/local/jenkins_jobs/',
-    path => '/bin:/usr/bin'
-  }
-
-}

modules/pypimirror/files/pull-repo.sh

@@ -9,10 +9,11 @@ project=$1
 pip_command='/usr/local/bin/pip install -M -U -I --exists-action=w --no-install'
 
 cd ${PIP_TEMP_DOWNLOAD}
-if [ ! -d ${project} ] ; then
+short_project=`echo ${project} | cut -f2 -d/`
-  git clone git://github.com/openstack/${project}.git ${project} >/dev/null 2>&1
+if [ ! -d ${short_project} ] ; then
+  git clone git://github.com/${project}.git ${short_project} >/dev/null 2>&1
 fi
-cd ${project}
+cd ${short_project}
 $pip_command pip
 git fetch origin
 for branch in `git branch -a | grep remotes.origin | grep -v origin.HEAD | awk '{print $1}' ` ; do

modules/pypimirror/templates/run-mirror.sh.erb

@@ -7,6 +7,6 @@ export PIP_TEMP_DOWNLOAD=<%= pip_download %>
 export MIRROR_FILE_PATH=<%= mirror_file_path %>
 export LOG_FILENAME=<%= log_filename %>
 <% projects.each do |project| -%>
-/usr/local/mirror_scripts/pull-repo.sh <%= project %> >>$LOG_FILENAME
+/usr/local/mirror_scripts/pull-repo.sh <%= project['name'] %> >>$LOG_FILENAME
 <% end -%>
 python /usr/local/mirror_scripts/process_cache.py ${PIP_DOWNLOAD_CACHE} ${MIRROR_FILE_PATH}