diff --git a/manifests/openstack.pp b/manifests/openstack.pp index 9360a175fb..d69cfaca0f 100644 --- a/manifests/openstack.pp +++ b/manifests/openstack.pp @@ -1,4 +1,122 @@ import "users" + +$openstack_project_list = [ { + name => 'openstack/keystone', + close_pull => 'true' + }, { + name => 'openstack/glance', + close_pull => 'true' + }, { + name => 'openstack/swift', + close_pull => 'true' + }, { + name => 'openstack/nova', + close_pull => 'true' + }, { + name => 'openstack/horizon', + close_pull => 'true' + }, { + name => 'openstack/quantum', + close_pull => 'true' + }, { + name => 'openstack/melange', + close_pull => 'true' + }, { + name => 'openstack/tempest', + close_pull => 'true' + }, { + name => 'openstack/openstack-ci', + close_pull => 'true' + }, { + name => 'openstack/openstack-ci-puppet', + close_pull => 'true' + }, { + name => 'openstack/openstack-puppet', + close_pull => 'true' + }, { + name => 'openstack/openstack-chef', + close_pull => 'true' + }, { + name => 'openstack/openstack-manuals', + close_pull => 'true' + }, { + name => 'openstack/compute-api', + close_pull => 'true' + }, { + name => 'openstack/image-api', + close_pull => 'true' + }, { + name => 'openstack/identity-api', + close_pull => 'true' + }, { + name => 'openstack/object-api', + close_pull => 'true' + }, { + name => 'openstack/netconn-api', + close_pull => 'true' + }, { + name => 'openstack-dev/devstack', + close_pull => 'true' + }, { + name => 'openstack-dev/openstack-qa', + close_pull => 'true' + }, { + name => 'openstack-dev/pbr', + close_pull => 'true' + }, { + name => 'openstack/python-novaclient', + close_pull => 'true' + }, { + name => 'openstack/python-glanceclient', + close_pull => 'true' + }, { + name => 'openstack-ci/git-review', + close_pull => 'true' + }, { + name => 'openstack-ci/lodgeit', + close_pull => 'true' + }, { + name => 'openstack-ci/meetbot', + close_pull => 'true' + }, { + name => 'openstack-ci/zuul', + close_pull => 'true' + }, { + name => 'openstack-ci/pypi-mirror', + close_pull => 'true' + }, { + name => 'openstack/openstack-common', + close_pull => 'true' + }, { + name => 'openstack/cinder', + close_pull => 'true' + }, { + name => 'openstack/python-openstackclient', + close_pull => 'true' + }, { + name => 'openstack-dev/openstack-nose', + close_pull => 'true' + }, { + name => 'openstack/python-cinderclient', + close_pull => 'true' + }, { + name => 'openstack/python-swiftclient', + close_pull => 'true' + }, { + name => 'stackforge/MRaaS', + close_pull => 'true' + }, { + name => 'stackforge/reddwarf', + close_pull => 'true' + }, { + name => 'stackforge/ceilometer', + close_pull => 'true' + }, { + name => 'heat-api/heat', + close_pull => 'true' + } + ] + # # Abstract classes: # diff --git a/manifests/site.pp b/manifests/site.pp index abe116d43b..4e5258e4d3 100644 --- a/manifests/site.pp +++ b/manifests/site.pp @@ -135,121 +135,7 @@ node "review.openstack.org" { core_packedgitwindowsize => '16k', sshd_threads => '100', httpd_maxwait => '5000min', - github_projects => [ { - name => 'openstack/keystone', - close_pull => 'true' - }, { - name => 'openstack/glance', - close_pull => 'true' - }, { - name => 'openstack/swift', - close_pull => 'true' - }, { - name => 'openstack/nova', - close_pull => 'true' - }, { - name => 'openstack/horizon', - close_pull => 'true' - }, { - name => 'openstack/quantum', - close_pull => 'true' - }, { - name => 'openstack/melange', - close_pull => 'true' - }, { - name => 'openstack/tempest', - close_pull => 'true' - }, { - name => 'openstack/openstack-ci', - close_pull => 'true' - }, { - name => 'openstack/openstack-ci-puppet', - close_pull => 'true' - }, { - name => 'openstack/openstack-puppet', - close_pull => 'true' - }, { - name => 'openstack/openstack-chef', - close_pull => 'true' - }, { - name => 'openstack/openstack-manuals', - close_pull => 'true' - }, { - name => 'openstack/compute-api', - close_pull => 'true' - }, { - name => 'openstack/image-api', - close_pull => 'true' - }, { - name => 'openstack/identity-api', - close_pull => 'true' - }, { - name => 'openstack/object-api', - close_pull => 'true' - }, { - name => 'openstack/netconn-api', - close_pull => 'true' - }, { - name => 'openstack-dev/devstack', - close_pull => 'true' - }, { - name => 'openstack-dev/openstack-qa', - close_pull => 'true' - }, { - name => 'openstack-dev/pbr', - close_pull => 'true' - }, { - name => 'openstack/python-novaclient', - close_pull => 'true' - }, { - name => 'openstack/python-glanceclient', - close_pull => 'true' - }, { - name => 'openstack-ci/git-review', - close_pull => 'true' - }, { - name => 'openstack-ci/lodgeit', - close_pull => 'true' - }, { - name => 'openstack-ci/meetbot', - close_pull => 'true' - }, { - name => 'openstack-ci/zuul', - close_pull => 'true' - }, { - name => 'openstack-ci/pypi-mirror', - close_pull => 'true' - }, { - name => 'openstack/openstack-common', - close_pull => 'true' - }, { - name => 'openstack/cinder', - close_pull => 'true' - }, { - name => 'openstack/python-openstackclient', - close_pull => 'true' - }, { - name => 'openstack-dev/openstack-nose', - close_pull => 'true' - }, { - name => 'openstack/python-cinderclient', - close_pull => 'true' - }, { - name => 'openstack/python-swiftclient', - close_pull => 'true' - }, { - name => 'stackforge/MRaaS', - close_pull => 'true' - }, { - name => 'stackforge/reddwarf', - close_pull => 'true' - }, { - name => 'stackforge/ceilometer', - close_pull => 'true' - }, { - name => 'heat-api/heat', - close_pull => 'true' - } ], + github_projects => $openstack_project_list, upstream_projects => [ { name => 'openstack-ci/gerrit', remote => 'https://gerrit.googlesource.com/gerrit' @@ -325,37 +211,6 @@ node "jenkins.openstack.org" { username => "gerrig", password => hiera('jenkins_jobs_password'), site => "openstack", - projects => [ - 'cinder', - 'devstack', - 'devstack-gate', - 'gerrit', - 'gerritbot', - 'gerritlib', - 'gerrit-verification-status-plugin', - 'glance', - 'horizon', - 'keystone', - 'nova', - 'openstack-ci-puppet', - 'openstack-common', - 'pbr', - 'pypi-mirror', - 'python-cinderclient', - 'python-glanceclient', - 'python-keystoneclient', - 'python-novaclient', - 'python-openstackclient', - 'python-quantumclient', - 'python-swiftclient', - 'quantum', - 'swift', - 'tempest', - 'zuul', - 'reddwarf', - 'ceilometer', - 'heat' - ] } class { 'zuul': } file { "/etc/zuul/layout.yaml": @@ -516,23 +371,7 @@ node "pypi.openstack.org" { class { "pypimirror": base_url => "http://pypi.openstack.org", - projects => [ - 'cinder', - 'glance', - 'horizon', - 'keystone', - 'nova', - 'openstack-common', - 'python-cinderclient', - 'python-glanceclient', - 'python-keystoneclient', - 'python-novaclient', - 'python-openstackclient', - 'python-quantumclient', - 'python-swiftclient', - 'quantum', - 'swift' - ] + projects => $openstack_project_list, } } diff --git a/modules/jenkins_jobs/files/jenkins_jobs.py b/modules/jenkins_jobs/files/jenkins_jobs.py index 34f03fbcc0..92bce197af 100644 --- a/modules/jenkins_jobs/files/jenkins_jobs.py +++ b/modules/jenkins_jobs/files/jenkins_jobs.py @@ -34,9 +34,9 @@ class JenkinsJobsException(Exception): pass parser = argparse.ArgumentParser() subparser = parser.add_subparsers(help='update, test or delete job', dest='command') parser_update = subparser.add_parser('update') -parser_update.add_argument('file', help='YAML file for update', type=file) +parser_update.add_argument('file', help='YAML file for update') parser_update = subparser.add_parser('test') -parser_update.add_argument('file', help='YAML file for test', type=file) +parser_update.add_argument('file', help='YAML file for test') parser_delete = subparser.add_parser('delete') parser_delete.add_argument('name', help='name of job') parser.add_argument('--conf', dest='conf', help='Configuration file') @@ -242,27 +242,33 @@ def delete_job(): remote_jenkins.delete_job(options.name) def update_job(test = False): - yparse = YamlParser(options.file) + if os.path.isdir(options.file): + files_to_process = [os.path.join(options.file, f) + for f in os.listdir(options.file)] + else: + files_to_process = [options.file] cache = CacheStorage() if not test: remote_jenkins = Jenkins(config.get('jenkins','url'), config.get('jenkins','user'), config.get('jenkins','password')) - while True: - try: - xml = yparse.get_next_xml() - job = yparse.get_name() - if test: - print xml.output() - continue - md5 = xml.md5() - if remote_jenkins.is_job(job) and not cache.is_cached(job): - old_md5 = remote_jenkins.get_job_md5(job) - cache.set(job, old_md5) + for in_file in files_to_process: + yparse = YamlParser(open(in_file, 'r')) + while True: + try: + xml = yparse.get_next_xml() + job = yparse.get_name() + if test: + print xml.output() + continue + md5 = xml.md5() + if remote_jenkins.is_job(job) and not cache.is_cached(job): + old_md5 = remote_jenkins.get_job_md5(job) + cache.set(job, old_md5) - if cache.has_changed(job, md5): - remote_jenkins.update_job(job, xml.output()) - cache.set(job, md5) - except JenkinsJobsException: - break + if cache.has_changed(job, md5): + remote_jenkins.update_job(job, xml.output()) + cache.set(job, md5) + except JenkinsJobsException: + break if options.command == 'delete': delete_job() diff --git a/modules/jenkins_jobs/manifests/init.pp b/modules/jenkins_jobs/manifests/init.pp index 574c6c2df2..e79984ba17 100644 --- a/modules/jenkins_jobs/manifests/init.pp +++ b/modules/jenkins_jobs/manifests/init.pp @@ -1,4 +1,4 @@ -class jenkins_jobs($url, $username, $password, $site, $projects) { +class jenkins_jobs($url, $username, $password, $site) { package { 'python-yaml': ensure => 'present' @@ -24,8 +24,10 @@ class jenkins_jobs($url, $username, $password, $site, $projects) { require => File['/usr/local/jenkins_jobs'] } - process_projects { $projects: - site => $site, + exec { "jenkins_job_${site}": + command => "python /usr/local/jenkins_jobs/jenkins_jobs.py update /usr/local/jenkins_jobs/projects/${site}", + cwd => '/usr/local/jenkins_jobs/', + path => '/bin:/usr/bin', require => [ File['/usr/local/jenkins_jobs/jenkins_jobs.ini'], Package['python-jenkins'] diff --git a/modules/jenkins_jobs/manifests/process_projects.pp b/modules/jenkins_jobs/manifests/process_projects.pp deleted file mode 100644 index 2d4c72617f..0000000000 --- a/modules/jenkins_jobs/manifests/process_projects.pp +++ /dev/null @@ -1,8 +0,0 @@ -define process_projects($site) { - exec { "jenkins_job_${name}": - command => "python /usr/local/jenkins_jobs/jenkins_jobs.py update /usr/local/jenkins_jobs/projects/${site}/${name}.yml", - cwd => '/usr/local/jenkins_jobs/', - path => '/bin:/usr/bin' - } - -} diff --git a/modules/pypimirror/files/pull-repo.sh b/modules/pypimirror/files/pull-repo.sh index 8fd48c2f88..46873935b2 100644 --- a/modules/pypimirror/files/pull-repo.sh +++ b/modules/pypimirror/files/pull-repo.sh @@ -9,10 +9,11 @@ project=$1 pip_command='/usr/local/bin/pip install -M -U -I --exists-action=w --no-install' cd ${PIP_TEMP_DOWNLOAD} -if [ ! -d ${project} ] ; then - git clone git://github.com/openstack/${project}.git ${project} >/dev/null 2>&1 +short_project=`echo ${project} | cut -f2 -d/` +if [ ! -d ${short_project} ] ; then + git clone git://github.com/${project}.git ${short_project} >/dev/null 2>&1 fi -cd ${project} +cd ${short_project} $pip_command pip git fetch origin for branch in `git branch -a | grep remotes.origin | grep -v origin.HEAD | awk '{print $1}' ` ; do diff --git a/modules/pypimirror/templates/run-mirror.sh.erb b/modules/pypimirror/templates/run-mirror.sh.erb index aae46f2055..62cc2e9ae5 100644 --- a/modules/pypimirror/templates/run-mirror.sh.erb +++ b/modules/pypimirror/templates/run-mirror.sh.erb @@ -7,6 +7,6 @@ export PIP_TEMP_DOWNLOAD=<%= pip_download %> export MIRROR_FILE_PATH=<%= mirror_file_path %> export LOG_FILENAME=<%= log_filename %> <% projects.each do |project| -%> -/usr/local/mirror_scripts/pull-repo.sh <%= project %> >>$LOG_FILENAME +/usr/local/mirror_scripts/pull-repo.sh <%= project['name'] %> >>$LOG_FILENAME <% end -%> python /usr/local/mirror_scripts/process_cache.py ${PIP_DOWNLOAD_CACHE} ${MIRROR_FILE_PATH}