diff --git a/modules/pypimirror/files/run_mirror.py b/modules/pypimirror/files/run_mirror.py new file mode 100755 index 0000000000..ab21588873 --- /dev/null +++ b/modules/pypimirror/files/run_mirror.py @@ -0,0 +1,81 @@ +#! /usr/bin/env python +# Copyright (C) 2011 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# run_mirrors reads a project config file called projects.yaml +# It should look like: + +# - project: PROJECT_NAME + +import logging +import os +import subprocess +import shlex +import yaml + +def run_command(cmd, status=False, env={}): + cmd_list = shlex.split(str(cmd)) + newenv = os.environ + newenv.update(env) + p = subprocess.Popen(cmd_list, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, env=newenv) + (out, nothing) = p.communicate() + if status: + return (p.returncode, out.strip()) + return out.strip() + + +def run_command_status(cmd, env={}): + return run_command(cmd, True, env) + + +logging.basicConfig(level=logging.ERROR) + +PROJECTS_YAML = os.environ.get('PROJECTS_YAML', + '/etc/openstackci/projects.yaml') +PIP_TEMP_DOWNLOAD = os.environ.get('PIP_TEMP_DOWNLOAD', + '/var/lib/pip-download') +GIT_SOURCE = os.environ.get('GIT_SOURCE', 'https://github.com') +pip_command = '/usr/local/bin/pip install -M -U -I --exists-action=w ' \ + '--no-install %s' + +run_command(pip_command % "pip") + +config = yaml.load(open(PROJECTS_YAML)) + +for section in config: + project = section['project'] + + os.chdir(PIP_TEMP_DOWNLOAD) + short_project = project.split('/')[1] + if not os.path.isdir(short_project): + run_command("git clone %s/%s.git %s" % (GIT_SOURCE, project, + short_project)) + os.chdir(short_project) + run_command("git fetch origin") + + for branch in run_command("git branch -a").split("\n"): + branch = branch.strip() + if (not branch.startswith("remotes/origin") + or "origin/HEAD" in branch): + continue + run_command("git reset --hard %s" % branch) + run_command("git clean -x -f -d -q") + print("*********************") + print("Fetching pip requires for %s:%s" % (project, branch)) + for requires_file in ("tools/pip-requires", "tools/test-requires"): + if os.path.exists(requires_file): + stanza = "-r %s" % requires_file + run_command(pip_command % stanza) + diff --git a/modules/pypimirror/manifests/init.pp b/modules/pypimirror/manifests/init.pp index da51394a3d..783c447712 100644 --- a/modules/pypimirror/manifests/init.pp +++ b/modules/pypimirror/manifests/init.pp @@ -3,6 +3,7 @@ class pypimirror ( $base_url, $mirror_file_path = "/var/lib/pypimirror", $pip_download = "/var/lib/pip-download", $pip_cache = "/var/cache/pip", + $git_source = "https://github.com", $projects = [] ) { @@ -39,6 +40,20 @@ class pypimirror ( $base_url, group => 'root', } + file { '/etc/openstackci': + ensure => "directory", + owner => "root", + } + + file { '/home/gerrit2/projects.yaml': + owner => 'root', + group => 'root', + mode => 444, + ensure => 'present', + source => 'puppet:///openstack_project/review.projects.yaml', + replace => true, + } + file { '/usr/local/mirror_scripts/run-mirror.sh': ensure => present, mode => 755, diff --git a/modules/pypimirror/templates/run-mirror.sh.erb b/modules/pypimirror/templates/run-mirror.sh.erb index 62cc2e9ae5..2fdce0e180 100644 --- a/modules/pypimirror/templates/run-mirror.sh.erb +++ b/modules/pypimirror/templates/run-mirror.sh.erb @@ -6,7 +6,5 @@ export PIP_DOWNLOAD_CACHE=<%= pip_cache %> export PIP_TEMP_DOWNLOAD=<%= pip_download %> export MIRROR_FILE_PATH=<%= mirror_file_path %> export LOG_FILENAME=<%= log_filename %> -<% projects.each do |project| -%> -/usr/local/mirror_scripts/pull-repo.sh <%= project['name'] %> >>$LOG_FILENAME -<% end -%> +python /usr/local/mirror_scripts/run-mirror.py <%= git_source %> >>$LOG_FILENAME python /usr/local/mirror_scripts/process_cache.py ${PIP_DOWNLOAD_CACHE} ${MIRROR_FILE_PATH}