diff --git a/manifests/site.pp b/manifests/site.pp index 6d0657d367..e6f8b03a54 100644 --- a/manifests/site.pp +++ b/manifests/site.pp @@ -376,11 +376,38 @@ node "eavesdrop.openstack.org" { node "pypi.openstack.org" { include openstack_cron + + # include jenkins slave so that build deps are there for the pip download + class { 'jenkins_slave': + ssh_key => "", + user => false + } + class { 'openstack_server': iptables_public_tcp_ports => [80] } + class { "pypimirror": base_url => "http://pypi.openstack.org", + projects => [ + 'cinder', + 'glance', + 'horizon', + 'keystone', + 'melange', + 'nova', + 'openstack-common', + 'python-cinderclient', + 'python-glanceclient', + 'python-keystoneclient', + 'python-melangeclient', + 'python-novaclient', + 'python-openstackclient', + 'python-quantumclient', + 'python-swiftclient', + 'quantum', + 'swift' + ] } } diff --git a/modules/pypimirror/files/pull-repo.sh b/modules/pypimirror/files/pull-repo.sh new file mode 100644 index 0000000000..9730ff8ac5 --- /dev/null +++ b/modules/pypimirror/files/pull-repo.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# This file is managed by puppet. +# https://github.com/openstack/openstack-ci-puppet + +export PIP_DOWNLOAD_CACHE=${PIP_DOWNLOAD_CACHE:-/var/cache/pip} +export PIP_TEMP_DOWNLOAD=${PIP_TEMP_DOWNLOAD:-/var/lib/pip-download} + +project=$1 +pip_command='pip install -M -U -I --exists-action=w --no-install' + +cd ${PIP_TEMP_DOWNLOAD} +if [ ! -d ${project} ] ; then + git clone git://github.com/openstack/${project}.git ${project} >/dev/null 2>&1 +fi +cd ${project} +git fetch origin +for branch in `git branch -a | grep remotes.origin | grep -v origin.HEAD | awk '{print $1}' ` ; do + git reset --hard $branch + git clean -x -f -d -q + echo "*********************" + echo "Fetching pip requires for $project:$branch" + for requires_file in tools/pip-requires tools/test-requires ; do + if [ -f ${requires_file} ] ; then + $pip_command -r $requires_file + fi + done +done diff --git a/modules/pypimirror/manifests/init.pp b/modules/pypimirror/manifests/init.pp index 5c41172cd7..f0f26da038 100644 --- a/modules/pypimirror/manifests/init.pp +++ b/modules/pypimirror/manifests/init.pp @@ -1,87 +1,76 @@ class pypimirror ( $base_url, $log_filename = "/var/log/pypimirror.log", $mirror_file_path = "/var/lib/pypimirror", - $fetch_since_days = 1, - $package_matches = ["*"], - $external_links = true, - $follow_external_index_pages = true ) + $pip_download = "/var/lib/pip-download", + $pip_cache = "/var/cache/pip", + $projects = [] ) { - if $external_links == true { - $external_links_real = 'True' - } - else { - $external_links_real = 'False' - } - - if $follow_external_index_pages == true { - $follow_external_index_pages_real = 'True' - } - else { - $follow_external_index_pages_real = 'False' - } - - $packages = [ 'nginx', - 'python-pip' ] - - package { $packages: + package { 'nginx': ensure => present, } - # Build the mirror config file based on options provided. + package { 'pip': + ensure => latest, + provider => 'pip', + require => Package['python-pip'], + } - file { 'pypimirror.cfg': - path => '/etc/pypimirror.cfg', - ensure => present, - mode => 644, + file { '/usr/local/mirror_scripts': + ensure => 'directory', + mode => 755, owner => 'root', group => 'root', - content => template('pypimirror/config.erb'), + } + + file { $pip_download: + ensure => 'directory', + mode => 755, + owner => 'root', + group => 'root', + } + + file { $pip_cache: + ensure => 'directory', + mode => 755, + owner => 'root', + group => 'root', + } + + file { '/usr/local/mirror_scripts/run-mirror.sh': + ensure => present, + mode => 755, + owner => 'root', + group => 'root', + content => template('pypimirror/run-mirror.sh.erb'), + require => File['/usr/local/mirror_scripts'], } - file { '/usr/local/z3c.pypimirror': - ensure => absent, + file { '/usr/local/mirror_scripts/pull-repo.sh': + ensure => present, + mode => 755, + owner => 'root', + group => 'root', + source => "puppet:///modules/pypimirror/pull-repo.sh", + require => File['/usr/local/mirror_scripts'], } - # if we already have the repo the pull updates - - exec { "update_pypi_mirror": - command => "git pull --ff-only", - cwd => "/usr/local/pypi-mirror", - path => "/bin:/usr/bin", - onlyif => "test -d /usr/local/pypi-mirror", - before => Exec["get_pypi_mirror"], - } - - # otherwise get a new clone of it - - exec { "get_pypi_mirror": - command => "git clone git://github.com/openstack-ci/pypi-mirror.git /usr/local/pypi-mirror", - path => "/bin:/usr/bin", - onlyif => "test ! -d /usr/local/pypi-mirror" - } - - exec { "install_pypi_mirror": - command => "python setup.py install", - cwd => "/usr/local/pypi-mirror", - path => "/bin:/usr/bin", - subscribe => [ Exec["get_pypi_mirror"], Exec["update_pypi_mirror"] ], - } - - exec { "initialize_mirror": - command => "pypimirror --initial-fetch /etc/pypimirror.cfg", - path => "/bin:/usr/bin:/usr/local/bin", - onlyif => "test ! -d ${mirror_file_path}", - require => [ Exec["get_pypi_mirror"], Exec["install_pypi_mirror"] ], + file { '/usr/local/mirror_scripts/process_cache.py': + ensure => present, + mode => 755, + owner => 'root', + group => 'root', + source => "puppet:///modules/pypimirror/process_cache.py", + require => File['/usr/local/mirror_scripts'], } # Add cron job to update the mirror cron { "update_mirror": user => root, - hour => 0, - command => '/usr/local/bin/pypimirror --initial-fetch /etc/pypimirror.cfg', - require => Exec["install_pypi_mirror"], + minute => "0", + command => '/usr/local/mirror_scripts/run-mirror.sh', + require => File["/usr/local/mirror_scripts/run-mirror.sh"], } # Rotate the mirror log file diff --git a/modules/pypimirror/templates/config.erb b/modules/pypimirror/templates/config.erb deleted file mode 100644 index 4394a645a0..0000000000 --- a/modules/pypimirror/templates/config.erb +++ /dev/null @@ -1,62 +0,0 @@ -[DEFAULT] -# the root folder of all mirrored packages. -# if necessary it will be created for you -mirror_file_path = <%= mirror_file_path %> - -# where's your mirror on the net? -base_url = <%= base_url %> - -# lock file to avoid duplicate runs of the mirror script -lock_file_name = /var/lock/pypi-poll-access.lock - -# days to fetch in past on update -fetch_since_days = <%= fetch_since_days %> - -# Pattern for package files, only those matching will be mirrored -filename_matches = - *.zip - *.tgz - *.egg - *.tar.gz - *.tar.bz2 - -# Pattern for package names; only packages having matching names will -# be mirrored -package_matches = - <% package_matches.each do |match| -%> - <%= match %> - <% end -%> - -# remove packages not on pypi (or externals) anymore -cleanup = True - -# create index.html files -create_indexes = True - -# be more verbose -verbose = True - -# resolve download_url links on pypi which point to files and download -# the files from there (if they match filename_matches). -# The filename and filesize (from the download header) are used -# to find out if the file is already on the mirror. Not all servers -# support the content-length header, so be prepared to download -# a lot of data on each mirror update. -# This is highly experimental and shouldn't be used right now. -# -# NOTE: This option should only be set to True if package_matches is not -# set to '*' - otherwise you will mirror a huge amount of data. BE CAREFUL -# using this option!!! -external_links = <%= external_links_real %> - -# similar to 'external_links' but also follows an index page if no -# download links are available on the referenced download_url page -# of a given package. -# -# NOTE: This option should only be set to True if package_matches is not -# set to '*' - otherwise you will mirror a huge amount of data. BE CAREFUL -# using this option!!! -follow_external_index_pages = <%= follow_external_index_pages_real %> - -# logfile -log_filename = <%= log_filename %> diff --git a/modules/pypimirror/templates/run-mirror.sh.erb b/modules/pypimirror/templates/run-mirror.sh.erb new file mode 100644 index 0000000000..451304dcf9 --- /dev/null +++ b/modules/pypimirror/templates/run-mirror.sh.erb @@ -0,0 +1,12 @@ +#!/bin/bash +# This file is managed by puppet. +# https://github.com/openstack/openstack-ci-puppet + +export PIP_DOWNLOAD_CACHE=<%= pip_cache %> +export PIP_TEMP_DOWNLOAD=<%= pip_download %> +export MIRROR_FILE_PATH=<%= mirror_file_path %> +export LOG_FILENAME=<%= log_filename %> +<% projects.each do |project| -%> +/usr/local/mirror_scripts/pull-repo.sh <%= project %> >>$LOG_FILENAME 2>&1 +<% end -%> +python /usr/local/mirror_scripts/process_cache.py ${PIP_DOWNLOAD_CACHE} ${MIRROR_FILE_PATH}