Use pip cache instead of full mirror.
Change-Id: I14ae08eb6c5a402f26de6c20234ac434f8f4b2c9
This commit is contained in:
Monty Taylor
parent
3f6941cd87
commit
008fb47583
@ -376,11 +376,38 @@ node "eavesdrop.openstack.org" {
|
||||
|
||||
node "pypi.openstack.org" {
|
||||
include openstack_cron
|
||||
|
||||
# include jenkins slave so that build deps are there for the pip download
|
||||
class { 'jenkins_slave':
|
||||
ssh_key => "",
|
||||
user => false
|
||||
}
|
||||
|
||||
class { 'openstack_server':
|
||||
iptables_public_tcp_ports => [80]
|
||||
}
|
||||
|
||||
class { "pypimirror":
|
||||
base_url => "http://pypi.openstack.org",
|
||||
projects => [
|
||||
'cinder',
|
||||
'glance',
|
||||
'horizon',
|
||||
'keystone',
|
||||
'melange',
|
||||
'nova',
|
||||
'openstack-common',
|
||||
'python-cinderclient',
|
||||
'python-glanceclient',
|
||||
'python-keystoneclient',
|
||||
'python-melangeclient',
|
||||
'python-novaclient',
|
||||
'python-openstackclient',
|
||||
'python-quantumclient',
|
||||
'python-swiftclient',
|
||||
'quantum',
|
||||
'swift'
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
27
modules/pypimirror/files/pull-repo.sh
Normal file
27
modules/pypimirror/files/pull-repo.sh
Normal file
@ -0,0 +1,27 @@
|
||||
#!/bin/bash
|
||||
# This file is managed by puppet.
|
||||
# https://github.com/openstack/openstack-ci-puppet
|
||||
|
||||
export PIP_DOWNLOAD_CACHE=${PIP_DOWNLOAD_CACHE:-/var/cache/pip}
|
||||
export PIP_TEMP_DOWNLOAD=${PIP_TEMP_DOWNLOAD:-/var/lib/pip-download}
|
||||
|
||||
project=$1
|
||||
pip_command='pip install -M -U -I --exists-action=w --no-install'
|
||||
|
||||
cd ${PIP_TEMP_DOWNLOAD}
|
||||
if [ ! -d ${project} ] ; then
|
||||
git clone git://github.com/openstack/${project}.git ${project} >/dev/null 2>&1
|
||||
fi
|
||||
cd ${project}
|
||||
git fetch origin
|
||||
for branch in `git branch -a | grep remotes.origin | grep -v origin.HEAD | awk '{print $1}' ` ; do
|
||||
git reset --hard $branch
|
||||
git clean -x -f -d -q
|
||||
echo "*********************"
|
||||
echo "Fetching pip requires for $project:$branch"
|
||||
for requires_file in tools/pip-requires tools/test-requires ; do
|
||||
if [ -f ${requires_file} ] ; then
|
||||
$pip_command -r $requires_file
|
||||
fi
|
||||
done
|
||||
done
|
||||
@ -1,87 +1,76 @@
|
||||
class pypimirror ( $base_url,
|
||||
$log_filename = "/var/log/pypimirror.log",
|
||||
$mirror_file_path = "/var/lib/pypimirror",
|
||||
$fetch_since_days = 1,
|
||||
$package_matches = ["*"],
|
||||
$external_links = true,
|
||||
$follow_external_index_pages = true )
|
||||
$pip_download = "/var/lib/pip-download",
|
||||
$pip_cache = "/var/cache/pip",
|
||||
$projects = [] )
|
||||
{
|
||||
|
||||
if $external_links == true {
|
||||
$external_links_real = 'True'
|
||||
}
|
||||
else {
|
||||
$external_links_real = 'False'
|
||||
}
|
||||
|
||||
if $follow_external_index_pages == true {
|
||||
$follow_external_index_pages_real = 'True'
|
||||
}
|
||||
else {
|
||||
$follow_external_index_pages_real = 'False'
|
||||
}
|
||||
|
||||
$packages = [ 'nginx',
|
||||
'python-pip' ]
|
||||
|
||||
package { $packages:
|
||||
package { 'nginx':
|
||||
ensure => present,
|
||||
}
|
||||
|
||||
# Build the mirror config file based on options provided.
|
||||
package { 'pip':
|
||||
ensure => latest,
|
||||
provider => 'pip',
|
||||
require => Package['python-pip'],
|
||||
}
|
||||
|
||||
file { 'pypimirror.cfg':
|
||||
path => '/etc/pypimirror.cfg',
|
||||
ensure => present,
|
||||
mode => 644,
|
||||
file { '/usr/local/mirror_scripts':
|
||||
ensure => 'directory',
|
||||
mode => 755,
|
||||
owner => 'root',
|
||||
group => 'root',
|
||||
content => template('pypimirror/config.erb'),
|
||||
}
|
||||
|
||||
file { $pip_download:
|
||||
ensure => 'directory',
|
||||
mode => 755,
|
||||
owner => 'root',
|
||||
group => 'root',
|
||||
}
|
||||
|
||||
file { $pip_cache:
|
||||
ensure => 'directory',
|
||||
mode => 755,
|
||||
owner => 'root',
|
||||
group => 'root',
|
||||
}
|
||||
|
||||
file { '/usr/local/mirror_scripts/run-mirror.sh':
|
||||
ensure => present,
|
||||
mode => 755,
|
||||
owner => 'root',
|
||||
group => 'root',
|
||||
content => template('pypimirror/run-mirror.sh.erb'),
|
||||
require => File['/usr/local/mirror_scripts'],
|
||||
}
|
||||
|
||||
file { '/usr/local/z3c.pypimirror':
|
||||
ensure => absent,
|
||||
file { '/usr/local/mirror_scripts/pull-repo.sh':
|
||||
ensure => present,
|
||||
mode => 755,
|
||||
owner => 'root',
|
||||
group => 'root',
|
||||
source => "puppet:///modules/pypimirror/pull-repo.sh",
|
||||
require => File['/usr/local/mirror_scripts'],
|
||||
}
|
||||
|
||||
# if we already have the repo the pull updates
|
||||
|
||||
exec { "update_pypi_mirror":
|
||||
command => "git pull --ff-only",
|
||||
cwd => "/usr/local/pypi-mirror",
|
||||
path => "/bin:/usr/bin",
|
||||
onlyif => "test -d /usr/local/pypi-mirror",
|
||||
before => Exec["get_pypi_mirror"],
|
||||
}
|
||||
|
||||
# otherwise get a new clone of it
|
||||
|
||||
exec { "get_pypi_mirror":
|
||||
command => "git clone git://github.com/openstack-ci/pypi-mirror.git /usr/local/pypi-mirror",
|
||||
path => "/bin:/usr/bin",
|
||||
onlyif => "test ! -d /usr/local/pypi-mirror"
|
||||
}
|
||||
|
||||
exec { "install_pypi_mirror":
|
||||
command => "python setup.py install",
|
||||
cwd => "/usr/local/pypi-mirror",
|
||||
path => "/bin:/usr/bin",
|
||||
subscribe => [ Exec["get_pypi_mirror"], Exec["update_pypi_mirror"] ],
|
||||
}
|
||||
|
||||
exec { "initialize_mirror":
|
||||
command => "pypimirror --initial-fetch /etc/pypimirror.cfg",
|
||||
path => "/bin:/usr/bin:/usr/local/bin",
|
||||
onlyif => "test ! -d ${mirror_file_path}",
|
||||
require => [ Exec["get_pypi_mirror"], Exec["install_pypi_mirror"] ],
|
||||
file { '/usr/local/mirror_scripts/process_cache.py':
|
||||
ensure => present,
|
||||
mode => 755,
|
||||
owner => 'root',
|
||||
group => 'root',
|
||||
source => "puppet:///modules/pypimirror/process_cache.py",
|
||||
require => File['/usr/local/mirror_scripts'],
|
||||
}
|
||||
|
||||
# Add cron job to update the mirror
|
||||
|
||||
cron { "update_mirror":
|
||||
user => root,
|
||||
hour => 0,
|
||||
command => '/usr/local/bin/pypimirror --initial-fetch /etc/pypimirror.cfg',
|
||||
require => Exec["install_pypi_mirror"],
|
||||
minute => "0",
|
||||
command => '/usr/local/mirror_scripts/run-mirror.sh',
|
||||
require => File["/usr/local/mirror_scripts/run-mirror.sh"],
|
||||
}
|
||||
|
||||
# Rotate the mirror log file
|
||||
|
||||
@ -1,62 +0,0 @@
|
||||
[DEFAULT]
|
||||
# the root folder of all mirrored packages.
|
||||
# if necessary it will be created for you
|
||||
mirror_file_path = <%= mirror_file_path %>
|
||||
|
||||
# where's your mirror on the net?
|
||||
base_url = <%= base_url %>
|
||||
|
||||
# lock file to avoid duplicate runs of the mirror script
|
||||
lock_file_name = /var/lock/pypi-poll-access.lock
|
||||
|
||||
# days to fetch in past on update
|
||||
fetch_since_days = <%= fetch_since_days %>
|
||||
|
||||
# Pattern for package files, only those matching will be mirrored
|
||||
filename_matches =
|
||||
*.zip
|
||||
*.tgz
|
||||
*.egg
|
||||
*.tar.gz
|
||||
*.tar.bz2
|
||||
|
||||
# Pattern for package names; only packages having matching names will
|
||||
# be mirrored
|
||||
package_matches =
|
||||
<% package_matches.each do |match| -%>
|
||||
<%= match %>
|
||||
<% end -%>
|
||||
|
||||
# remove packages not on pypi (or externals) anymore
|
||||
cleanup = True
|
||||
|
||||
# create index.html files
|
||||
create_indexes = True
|
||||
|
||||
# be more verbose
|
||||
verbose = True
|
||||
|
||||
# resolve download_url links on pypi which point to files and download
|
||||
# the files from there (if they match filename_matches).
|
||||
# The filename and filesize (from the download header) are used
|
||||
# to find out if the file is already on the mirror. Not all servers
|
||||
# support the content-length header, so be prepared to download
|
||||
# a lot of data on each mirror update.
|
||||
# This is highly experimental and shouldn't be used right now.
|
||||
#
|
||||
# NOTE: This option should only be set to True if package_matches is not
|
||||
# set to '*' - otherwise you will mirror a huge amount of data. BE CAREFUL
|
||||
# using this option!!!
|
||||
external_links = <%= external_links_real %>
|
||||
|
||||
# similar to 'external_links' but also follows an index page if no
|
||||
# download links are available on the referenced download_url page
|
||||
# of a given package.
|
||||
#
|
||||
# NOTE: This option should only be set to True if package_matches is not
|
||||
# set to '*' - otherwise you will mirror a huge amount of data. BE CAREFUL
|
||||
# using this option!!!
|
||||
follow_external_index_pages = <%= follow_external_index_pages_real %>
|
||||
|
||||
# logfile
|
||||
log_filename = <%= log_filename %>
|
||||
12
modules/pypimirror/templates/run-mirror.sh.erb
Normal file
12
modules/pypimirror/templates/run-mirror.sh.erb
Normal file
@ -0,0 +1,12 @@
|
||||
#!/bin/bash
|
||||
# This file is managed by puppet.
|
||||
# https://github.com/openstack/openstack-ci-puppet
|
||||
|
||||
export PIP_DOWNLOAD_CACHE=<%= pip_cache %>
|
||||
export PIP_TEMP_DOWNLOAD=<%= pip_download %>
|
||||
export MIRROR_FILE_PATH=<%= mirror_file_path %>
|
||||
export LOG_FILENAME=<%= log_filename %>
|
||||
<% projects.each do |project| -%>
|
||||
/usr/local/mirror_scripts/pull-repo.sh <%= project %> >>$LOG_FILENAME 2>&1
|
||||
<% end -%>
|
||||
python /usr/local/mirror_scripts/process_cache.py ${PIP_DOWNLOAD_CACHE} ${MIRROR_FILE_PATH}
|
||||
Loading…
Reference in New Issue
Block a user