Add wheel mirror to mirrors
This patch adds a new "wheel" directory to the pypi mirrors, as an rsync target for our built wheel packages. A rewrite rule has been added in anticipation of serving the wheels from an AFS drive. Since AFS has a practical folder size limit, we are using /a/a /s/sp/split /s/st/style directory structure that should be AFS-tolerant. The rewrite rule creates the necessary mappings that make the packages available to pip. Example: HTTP GET /Babel/ -> /B/Ba/Babel/ Furthermore, a cron job has been added to periodically generate a human-readable index of these mappings, in accordance with PEP503. While frequently regenerated, this index should only change meaningfully if a new package is added to the wheel, as it only represents the package names themselves, rather than the available versions of said package. Change-Id: I743fc3ec629eea225c981d6e870751f33e77d7c6
This commit is contained in:
Michael Krotscheck
parent
5e63349100
commit
787b408d84
@ -0,0 +1,30 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
# This script generates an index file for every directory in the wheel mirror
|
||||||
|
# hierarchy.
|
||||||
|
|
||||||
|
DATA_DIRECTORY=$1
|
||||||
|
|
||||||
|
function build_index {
|
||||||
|
index="<!DOCTYPE html>"
|
||||||
|
index="$index<html><body><ul>"
|
||||||
|
|
||||||
|
# Match all strings that are /A/A(B/AB[C..])?$
|
||||||
|
regex="\/([^/])\/\1(([^/])\/\1\3[^/]+)?$"
|
||||||
|
for f in $(find $1 -type d -mindepth 2 -maxdepth 3); do
|
||||||
|
# Pull only the AFS-Matching directories
|
||||||
|
if [[ $f =~ $regex ]]; then
|
||||||
|
# Get the last name in the folder path, the package name.
|
||||||
|
dir=$(basename $BASH_REMATCH)
|
||||||
|
# Echo it out.
|
||||||
|
index="$index<li><a href=\"$dir/\">$dir</a></li>"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
index="$index</ul></body></html>"
|
||||||
|
echo $index > $2
|
||||||
|
}
|
||||||
|
|
||||||
|
for dir in $DATA_DIRECTORY/*; do
|
||||||
|
build_index $dir "$dir/index.html"
|
||||||
|
done
|
||||||
@ -12,6 +12,8 @@ class openstack_project::pypi (
|
|||||||
|
|
||||||
$mirror_root = '/srv/static'
|
$mirror_root = '/srv/static'
|
||||||
$pypi_root = "${mirror_root}/mirror"
|
$pypi_root = "${mirror_root}/mirror"
|
||||||
|
$www_root = "${pypi_root}/web"
|
||||||
|
$wheel_root = "${www_root}/wheel"
|
||||||
|
|
||||||
if ! defined(File[$mirror_root]) {
|
if ! defined(File[$mirror_root]) {
|
||||||
file { $mirror_root:
|
file { $mirror_root:
|
||||||
@ -24,16 +26,28 @@ class openstack_project::pypi (
|
|||||||
require => File[$mirror_root]
|
require => File[$mirror_root]
|
||||||
}
|
}
|
||||||
|
|
||||||
|
class { 'openstack_project::wheel_mirror':
|
||||||
|
data_directory => "${wheel_root}",
|
||||||
|
require => Class['Openstack_project::Pypi_mirror'],
|
||||||
|
}
|
||||||
|
|
||||||
include ::httpd
|
include ::httpd
|
||||||
|
|
||||||
|
if ! defined(Httpd::Mod['rewrite']) {
|
||||||
|
httpd::mod { 'rewrite':
|
||||||
|
ensure => present,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
::httpd::vhost { $vhost_name:
|
::httpd::vhost { $vhost_name:
|
||||||
port => 80,
|
port => 80,
|
||||||
priority => '50',
|
priority => '50',
|
||||||
docroot => "${pypi_root}/web",
|
docroot => $www_root,
|
||||||
require => Class['Openstack_project::Pypi_mirror'],
|
require => Class['Openstack_project::Pypi_mirror'],
|
||||||
|
template => 'openstack_project/pypi.vhost.erb',
|
||||||
}
|
}
|
||||||
|
|
||||||
file { "${pypi_root}/web/robots.txt":
|
file { "${www_root}/robots.txt":
|
||||||
ensure => present,
|
ensure => present,
|
||||||
owner => 'root',
|
owner => 'root',
|
||||||
group => 'root',
|
group => 'root',
|
||||||
|
|||||||
43
modules/openstack_project/manifests/wheel_mirror.pp
Normal file
43
modules/openstack_project/manifests/wheel_mirror.pp
Normal file
@ -0,0 +1,43 @@
|
|||||||
|
# == Class: openstack_project::wheel_mirror
|
||||||
|
#
|
||||||
|
class openstack_project::wheel_mirror (
|
||||||
|
$data_directory = '/srv/static/wheel',
|
||||||
|
$config_directory = '/etc/wheel_mirror'
|
||||||
|
) {
|
||||||
|
|
||||||
|
# The wheel mirror is a directory of python wheels, which have been rsynced'
|
||||||
|
# from the wheel build slaves.
|
||||||
|
file { "${data_directory}":
|
||||||
|
ensure => directory,
|
||||||
|
owner => 'root',
|
||||||
|
group => 'root',
|
||||||
|
}
|
||||||
|
|
||||||
|
file { "${config_directory}":
|
||||||
|
ensure => directory,
|
||||||
|
owner => 'root',
|
||||||
|
group => 'root',
|
||||||
|
}
|
||||||
|
|
||||||
|
file { "${config_directory}/rebuild_wheel_afs_index.sh":
|
||||||
|
ensure => present,
|
||||||
|
owner => 'root',
|
||||||
|
group => 'root',
|
||||||
|
mode => '0755',
|
||||||
|
source => "puppet:///modules/openstack_project/mirror/rebuild_wheel_afs_index.sh",
|
||||||
|
require => [
|
||||||
|
File["${config_directory}"],
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
# */15 * * * *
|
||||||
|
cron { 'rebuild wheel afs index':
|
||||||
|
name => 'rebuild-wheel-afs-index.cron',
|
||||||
|
command => "/bin/bash ${config_directory}/rebuild_wheel_afs_index.sh ${data_directory}",
|
||||||
|
user => root,
|
||||||
|
minute => '*/15',
|
||||||
|
require => [
|
||||||
|
File["${config_directory}/rebuild_wheel_afs_index.sh"],
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
37
modules/openstack_project/templates/pypi.vhost.erb
Normal file
37
modules/openstack_project/templates/pypi.vhost.erb
Normal file
@ -0,0 +1,37 @@
|
|||||||
|
# ************************************
|
||||||
|
# Managed by Puppet
|
||||||
|
# ************************************
|
||||||
|
|
||||||
|
NameVirtualHost <%= @vhost_name %>:<%= @port %>
|
||||||
|
<VirtualHost <%= @vhost_name %>:<%= @port %>>
|
||||||
|
ServerName <%= @srvname %>
|
||||||
|
<% if @serveraliases.is_a? Array -%>
|
||||||
|
<% @serveraliases.each do |name| -%><%= " ServerAlias #{name}\n" %><% end -%>
|
||||||
|
<% elsif @serveraliases != nil -%>
|
||||||
|
<%= " ServerAlias #{@serveraliases}" -%>
|
||||||
|
<% end -%>
|
||||||
|
DocumentRoot <%= @www_root %>
|
||||||
|
<Directory <%= @www_root %>>
|
||||||
|
Options <%= @options %>
|
||||||
|
AllowOverride None
|
||||||
|
Order allow,deny
|
||||||
|
allow from all
|
||||||
|
Satisfy any
|
||||||
|
<IfVersion >= 2.4>
|
||||||
|
Require all granted
|
||||||
|
</IfVersion>
|
||||||
|
</Directory>
|
||||||
|
|
||||||
|
# URL's are:
|
||||||
|
# /wheel/{distro}-{distro-version}/a/a/a-etc.whl
|
||||||
|
# /wheel/{distro}-{distro-version}/a/ab/abcd/abcd-etc.whl
|
||||||
|
# /wheel/{distro}-{distro-version}/a/ab/abcde/abcde-etc.whl
|
||||||
|
RewriteEngine On
|
||||||
|
RewriteRule ^/wheel/([^/]+)/([^/])/(.*)$ /wheel/$1/$2/$2/$3 [L]
|
||||||
|
RewriteRule ^/wheel/([^/]+)/([^/])([^/])([^/]*)/(.*)$ /wheel/$1/$2/$2$3/$2$3$4/$5 [L]
|
||||||
|
|
||||||
|
ErrorLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/<%= @name %>_error.log
|
||||||
|
LogLevel warn
|
||||||
|
CustomLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/<%= @name %>_access.log combined
|
||||||
|
ServerSignature Off
|
||||||
|
</VirtualHost>
|
||||||
Loading…
Reference in New Issue
Block a user