From 520065039ccd953c3b6105a15192f129966ef82d Mon Sep 17 00:00:00 2001 From: David Shrewsbury Date: Thu, 10 May 2012 17:52:07 -0400 Subject: [PATCH] Add pypimirror module. New module to setup a Python Package Index mirror. Nginx is setup to serve as the mirror web server. The default setup is to mirror everything. Example of setting up a new mirror, restricting what we will mirror: node "mymirror.mydomain.org" { include openstack_cron class { "pypimirror": base_url => "http://mymirror.mydomain.org", package_matches = ["nose", "pep8", "simplejson"], } } To use it: pip install -i http://mymirror.mydomain.org Change-Id: I36234b8242df0e9d1e7dca7e896005e9e46a9edf --- modules/pypimirror/files/default | 9 ++ modules/pypimirror/manifests/init.pp | 109 ++++++++++++++++++++++++ modules/pypimirror/templates/config.erb | 62 ++++++++++++++ 3 files changed, 180 insertions(+) create mode 100644 modules/pypimirror/files/default create mode 100644 modules/pypimirror/manifests/init.pp create mode 100644 modules/pypimirror/templates/config.erb diff --git a/modules/pypimirror/files/default b/modules/pypimirror/files/default new file mode 100644 index 0000000000..422a1c1532 --- /dev/null +++ b/modules/pypimirror/files/default @@ -0,0 +1,9 @@ +server { + listen 80; + + index index.html index.htm; + + location / { + root /usr/local/pypimirror; + } +} diff --git a/modules/pypimirror/manifests/init.pp b/modules/pypimirror/manifests/init.pp new file mode 100644 index 0000000000..7f6d604c50 --- /dev/null +++ b/modules/pypimirror/manifests/init.pp @@ -0,0 +1,109 @@ +class pypimirror ( $base_url, + $log_filename = "/var/log/pypimirror.log", + $mirror_file_path = "/var/lib/pypimirror", + $fetch_since_days = 1, + $package_matches = ["*"], + $external_links = true, + $follow_external_index_pages = false ) +{ + + if $external_links == true { + $external_links_real = 'True' + } + else { + $external_links_real = 'False' + } + + if $follow_external_index_pages == true { + $follow_external_index_pages_real = 'True' + } + else { + $follow_external_index_pages_real = 'False' + } + + $packages = [ 'mercurial', + 'nginx', + 'python-pip' ] + + package { $packages: + ensure => present, + } + + # Build the mirror config file based on options provided. + + file { 'pypimirror.cfg': + path => '/etc/pypimirror.cfg', + ensure => present, + mode => 644, + owner => 'root', + group => 'root', + content => template('pypimirror/config.erb'), + } + + # if we already have the repo the pull updates + + exec { "update_z3c.pypimirror": + command => "hg pull", + cwd => "/usr/local/z3c.pypimirror", + path => "/bin:/usr/bin", + onlyif => "test -d /usr/local/z3c.pypimirror", + before => Exec["get_z3c.pypimirror"], + } + + # otherwise get a new clone of it + + exec { "get_z3c.pypimirror": + command => "hg clone https://bitbucket.org/rsyring/z3c.pypimirror /usr/local/z3c.pypimirror", + path => "/bin:/usr/bin", + onlyif => "test ! -d /usr/local/z3c.pypimirror" + } + + exec { "install_z3c.pypimirror": + command => "python setup.py install", + cwd => "/usr/local/z3c.pypimirror", + path => "/bin:/usr/bin", + subscribe => [ Exec["get_z3c.pypimirror"], Exec["update_z3c.pypimirror"] ], + } + + exec { "initialize_mirror": + command => "pypimirror --initial-fetch /etc/pypimirror.cfg", + path => "/bin:/usr/bin:/usr/local/bin", + onlyif => "test ! -d ${mirror_file_path}", + require => [ Exec["get_z3c.pypimirror"], Exec["install_z3c.pypimirror"] ], + } + + # Add cron job to update the mirror + + cron { "update_mirror": + user => root, + hour => 0, + command => '/usr/local/bin/pypimirror --update-fetch /etc/pypimirror.cfg', + require => Exec["install_z3c.pypimirror"], + } + + # Rotate the mirror log file + + include logrotate + logrotate::file {"pypimirror": + log => $log_filename, + options => ["compress", "delaycompress", "missingok", "rotate 7", "daily", "notifempty"], + require => Cron["update_mirror"], + } + + # Setup the web server + + service { "nginx": + ensure => running, + hasrestart => true + } + + file { "/etc/nginx/sites-available/default": + ensure => present, + source => "puppet:///modules/pypimirror/default", + replace => true, + owner => "root", + group => "root", + require => Package["nginx"], + notify => Service["nginx"], + } +} diff --git a/modules/pypimirror/templates/config.erb b/modules/pypimirror/templates/config.erb new file mode 100644 index 0000000000..4394a645a0 --- /dev/null +++ b/modules/pypimirror/templates/config.erb @@ -0,0 +1,62 @@ +[DEFAULT] +# the root folder of all mirrored packages. +# if necessary it will be created for you +mirror_file_path = <%= mirror_file_path %> + +# where's your mirror on the net? +base_url = <%= base_url %> + +# lock file to avoid duplicate runs of the mirror script +lock_file_name = /var/lock/pypi-poll-access.lock + +# days to fetch in past on update +fetch_since_days = <%= fetch_since_days %> + +# Pattern for package files, only those matching will be mirrored +filename_matches = + *.zip + *.tgz + *.egg + *.tar.gz + *.tar.bz2 + +# Pattern for package names; only packages having matching names will +# be mirrored +package_matches = + <% package_matches.each do |match| -%> + <%= match %> + <% end -%> + +# remove packages not on pypi (or externals) anymore +cleanup = True + +# create index.html files +create_indexes = True + +# be more verbose +verbose = True + +# resolve download_url links on pypi which point to files and download +# the files from there (if they match filename_matches). +# The filename and filesize (from the download header) are used +# to find out if the file is already on the mirror. Not all servers +# support the content-length header, so be prepared to download +# a lot of data on each mirror update. +# This is highly experimental and shouldn't be used right now. +# +# NOTE: This option should only be set to True if package_matches is not +# set to '*' - otherwise you will mirror a huge amount of data. BE CAREFUL +# using this option!!! +external_links = <%= external_links_real %> + +# similar to 'external_links' but also follows an index page if no +# download links are available on the referenced download_url page +# of a given package. +# +# NOTE: This option should only be set to True if package_matches is not +# set to '*' - otherwise you will mirror a huge amount of data. BE CAREFUL +# using this option!!! +follow_external_index_pages = <%= follow_external_index_pages_real %> + +# logfile +log_filename = <%= log_filename %>