Add pypimirror module.

New module to setup a Python Package Index mirror. Nginx is setup
to serve as the mirror web server. The default setup is to mirror
everything. Example of setting up a new mirror, restricting what
we will mirror:

  node "mymirror.mydomain.org" {
    include openstack_cron
    class { "pypimirror":
      base_url => "http://mymirror.mydomain.org",
      package_matches = ["nose", "pep8", "simplejson"],
    }
  }

To use it:

  pip install <package> -i http://mymirror.mydomain.org

Change-Id: I36234b8242df0e9d1e7dca7e896005e9e46a9edf

modules/pypimirror/files/default

@@ -0,0 +1,9 @@
+server {
+  listen 80;
+
+  index index.html index.htm;
+
+  location / {
+    root /usr/local/pypimirror;
+  }
+}

modules/pypimirror/manifests/init.pp

@@ -0,0 +1,109 @@
+class pypimirror ( $base_url,
+                   $log_filename = "/var/log/pypimirror.log",
+                   $mirror_file_path = "/var/lib/pypimirror",
+                   $fetch_since_days = 1,
+                   $package_matches = ["*"],
+                   $external_links = true,
+                   $follow_external_index_pages = false )
+{
+
+  if $external_links == true {
+    $external_links_real = 'True'
+  }
+  else {
+    $external_links_real = 'False'
+  }
+
+  if $follow_external_index_pages == true {
+    $follow_external_index_pages_real = 'True'
+  }
+  else {
+    $follow_external_index_pages_real = 'False'
+  }
+
+  $packages = [ 'mercurial',
+                'nginx',
+                'python-pip' ]
+
+  package { $packages:
+    ensure => present,
+  }
+
+  # Build the mirror config file based on options provided.
+
+  file { 'pypimirror.cfg':
+    path => '/etc/pypimirror.cfg',
+    ensure => present,
+    mode => 644,
+    owner => 'root',
+    group => 'root',
+    content => template('pypimirror/config.erb'),
+  }
+
+  # if we already have the repo the pull updates
+
+  exec { "update_z3c.pypimirror":
+    command => "hg pull",
+    cwd => "/usr/local/z3c.pypimirror",
+    path => "/bin:/usr/bin",
+    onlyif => "test -d /usr/local/z3c.pypimirror",
+    before => Exec["get_z3c.pypimirror"],
+  }
+
+  # otherwise get a new clone of it
+
+  exec { "get_z3c.pypimirror":
+    command => "hg clone https://bitbucket.org/rsyring/z3c.pypimirror /usr/local/z3c.pypimirror",
+    path => "/bin:/usr/bin",
+    onlyif => "test ! -d /usr/local/z3c.pypimirror"
+  }
+
+  exec { "install_z3c.pypimirror":
+    command => "python setup.py install",
+    cwd => "/usr/local/z3c.pypimirror",
+    path => "/bin:/usr/bin",
+    subscribe => [ Exec["get_z3c.pypimirror"], Exec["update_z3c.pypimirror"] ],
+  }
+
+  exec { "initialize_mirror":
+    command => "pypimirror --initial-fetch /etc/pypimirror.cfg",
+    path => "/bin:/usr/bin:/usr/local/bin",
+    onlyif => "test ! -d ${mirror_file_path}",
+    require => [ Exec["get_z3c.pypimirror"], Exec["install_z3c.pypimirror"] ],
+  }
+
+  # Add cron job to update the mirror
+
+  cron { "update_mirror":
+    user => root,
+    hour => 0,
+    command => '/usr/local/bin/pypimirror --update-fetch /etc/pypimirror.cfg',
+    require => Exec["install_z3c.pypimirror"],
+  }
+
+  # Rotate the mirror log file
+
+  include logrotate
+  logrotate::file {"pypimirror":
+    log => $log_filename,
+    options => ["compress", "delaycompress", "missingok", "rotate 7", "daily", "notifempty"],
+    require => Cron["update_mirror"],
+  }
+
+  # Setup the web server
+
+  service { "nginx":
+    ensure => running,
+    hasrestart => true
+  }
+
+  file { "/etc/nginx/sites-available/default":
+    ensure => present,
+    source => "puppet:///modules/pypimirror/default",
+    replace => true,
+    owner => "root",
+    group => "root",
+    require => Package["nginx"],
+    notify => Service["nginx"],
+  }
+}

modules/pypimirror/templates/config.erb

@@ -0,0 +1,62 @@
+[DEFAULT]
+# the root folder of all mirrored packages.
+# if necessary it will be created for you
+mirror_file_path = <%= mirror_file_path %>
+
+# where's your mirror on the net?
+base_url = <%= base_url %>
+
+# lock file to avoid duplicate runs of the mirror script
+lock_file_name = /var/lock/pypi-poll-access.lock
+
+# days to fetch in past on update
+fetch_since_days = <%= fetch_since_days %>
+
+# Pattern for package files, only those matching will be mirrored
+filename_matches =
+    *.zip
+    *.tgz
+    *.egg
+    *.tar.gz
+    *.tar.bz2
+
+# Pattern for package names; only packages having matching names will
+# be mirrored
+package_matches = 
+ <% package_matches.each do |match| -%>
+   <%= match %>
+ <% end -%>
+
+# remove packages not on pypi (or externals) anymore
+cleanup = True
+
+# create index.html files
+create_indexes = True
+
+# be more verbose
+verbose = True
+
+# resolve download_url links on pypi which point to files and download
+# the files from there (if they match filename_matches).
+# The filename and filesize (from the download header) are used
+# to find out if the file is already on the mirror. Not all servers
+# support the content-length header, so be prepared to download
+# a lot of data on each mirror update.
+# This is highly experimental and shouldn't be used right now.
+# 
+# NOTE: This option should only be set to True if package_matches is not 
+# set to '*' - otherwise you will mirror a huge amount of data. BE CAREFUL
+# using this option!!!
+external_links = <%= external_links_real %>
+
+# similar to 'external_links' but also follows an index page if no
+# download links are available on the referenced download_url page
+# of a given package.
+#
+# NOTE: This option should only be set to True if package_matches is not 
+# set to '*' - otherwise you will mirror a huge amount of data. BE CAREFUL
+# using this option!!!
+follow_external_index_pages = <%= follow_external_index_pages_real %>
+
+# logfile 
+log_filename = <%= log_filename %>