Update to multi-mirror jeepyb.

Requires https://review.openstack.org/#/c/23771/

Change-Id: If2db2516789a01fc60e6dbd0be062a8bf00ed9bc
Reviewed-on: https://review.openstack.org/23774
Reviewed-by: Jeremy Stanley <fungi@yuggoth.org>
Approved: James E. Blair <corvus@inaugust.com>
Tested-by: Jenkins

modules/openstack_project/files/pypi-mirror.yaml

@@ -0,0 +1,15 @@
+# This file is managed by puppet.
+# https://github.com/openstack-infra/config
+
+cache-root: /var/cache/pypimirror
+
+mirrors:
+  - name: openstack
+    projects:
+      - https://github.com/openstack/requirements
+    output: /var/lib/pypimirror/openstack
+
+  - name: openstack-infra
+    projects:
+      - https://github.com/openstack-infra/config
+    output: /var/lib/pypimirror/openstack-infra

modules/openstack_project/manifests/pypi.pp

@@ -18,6 +18,19 @@ class openstack_project::pypi (
   }
 
   class { 'pypimirror':
-    projects => $openstack_project::project_list,
+    mirror_config => '/etc/openstackci/pypi-mirror.yaml',
   }
+
+  file { '/etc/openstackci':
+    ensure  => directory,
+    mode    => '0755',
+    owner   => 'root',
+    group   => 'root',
+  }
+
+  file { '/etc/openstackci/pypi-mirror.yaml':
+    ensure => present,
+    source => 'puppet:///modules/openstack_project/pypi-mirror.yaml',
+  }
+
 }

modules/pypimirror/manifests/init.pp

@@ -2,95 +2,86 @@
 #
 class pypimirror(
   $vhost_name = $::fqdn,
-  $log_filename = '/var/log/pypimirror.log',
+  $mirror_config = '',
-  $mirror_file_path = '/var/lib/pypimirror',
+  $mirror_root = '/var/lib/pypimirror',
-  $pip_download = '/var/lib/pip-download',
-  $pip_cache = '/var/cache/pip',
-  $git_source = 'https://github.com',
-  $local_git_dir = '/var/lib/git',
-  $ssh_project_key = 'UNDEF',
-  $projects = []
 ) {
 
   include apache
   include pip
-  include remove_nginx
   include jeepyb
 
+  $log_root = '/var/log/pypimirror/'
+  $log_filename = "${log_root}/pypimirror.log"
+  $cache_root = '/var/cache/pypimirror'
+
   if ! defined(Package['python-yaml']) {
     package { 'python-yaml':
       ensure => present,
     }
   }
 
-  file { '/usr/local/mirror_scripts':
+  user { 'mirror':
-    ensure => directory,
+    ensure     => present,
-    mode   => '0755',
+    home       => '/home/mirror',
-    owner  => 'root',
+    shell      => '/bin/bash',
-    group  => 'root',
+    gid        => 'mirror',
+    managehome => true,
+    require    => Group['mirror'],
   }
 
-  file { $pip_download:
+  group { 'mirror':
-    ensure => directory,
+    ensure => present,
-    mode   => '0755',
-    owner  => 'root',
-    group  => 'root',
   }
 
-  file { $pip_cache:
+  file { $log_root:
-    ensure => directory,
+    ensure  => directory,
-    mode   => '0755',
+    mode    => '0755',
-    owner  => 'root',
+    owner   => 'mirror',
-    group  => 'root',
+    group   => 'mirror',
+    require => User['mirror'],
   }
 
-  file { '/etc/openstackci':
+  file { $cache_root:
-    ensure => directory,
+    ensure  => directory,
-    owner  => 'root',
+    mode    => '0755',
+    owner   => 'mirror',
+    group   => 'mirror',
+    require => User['mirror'],
   }
 
-  file { '/etc/openstackci/projects.yaml':
+  file { $mirror_root:
-    ensure  => present,
+    ensure  => directory,
-    owner   => 'root',
+    mode    => '0755',
-    group   => 'root',
+    owner   => 'mirror',
-    mode    => '0444',
+    group   => 'mirror',
-    content => template('openstack_project/review.projects.yaml.erb'),
+    require => User['mirror'],
-    replace => true,
   }
 
-  file { '/usr/local/mirror_scripts/run-mirror.sh':
+  file { '/usr/local/bin/run-mirror.sh':
     ensure  => present,
     mode    => '0755',
     owner   => 'root',
     group   => 'root',
     content => template('pypimirror/run-mirror.sh.erb'),
-    require => [
-      File['/usr/local/mirror_scripts'],
-      Class[pip],
-    ],
-  }
-
-  file { '/usr/local/mirror_scripts/run_mirror.py':
-    ensure  => absent,
-  }
-
-  file { '/usr/local/mirror_scripts/pull-repo.sh':
-    ensure  => absent,
-  }
-
-  file { '/usr/local/mirror_scripts/process_cache.py':
-    ensure  => absent,
   }
 
   # Add cron job to update the mirror
 
   cron { 'update_mirror':
+    ensure  => absent,
     user    => 'root',
     hour    => '0',
-    command => '/usr/local/mirror_scripts/run-mirror.sh',
+    command => '/usr/local/bin/run-mirror.sh',
     require => File['/usr/local/mirror_scripts/run-mirror.sh'],
   }
 
+  cron { 'update_pypi_mirror':
+    user    => 'mirror',
+    hour    => '0',
+    command => '/usr/local/bin/run-mirror.sh',
+    require => File['/usr/local/bin/run-mirror.sh'],
+  }
+
   # Rotate the mirror log file
 
   include logrotate
@@ -109,7 +100,7 @@ class pypimirror(
 
   apache::vhost { $vhost_name:
     port     => 80,
-    docroot  => $mirror_file_path,
+    docroot  => $mirror_root,
     priority => 50,
   }
 }

modules/pypimirror/templates/run-mirror.sh.erb

@@ -2,9 +2,4 @@
 # This file is managed by puppet.
 # https://github.com/openstack-infra/config
 
-export PIP_DOWNLOAD_CACHE=<%= pip_cache %>
+/usr/local/bin/run-mirror -c <%= mirror_config %> >> <%= log_filename %>
-export PIP_TEMP_DOWNLOAD=<%= pip_download %>
-export MIRROR_FILE_PATH=<%= mirror_file_path %>
-export LOG_FILENAME=<%= log_filename %>
-/usr/local/bin/run-mirror <%= git_source %> >>$LOG_FILENAME
-/usr/local/bin/process-cache ${PIP_DOWNLOAD_CACHE} ${MIRROR_FILE_PATH}