Serve pypi proxy cache from port 80

Oh easy_install.

In the configure-mirrors zuul-jobs role we write out ~/.pydistutils.cfg.
One of the things we do is:

  allow_hosts = {{ mirror_fqdn }}

which results in things like

  allow_hosts = mirror.dfw.rax.openstack.org

True story though - that doesn't match port 8080, so it breaks
easy_install if we set

  index_url = http://mirror.dfw.rax.openstack.org:8080/pypi/simple

Is there anything easy_install does right? No?

We should probably completely rethink how we're plumbing mirror info
into our nodes, but it is non-trivial to get a port into that file and
various OpenStack projects are currently blocked on the pypi mirror
being out of date.

Serve the pypi proxy content from port 80.

Change-Id: I518cf55fc6efe7db9493e4f5ec3601cb67f54999

modules/openstack_project/templates/mirror.vhost.erb

@@ -38,17 +38,53 @@ LogFormat "%h %l %u %t \"%r\" %>s %b %{cache-status}e \"%{Referer}i\" \"%{User-a
         </IfVersion>
     </Directory>
 
-    RewriteEngine On
+    # Caching reverse proxy for things that don't make sense in AFS
+    #
+    # General cache rules
+    CacheRoot "/var/cache/apache2/proxy"
+    CacheDirLevels 5
+    CacheDirLength 2
+    # SSL support
+    SSLProxyEngine on
+    # Prevent thundering herds.
+    CacheLock on
+    CacheLockPath "/tmp/mod_cache-lock"
+    CacheLockMaxAge 5
+    # 5GiB
+    CacheMaxFileSize 5368709120
+    CacheStoreExpired On
 
-    # Pypi's bandersnatch URL's are:
-    # /pypi/simple/index.html
-    # /pypi/simple/a/a/(index.html)?
-    # /pypi/simple/a/a/a-etc.whl
-    # /pypi/simple/a/abcd/(index.html)?
-    # /pypi/simple/a/abcd/abcd-etc.whl
-    RewriteCond %{REQUEST_URI} ^/pypi/simple/([^/])([^/]*)
-    RewriteCond %{DOCUMENT_ROOT}/pypi/simple/$1/$1$2 -d
-    RewriteRule ^/pypi/simple/([^/])([^/]*)(/.*)?$ /pypi/simple/$1/$1$2$3 [L]
+    # Added Aug 2017 in an attempt to avoid occasional 502 errors (around
+    # 0.05% of requests) of the type:
+    #
+    #  End of file found: ... AH01102: error reading status line from remote server ...
+    #
+    # Per [1]:
+    #
+    #  This avoids the "proxy: error reading status line from remote
+    #  server" error message caused by the race condition that the backend
+    #  server closed the pooled connection after the connection check by the
+    #  proxy and before data sent by the proxy reached the backend.
+    #
+    # [1] https://httpd.apache.org/docs/2.4/mod/mod_proxy_http.html
+    SetEnv proxy-initial-not-pooled 1
+
+    RewriteEngine On
+    # pypi
+    CacheEnable disk  "/pypi"
+    ProxyPass "/pypi/" "https://pypi.org/" ttl=120 keepalive=On retry=0
+    ProxyPassReverse "/pypi/" "https://pypi.org/
+
+    # files.pythonhosted.org
+    CacheEnable disk  "/pypifiles"
+    ProxyPass "/pypifiles/" "https://files.pythonhosted.org/" ttl=120 keepalive=On retry=0
+    ProxyPassReverse "/pypifiles/" "https://files.pythonhosted.org/"
+
+    # Rewrite the locations of the actual files
+    <Location /pypi>
+      SetOutputFilter INFLATE;SUBSTITUTE;DEFLATE
+      Substitute "s|https://files.pythonhosted.org/|/pypifiles/|ni"
+    </Location>
 
     # Wheel URL's are:
     # /wheel/{distro}-{distro-version}/a/a/a-etc.whl
@@ -58,29 +94,21 @@ LogFormat "%h %l %u %t \"%r\" %>s %b %{cache-status}e \"%{Referer}i\" \"%{User-a
     RewriteCond %{DOCUMENT_ROOT}/wheel/$1/$2/$2$3 -d
     RewriteRule ^/wheel/([^/]+)/([^/])([^/]*)(/.*)?$ /wheel/$1/$2/$2$3$4 [L]
 
-    # TODO(jhesketh): Remove this after bandersnatch implements pep503
-    # https://bitbucket.org/pypa/bandersnatch/pull-requests/20/fully-implement-pep-503-normalization/diff
-
     # Special cases for openstack.nose_plugin & backports.*
+    RewriteCond %{REQUEST_URI} ^/wheel/
     RewriteRule ^(.*)/openstack-nose-plugin(.*)$ $1/openstack.nose_plugin$2
+    RewriteCond %{REQUEST_URI} ^/wheel/
     RewriteRule ^(.*)/backports-(.*)$ $1/backports.$2
 
-    RewriteCond %{REQUEST_URI} ^/pypi/simple/([^/])([^/]*)
-    RewriteCond %{DOCUMENT_ROOT}/pypi/simple/$1/$1$2 -d
-    RewriteRule ^/pypi/simple/([^/])([^/]*)(/.*)?$ /pypi/simple/$1/$1$2$3 [L]
-
     # Try again but replacing -'s with .'s
+    RewriteCond %{REQUEST_URI} ^/wheel/
     RewriteCond %{DOCUMENT_ROOT}/%{REQUEST_URI} !-f
     RewriteCond %{DOCUMENT_ROOT}/%{REQUEST_URI} !-d
     RewriteRule (.*)-(.*) $1.$2 [N]
 
-    RewriteCond %{REQUEST_URI} ^/pypi/simple/([^/])([^/]*)
-    RewriteCond %{DOCUMENT_ROOT}/pypi/simple/$1/$1$2 -d
-    RewriteRule ^/pypi/simple/([^/])([^/]*)(/.*)?$ /pypi/simple/$1/$1$2$3 [L]
-
     ErrorLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/<%= @name %>_error.log
     LogLevel warn
-    CustomLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/<%= @name %>_access.log combined
+    CustomLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/<%= @name %>_access.log combined-cache
     ServerSignature Off
 </VirtualHost>