41fcff81b2
Oh easy_install.
In the configure-mirrors zuul-jobs role we write out ~/.pydistutils.cfg.
One of the things we do is:
allow_hosts = {{ mirror_fqdn }}
which results in things like
allow_hosts = mirror.dfw.rax.openstack.org
True story though - that doesn't match port 8080, so it breaks
easy_install if we set
index_url = http://mirror.dfw.rax.openstack.org:8080/pypi/simple
Is there anything easy_install does right? No?
We should probably completely rethink how we're plumbing mirror info
into our nodes, but it is non-trivial to get a port into that file and
various OpenStack projects are currently blocked on the pypi mirror
being out of date.
Serve the pypi proxy content from port 80.
Change-Id: I518cf55fc6efe7db9493e4f5ec3601cb67f54999
395 lines
14 KiB
Plaintext
395 lines
14 KiB
Plaintext
# ************************************
|
|
# Managed by Puppet
|
|
# ************************************
|
|
|
|
NameVirtualHost <%= @vhost_name %>:<%= @port %>
|
|
|
|
# Dedicated port for proxy caching, as not to affect afs mirrors.
|
|
Listen 8080
|
|
NameVirtualHost <%= @vhost_name %>:8080
|
|
|
|
Listen 8081
|
|
NameVirtualHost <%= @vhost_name %>:8081
|
|
|
|
Listen 8082
|
|
NameVirtualHost <%= @vhost_name %>:8082
|
|
|
|
|
|
LogFormat "%h %l %u %t \"%r\" %>s %b %{cache-status}e \"%{Referer}i\" \"%{User-agent}i\"" combined-cache
|
|
|
|
<VirtualHost <%= @vhost_name %>:<%= @port %>>
|
|
ServerName <%= @srvname %>
|
|
<% if @serveraliases.is_a? Array -%>
|
|
<% @serveraliases.each do |name| -%>
|
|
<%= " ServerAlias #{name}\n" %>
|
|
<% end %>
|
|
<% elsif @serveraliases != nil -%>
|
|
<%= " ServerAlias #{@serveraliases}" -%>
|
|
<% end %>
|
|
DocumentRoot <%= @docroot %>
|
|
<Directory <%= @docroot %>>
|
|
Options <%= @options %>
|
|
AllowOverride None
|
|
Order allow,deny
|
|
allow from all
|
|
Satisfy any
|
|
<IfVersion >= 2.4>
|
|
Require all granted
|
|
</IfVersion>
|
|
</Directory>
|
|
|
|
# Caching reverse proxy for things that don't make sense in AFS
|
|
#
|
|
# General cache rules
|
|
CacheRoot "/var/cache/apache2/proxy"
|
|
CacheDirLevels 5
|
|
CacheDirLength 2
|
|
# SSL support
|
|
SSLProxyEngine on
|
|
# Prevent thundering herds.
|
|
CacheLock on
|
|
CacheLockPath "/tmp/mod_cache-lock"
|
|
CacheLockMaxAge 5
|
|
# 5GiB
|
|
CacheMaxFileSize 5368709120
|
|
CacheStoreExpired On
|
|
|
|
# Added Aug 2017 in an attempt to avoid occasional 502 errors (around
|
|
# 0.05% of requests) of the type:
|
|
#
|
|
# End of file found: ... AH01102: error reading status line from remote server ...
|
|
#
|
|
# Per [1]:
|
|
#
|
|
# This avoids the "proxy: error reading status line from remote
|
|
# server" error message caused by the race condition that the backend
|
|
# server closed the pooled connection after the connection check by the
|
|
# proxy and before data sent by the proxy reached the backend.
|
|
#
|
|
# [1] https://httpd.apache.org/docs/2.4/mod/mod_proxy_http.html
|
|
SetEnv proxy-initial-not-pooled 1
|
|
|
|
RewriteEngine On
|
|
# pypi
|
|
CacheEnable disk "/pypi"
|
|
ProxyPass "/pypi/" "https://pypi.org/" ttl=120 keepalive=On retry=0
|
|
ProxyPassReverse "/pypi/" "https://pypi.org/
|
|
|
|
# files.pythonhosted.org
|
|
CacheEnable disk "/pypifiles"
|
|
ProxyPass "/pypifiles/" "https://files.pythonhosted.org/" ttl=120 keepalive=On retry=0
|
|
ProxyPassReverse "/pypifiles/" "https://files.pythonhosted.org/"
|
|
|
|
# Rewrite the locations of the actual files
|
|
<Location /pypi>
|
|
SetOutputFilter INFLATE;SUBSTITUTE;DEFLATE
|
|
Substitute "s|https://files.pythonhosted.org/|/pypifiles/|ni"
|
|
</Location>
|
|
|
|
# Wheel URL's are:
|
|
# /wheel/{distro}-{distro-version}/a/a/a-etc.whl
|
|
# /wheel/{distro}-{distro-version}/a/abcd/abcd-etc.whl
|
|
# /wheel/{distro}-{distro-version}/a/abcde/abcde-etc.whl
|
|
RewriteCond %{REQUEST_URI} ^/wheel/([^/]+)/([^/])([^/]*)
|
|
RewriteCond %{DOCUMENT_ROOT}/wheel/$1/$2/$2$3 -d
|
|
RewriteRule ^/wheel/([^/]+)/([^/])([^/]*)(/.*)?$ /wheel/$1/$2/$2$3$4 [L]
|
|
|
|
# Special cases for openstack.nose_plugin & backports.*
|
|
RewriteCond %{REQUEST_URI} ^/wheel/
|
|
RewriteRule ^(.*)/openstack-nose-plugin(.*)$ $1/openstack.nose_plugin$2
|
|
RewriteCond %{REQUEST_URI} ^/wheel/
|
|
RewriteRule ^(.*)/backports-(.*)$ $1/backports.$2
|
|
|
|
# Try again but replacing -'s with .'s
|
|
RewriteCond %{REQUEST_URI} ^/wheel/
|
|
RewriteCond %{DOCUMENT_ROOT}/%{REQUEST_URI} !-f
|
|
RewriteCond %{DOCUMENT_ROOT}/%{REQUEST_URI} !-d
|
|
RewriteRule (.*)-(.*) $1.$2 [N]
|
|
|
|
ErrorLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/<%= @name %>_error.log
|
|
LogLevel warn
|
|
CustomLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/<%= @name %>_access.log combined-cache
|
|
ServerSignature Off
|
|
</VirtualHost>
|
|
|
|
<VirtualHost <%= @vhost_name %>:8080>
|
|
ServerName <%= @srvname %>:8080
|
|
<% if @serveraliases.is_a? Array -%>
|
|
<% @serveraliases.each do |name| -%>
|
|
<%= " ServerAlias #{name}:8080\n" %>
|
|
<% end %>
|
|
<% elsif @serveraliases != nil -%>
|
|
<%= " ServerAlias #{@serveraliases}:8080" -%>
|
|
<% end %>
|
|
|
|
# Disable directory listing by default.
|
|
<Directory />
|
|
Order Deny,Allow
|
|
Deny from all
|
|
Options None
|
|
AllowOverride None
|
|
</Directory>
|
|
|
|
ErrorLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/<%= @name %>_8080_error.log
|
|
LogLevel warn
|
|
CustomLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/<%= @name %>_8080_access.log combined-cache
|
|
ServerSignature Off
|
|
|
|
# Caching reverse proxy for things that don't make sense in AFS
|
|
#
|
|
# General cache rules
|
|
CacheRoot "/var/cache/apache2/proxy"
|
|
CacheDirLevels 5
|
|
CacheDirLength 2
|
|
# SSL support
|
|
SSLProxyEngine on
|
|
# Prevent thundering herds.
|
|
CacheLock on
|
|
CacheLockPath "/tmp/mod_cache-lock"
|
|
CacheLockMaxAge 5
|
|
# 5GiB
|
|
CacheMaxFileSize 5368709120
|
|
CacheStoreExpired On
|
|
|
|
# Added Aug 2017 in an attempt to avoid occasional 502 errors (around
|
|
# 0.05% of requests) of the type:
|
|
#
|
|
# End of file found: ... AH01102: error reading status line from remote server ...
|
|
#
|
|
# Per [1]:
|
|
#
|
|
# This avoids the "proxy: error reading status line from remote
|
|
# server" error message caused by the race condition that the backend
|
|
# server closed the pooled connection after the connection check by the
|
|
# proxy and before data sent by the proxy reached the backend.
|
|
#
|
|
# [1] https://httpd.apache.org/docs/2.4/mod/mod_proxy_http.html
|
|
SetEnv proxy-initial-not-pooled 1
|
|
|
|
# Per site caching reverse proxy rules
|
|
# Only cache specific backends, rely on afs cache otherwise.
|
|
|
|
# buildlogs.centos.org (302 redirects to buildlogs.cdn.centos.org)
|
|
CacheEnable disk "/buildlogs.centos"
|
|
ProxyPass "/buildlogs.centos/" "https://buildlogs.centos.org/" ttl=120 disablereuse=On retry=0
|
|
ProxyPassReverse "/buildlogs.centos/" "https://buildlogs.centos.org/"
|
|
|
|
# buildlogs.cdn.centos.org
|
|
CacheEnable disk "/buildlogs.cdn.centos"
|
|
ProxyPass "/buildlogs.cdn.centos/" "https://buildlogs.cdn.centos.org/" ttl=120 disablereuse=On retry=0
|
|
ProxyPassReverse "/buildlogs.cdn.centos/" "https://buildlogs.cdn.centos.org/"
|
|
|
|
# rdo
|
|
CacheEnable disk "/rdo"
|
|
ProxyPass "/rdo/" "https://trunk.rdoproject.org/" ttl=120 keepalive=On retry=0
|
|
ProxyPassReverse "/rdo/" "https://trunk.rdoproject.org/"
|
|
|
|
# tarballs
|
|
CacheEnable disk "/tarballs"
|
|
ProxyPass "/tarballs/" "https://tarballs.openstack.org/" ttl=120 keepalive=On retry=0
|
|
ProxyPassReverse "/tarballs/" "https://tarballs.openstack.org/"
|
|
|
|
# pypi
|
|
CacheEnable disk "/pypi"
|
|
ProxyPass "/pypi/" "https://pypi.org/" ttl=120 keepalive=On retry=0
|
|
ProxyPassReverse "/pypi/" "https://pypi.org/
|
|
|
|
# files.pythonhosted.org
|
|
CacheEnable disk "/pypifiles"
|
|
ProxyPass "/pypifiles/" "https://files.pythonhosted.org/" ttl=120 keepalive=On retry=0
|
|
ProxyPassReverse "/pypifiles/" "https://files.pythonhosted.org/"
|
|
|
|
# Rewrite the locations of the actual files
|
|
<Location /pypi>
|
|
SetOutputFilter INFLATE;SUBSTITUTE;DEFLATE
|
|
Substitute "s|https://files.pythonhosted.org/|/pypifiles/|ni"
|
|
</Location>
|
|
|
|
# images.linuxcontainers.org
|
|
CacheEnable disk "/images.linuxcontainers"
|
|
ProxyPass "/images.linuxcontainers/" "http://us.images.linuxcontainers.org/" ttl=120 keepalive=On retry=0
|
|
ProxyPassReverse "/images.linuxcontainers/" "http://us.images.linuxcontainers.org/"
|
|
|
|
# registry.npmjs.org
|
|
CacheEnable disk "/registry.npmjs"
|
|
ProxyPass "/registry.npmjs/" "https://registry.npmjs.org/" ttl=120 keepalive=On retry=0
|
|
ProxyPassReverse "/registry.npmjs/" "https://registry.npmjs.org/"
|
|
|
|
# api.rubygems.org
|
|
CacheEnable disk "/api.rubygems"
|
|
ProxyPass "/api.rubygems/" "https://api.rubygems.org/" ttl=120 keepalive=On retry=0
|
|
ProxyPassReverse "/api.rubygems/" "https://api.rubygems.org/"
|
|
|
|
# rubygems.org
|
|
CacheEnable disk "/rubygems"
|
|
ProxyPass "/rubygems/" "https://rubygems.org/" ttl=120 keepalive=On retry=0
|
|
ProxyPassReverse "/rubygems/" "https://rubygems.org/"
|
|
|
|
# opendaylight
|
|
CacheEnable disk "/opendaylight"
|
|
ProxyPass "/opendaylight/" "https://nexus.opendaylight.org/" ttl=120 keepalive=On retry=0
|
|
ProxyPassReverse "/opendaylight/" "https://nexus.opendaylight.org/"
|
|
|
|
# elastico
|
|
CacheEnable disk "/elastic"
|
|
ProxyPass "/elastic/" "https://packages.elastic.co/" ttl=120 keepalive=On retry=0
|
|
ProxyPassReverse "/elastic/" "https://packages.elastic.co/"
|
|
|
|
# grafana
|
|
CacheEnable disk "/grafana"
|
|
ProxyPass "/grafana" "https://packagecloud.io/grafana/" ttl=120 keepalive=On retry=0
|
|
ProxyPassReverse "/grafana/" "https://packagecloud.io/grafana/"
|
|
|
|
# OracleLinux
|
|
CacheEnable disk "/oraclelinux"
|
|
ProxyPass "/oraclelinux/" "http://yum.oracle.com/repo/OracleLinux/" ttl=120 keepalive=On retry=0
|
|
ProxyPassReverse "/oraclelinux/" "http://yum.oracle.com/repo/OracleLinux/"
|
|
|
|
# Percona
|
|
CacheEnable disk "/percona"
|
|
ProxyPass "/percona/" "https://repo.percona.com/" ttl=120 keepalive=On retry=0
|
|
ProxyPassReverse "/percona/" "https://repo.percona.com/"
|
|
|
|
# MariaDB
|
|
CacheEnable disk "/MariaDB"
|
|
ProxyPass "/MariaDB/" "https://downloads.mariadb.com/MariaDB/" ttl=120 keepalive=On retry=0
|
|
ProxyPassReverse "/MariaDB/" "https://downloads.mariadb.com/MariaDB/"
|
|
|
|
# Docker
|
|
CacheEnable disk "/docker"
|
|
ProxyPass "/docker/" "https://download.docker.com/linux/" ttl=120 keepalive=On retry=0
|
|
ProxyPassReverse "/docker/" "https://download.docker.com/linux/"
|
|
|
|
# Alpine
|
|
CacheEnable disk "/alpine"
|
|
ProxyPass "/alpine/" "http://dl-cdn.alpinelinux.org/alpine/" ttl=120 keepalive=On retry=0
|
|
ProxyPassReverse "/alpine/" "http://dl-cdn.alpinelinux.org/alpine/
|
|
|
|
</VirtualHost>
|
|
|
|
<VirtualHost <%= @vhost_name %>:8081>
|
|
ServerName <%= @srvname %>:8081
|
|
<% if @serveraliases.is_a? Array -%>
|
|
<% @serveraliases.each do |name| -%>
|
|
<%= " ServerAlias #{name}:8081\n" %>
|
|
<% end %>
|
|
<% elsif @serveraliases != nil -%>
|
|
<%= " ServerAlias #{@serveraliases}:8081" -%>
|
|
<% end %>
|
|
|
|
# Disable directory listing by default.
|
|
<Directory />
|
|
Order Deny,Allow
|
|
Deny from all
|
|
Options None
|
|
AllowOverride None
|
|
</Directory>
|
|
|
|
ErrorLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/<%= @name %>_8081_error.log
|
|
LogLevel warn
|
|
CustomLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/<%= @name %>_8081_access.log combined-cache
|
|
ServerSignature Off
|
|
|
|
# Caching reverse proxy for things that don't make sense in AFS
|
|
#
|
|
# General cache rules
|
|
CacheRoot "/var/cache/apache2/proxy"
|
|
CacheDirLevels 5
|
|
CacheDirLength 2
|
|
# SSL support
|
|
SSLProxyEngine on
|
|
# Prevent thundering herds.
|
|
CacheLock on
|
|
CacheLockPath "/tmp/mod_cache-lock"
|
|
CacheLockMaxAge 5
|
|
# 5GiB
|
|
CacheMaxFileSize 5368709120
|
|
# Ignore expire headers as the urls use sha256 hashes.
|
|
CacheIgnoreQueryString On
|
|
# NOTE(pabelanger): In the case of docker, if neither an expiry date nor
|
|
# last-modified date are provided default expire to 1 day. This is up from
|
|
# 1 hour.
|
|
CacheDefaultExpire 86400
|
|
CacheStoreExpired On
|
|
|
|
# registry-1.docker.io
|
|
CacheEnable disk "/registry-1.docker"
|
|
ProxyPass "/registry-1.docker/" "https://registry-1.docker.io/" ttl=120 keepalive=On retry=0
|
|
ProxyPassReverse "/registry-1.docker/" "https://registry-1.docker.io/"
|
|
|
|
# dseasb33srnrn.cloudfront.net
|
|
CacheEnable disk "/cloudfront"
|
|
ProxyPass "/cloudfront/" "https://dseasb33srnrn.cloudfront.net/" ttl=120 keepalive=On retry=0
|
|
ProxyPassReverse "/cloudfront/" "https://dseasb33srnrn.cloudfront.net/"
|
|
|
|
# production.cloudflare.docker.com
|
|
CacheEnable disk "/cloudflare"
|
|
ProxyPass "/cloudflare/" "https://production.cloudflare.docker.com/" ttl=120 keepalive=On retry=0
|
|
ProxyPassReverse "/cloudflare/" "https://production.cloudflare.docker.com/"
|
|
|
|
</VirtualHost>
|
|
|
|
<VirtualHost <%= @vhost_name %>:8082>
|
|
ServerName <%= @srvname %>:8082
|
|
<% if @serveraliases.is_a? Array -%>
|
|
<% @serveraliases.each do |name| -%>
|
|
<%= " ServerAlias #{name}:8082\n" %>
|
|
<% end %>
|
|
<% elsif @serveraliases != nil -%>
|
|
<%= " ServerAlias #{@serveraliases}:8082" -%>
|
|
<% end %>
|
|
|
|
# Disable directory listing by default.
|
|
<Directory />
|
|
Order Deny,Allow
|
|
Deny from all
|
|
Options None
|
|
AllowOverride None
|
|
</Directory>
|
|
|
|
ErrorLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/<%= @name %>_8082_error.log
|
|
LogLevel warn
|
|
CustomLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/<%= @name %>_8082_access.log combined-cache
|
|
ServerSignature Off
|
|
|
|
# Caching reverse proxy for things that don't make sense in AFS
|
|
#
|
|
# General cache rules
|
|
CacheRoot "/var/cache/apache2/proxy"
|
|
CacheDirLevels 5
|
|
CacheDirLength 2
|
|
# SSL support
|
|
SSLProxyEngine on
|
|
# Prevent thundering herds.
|
|
CacheLock on
|
|
CacheLockPath "/tmp/mod_cache-lock"
|
|
CacheLockMaxAge 5
|
|
# 5GiB
|
|
CacheMaxFileSize 5368709120
|
|
# Ignore expire headers as the urls use sha256 hashes.
|
|
CacheIgnoreQueryString On
|
|
# NOTE(pabelanger): In the case of docker, if neither an expiry date nor
|
|
# last-modified date are provided default expire to 1 day. This is up from
|
|
# 1 hour.
|
|
CacheDefaultExpire 86400
|
|
CacheStoreExpired On
|
|
|
|
# NOTE(SamYaple): Docker has historically not allowed to have anything in
|
|
# the URI, but it wasn't enforced until they started validating the config
|
|
# in docker-ce 17.10. Prior to that, it was working fine.
|
|
CacheEnable disk "/"
|
|
ProxyPass "/" "https://registry-1.docker.io/" ttl=120 keepalive=On retry=0
|
|
ProxyPassReverse "/" "https://registry-1.docker.io/"
|
|
|
|
# dseasb33srnrn.cloudfront.net
|
|
CacheEnable disk "/cloudfront"
|
|
ProxyPass "/cloudfront/" "https://dseasb33srnrn.cloudfront.net/" ttl=120 keepalive=On retry=0
|
|
ProxyPassReverse "/cloudfront/" "https://dseasb33srnrn.cloudfront.net/"
|
|
|
|
# production.cloudflare.docker.com
|
|
CacheEnable disk "/cloudflare"
|
|
ProxyPass "/cloudflare/" "https://production.cloudflare.docker.com/" ttl=120 keepalive=On retry=0
|
|
ProxyPassReverse "/cloudflare/" "https://production.cloudflare.docker.com/"
|
|
|
|
</VirtualHost>
|