130c0a47fc
Newer pip sets cache-control: max-age=0 on requests for pypi indexes. This tells the proxy cache not to serve the index content from the cache. Unfortauntely this then means we fail to get this data which could be cached if we can't talk to the backend for some reason. By default pypi seems to set a 600 second max-age on these indexes which should be far better than not caching at all. So set the config to have apache ignore client side max-age and other cache disabling directives. Change-Id: I2da7006dcd8a8f4212f5e766ef35ba7b98177ed0
411 lines
15 KiB
Plaintext
411 lines
15 KiB
Plaintext
# ************************************
|
|
# Managed by Puppet
|
|
# ************************************
|
|
|
|
NameVirtualHost <%= @vhost_name %>:<%= @port %>
|
|
|
|
# Dedicated port for proxy caching, as not to affect afs mirrors.
|
|
Listen 8080
|
|
NameVirtualHost <%= @vhost_name %>:8080
|
|
|
|
Listen 8081
|
|
NameVirtualHost <%= @vhost_name %>:8081
|
|
|
|
Listen 8082
|
|
NameVirtualHost <%= @vhost_name %>:8082
|
|
|
|
|
|
LogFormat "%h %l %u %t \"%r\" %>s %b %{cache-status}e \"%{Referer}i\" \"%{User-agent}i\"" combined-cache
|
|
|
|
<VirtualHost <%= @vhost_name %>:<%= @port %>>
|
|
ServerName <%= @srvname %>
|
|
<% if @serveraliases.is_a? Array -%>
|
|
<% @serveraliases.each do |name| -%>
|
|
<%= " ServerAlias #{name}\n" %>
|
|
<% end %>
|
|
<% elsif @serveraliases != nil -%>
|
|
<%= " ServerAlias #{@serveraliases}" -%>
|
|
<% end %>
|
|
DocumentRoot <%= @docroot %>
|
|
<Directory <%= @docroot %>>
|
|
Options <%= @options %>
|
|
AllowOverride None
|
|
Order allow,deny
|
|
allow from all
|
|
Satisfy any
|
|
<IfVersion >= 2.4>
|
|
Require all granted
|
|
</IfVersion>
|
|
</Directory>
|
|
|
|
# Caching reverse proxy for things that don't make sense in AFS
|
|
#
|
|
# General cache rules
|
|
CacheRoot "/var/cache/apache2/proxy"
|
|
CacheDirLevels 5
|
|
CacheDirLength 2
|
|
# SSL support
|
|
SSLProxyEngine on
|
|
# Prevent thundering herds.
|
|
CacheLock on
|
|
CacheLockPath "/tmp/mod_cache-lock"
|
|
CacheLockMaxAge 5
|
|
# 5GiB
|
|
CacheMaxFileSize 5368709120
|
|
CacheStoreExpired On
|
|
# Pip sets Cache-Control: max-age=0 on requests for pypi index pages.
|
|
# This means we don't use the cache for those requests. This setting
|
|
# should force the proxy to ignore cache-control on the request side
|
|
# but we should still cache things based on the cache-control responses
|
|
# from the backed servers.
|
|
CacheIgnoreCacheControl On
|
|
|
|
# Added Aug 2017 in an attempt to avoid occasional 502 errors (around
|
|
# 0.05% of requests) of the type:
|
|
#
|
|
# End of file found: ... AH01102: error reading status line from remote server ...
|
|
#
|
|
# Per [1]:
|
|
#
|
|
# This avoids the "proxy: error reading status line from remote
|
|
# server" error message caused by the race condition that the backend
|
|
# server closed the pooled connection after the connection check by the
|
|
# proxy and before data sent by the proxy reached the backend.
|
|
#
|
|
# [1] https://httpd.apache.org/docs/2.4/mod/mod_proxy_http.html
|
|
SetEnv proxy-initial-not-pooled 1
|
|
|
|
RewriteEngine On
|
|
# pypi
|
|
CacheEnable disk "/pypi"
|
|
ProxyPass "/pypi/" "https://pypi.org/" ttl=120 keepalive=On retry=0
|
|
ProxyPassReverse "/pypi/" "https://pypi.org/
|
|
|
|
# files.pythonhosted.org
|
|
CacheEnable disk "/pypifiles"
|
|
ProxyPass "/pypifiles/" "https://files.pythonhosted.org/" ttl=120 keepalive=On retry=0
|
|
ProxyPassReverse "/pypifiles/" "https://files.pythonhosted.org/"
|
|
|
|
# Rewrite the locations of the actual files
|
|
<Location /pypi>
|
|
SetOutputFilter INFLATE;SUBSTITUTE;DEFLATE
|
|
Substitute "s|https://files.pythonhosted.org/|/pypifiles/|ni"
|
|
</Location>
|
|
|
|
# Wheel URL's are:
|
|
# /wheel/{distro}-{distro-version}/a/a/a-etc.whl
|
|
# /wheel/{distro}-{distro-version}/a/abcd/abcd-etc.whl
|
|
# /wheel/{distro}-{distro-version}/a/abcde/abcde-etc.whl
|
|
RewriteCond %{REQUEST_URI} ^/wheel/([^/]+)/([^/])([^/]*)
|
|
RewriteCond %{DOCUMENT_ROOT}/wheel/$1/$2/$2$3 -d
|
|
RewriteRule ^/wheel/([^/]+)/([^/])([^/]*)(/.*)?$ /wheel/$1/$2/$2$3$4 [L]
|
|
|
|
# Special cases for openstack.nose_plugin & backports.*
|
|
RewriteCond %{REQUEST_URI} ^/wheel/
|
|
RewriteRule ^(.*)/openstack-nose-plugin(.*)$ $1/openstack.nose_plugin$2
|
|
RewriteCond %{REQUEST_URI} ^/wheel/
|
|
RewriteRule ^(.*)/backports-(.*)$ $1/backports.$2
|
|
|
|
# Try again but replacing -'s with .'s
|
|
RewriteCond %{REQUEST_URI} ^/wheel/
|
|
RewriteCond %{DOCUMENT_ROOT}/%{REQUEST_URI} !-f
|
|
RewriteCond %{DOCUMENT_ROOT}/%{REQUEST_URI} !-d
|
|
RewriteRule (.*)-(.*) $1.$2 [N]
|
|
|
|
ErrorLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/<%= @name %>_error.log
|
|
LogLevel warn
|
|
CustomLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/<%= @name %>_access.log combined-cache
|
|
ServerSignature Off
|
|
</VirtualHost>
|
|
|
|
<VirtualHost <%= @vhost_name %>:8080>
|
|
ServerName <%= @srvname %>:8080
|
|
<% if @serveraliases.is_a? Array -%>
|
|
<% @serveraliases.each do |name| -%>
|
|
<%= " ServerAlias #{name}:8080\n" %>
|
|
<% end %>
|
|
<% elsif @serveraliases != nil -%>
|
|
<%= " ServerAlias #{@serveraliases}:8080" -%>
|
|
<% end %>
|
|
|
|
# Disable directory listing by default.
|
|
<Directory />
|
|
Order Deny,Allow
|
|
Deny from all
|
|
Options None
|
|
AllowOverride None
|
|
</Directory>
|
|
|
|
ErrorLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/<%= @name %>_8080_error.log
|
|
LogLevel warn
|
|
CustomLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/<%= @name %>_8080_access.log combined-cache
|
|
ServerSignature Off
|
|
|
|
# Caching reverse proxy for things that don't make sense in AFS
|
|
#
|
|
# General cache rules
|
|
CacheRoot "/var/cache/apache2/proxy"
|
|
CacheDirLevels 5
|
|
CacheDirLength 2
|
|
# SSL support
|
|
SSLProxyEngine on
|
|
# Prevent thundering herds.
|
|
CacheLock on
|
|
CacheLockPath "/tmp/mod_cache-lock"
|
|
CacheLockMaxAge 5
|
|
# 5GiB
|
|
CacheMaxFileSize 5368709120
|
|
CacheStoreExpired On
|
|
|
|
# Added Aug 2017 in an attempt to avoid occasional 502 errors (around
|
|
# 0.05% of requests) of the type:
|
|
#
|
|
# End of file found: ... AH01102: error reading status line from remote server ...
|
|
#
|
|
# Per [1]:
|
|
#
|
|
# This avoids the "proxy: error reading status line from remote
|
|
# server" error message caused by the race condition that the backend
|
|
# server closed the pooled connection after the connection check by the
|
|
# proxy and before data sent by the proxy reached the backend.
|
|
#
|
|
# [1] https://httpd.apache.org/docs/2.4/mod/mod_proxy_http.html
|
|
SetEnv proxy-initial-not-pooled 1
|
|
|
|
# Per site caching reverse proxy rules
|
|
# Only cache specific backends, rely on afs cache otherwise.
|
|
|
|
# buildlogs.centos.org (302 redirects to buildlogs.cdn.centos.org)
|
|
CacheEnable disk "/buildlogs.centos"
|
|
ProxyPass "/buildlogs.centos/" "https://buildlogs.centos.org/" ttl=120 disablereuse=On retry=0
|
|
ProxyPassReverse "/buildlogs.centos/" "https://buildlogs.centos.org/"
|
|
|
|
# buildlogs.cdn.centos.org
|
|
CacheEnable disk "/buildlogs.cdn.centos"
|
|
ProxyPass "/buildlogs.cdn.centos/" "https://buildlogs.cdn.centos.org/" ttl=120 disablereuse=On retry=0
|
|
ProxyPassReverse "/buildlogs.cdn.centos/" "https://buildlogs.cdn.centos.org/"
|
|
|
|
# rdo
|
|
CacheEnable disk "/rdo"
|
|
ProxyPass "/rdo/" "https://trunk.rdoproject.org/" ttl=120 keepalive=On retry=0
|
|
ProxyPassReverse "/rdo/" "https://trunk.rdoproject.org/"
|
|
|
|
# cbs.centos.org
|
|
CacheEnable disk "/cbs.centos"
|
|
ProxyPass "/cbs.centos/" "https://cbs.centos.org/repos/" ttl=120 keepalive=On retry=0
|
|
ProxyPassReverse "/cbs.centos/" "https://cbs.centos.org/repos/"
|
|
|
|
# tarballs
|
|
CacheEnable disk "/tarballs"
|
|
ProxyPass "/tarballs/" "https://tarballs.openstack.org/" ttl=120 keepalive=On retry=0
|
|
ProxyPassReverse "/tarballs/" "https://tarballs.openstack.org/"
|
|
|
|
# pypi
|
|
CacheEnable disk "/pypi"
|
|
ProxyPass "/pypi/" "https://pypi.org/" ttl=120 keepalive=On retry=0
|
|
ProxyPassReverse "/pypi/" "https://pypi.org/
|
|
|
|
# files.pythonhosted.org
|
|
CacheEnable disk "/pypifiles"
|
|
ProxyPass "/pypifiles/" "https://files.pythonhosted.org/" ttl=120 keepalive=On retry=0
|
|
ProxyPassReverse "/pypifiles/" "https://files.pythonhosted.org/"
|
|
|
|
# Rewrite the locations of the actual files
|
|
<Location /pypi>
|
|
SetOutputFilter INFLATE;SUBSTITUTE;DEFLATE
|
|
Substitute "s|https://files.pythonhosted.org/|/pypifiles/|ni"
|
|
</Location>
|
|
|
|
# images.linuxcontainers.org
|
|
CacheEnable disk "/images.linuxcontainers"
|
|
ProxyPass "/images.linuxcontainers/" "http://us.images.linuxcontainers.org/" ttl=120 keepalive=On retry=0
|
|
ProxyPassReverse "/images.linuxcontainers/" "http://us.images.linuxcontainers.org/"
|
|
|
|
# registry.npmjs.org
|
|
CacheEnable disk "/registry.npmjs"
|
|
ProxyPass "/registry.npmjs/" "https://registry.npmjs.org/" ttl=120 keepalive=On retry=0
|
|
ProxyPassReverse "/registry.npmjs/" "https://registry.npmjs.org/"
|
|
|
|
# api.rubygems.org
|
|
CacheEnable disk "/api.rubygems"
|
|
ProxyPass "/api.rubygems/" "https://api.rubygems.org/" ttl=120 keepalive=On retry=0
|
|
ProxyPassReverse "/api.rubygems/" "https://api.rubygems.org/"
|
|
|
|
# rubygems.org
|
|
CacheEnable disk "/rubygems"
|
|
ProxyPass "/rubygems/" "https://rubygems.org/" ttl=120 keepalive=On retry=0
|
|
ProxyPassReverse "/rubygems/" "https://rubygems.org/"
|
|
|
|
# opendaylight
|
|
CacheEnable disk "/opendaylight"
|
|
ProxyPass "/opendaylight/" "https://nexus.opendaylight.org/" ttl=120 keepalive=On retry=0
|
|
ProxyPassReverse "/opendaylight/" "https://nexus.opendaylight.org/"
|
|
|
|
# elastico
|
|
CacheEnable disk "/elastic"
|
|
ProxyPass "/elastic/" "https://packages.elastic.co/" ttl=120 keepalive=On retry=0
|
|
ProxyPassReverse "/elastic/" "https://packages.elastic.co/"
|
|
|
|
# grafana
|
|
CacheEnable disk "/grafana"
|
|
ProxyPass "/grafana" "https://packagecloud.io/grafana/" ttl=120 keepalive=On retry=0
|
|
ProxyPassReverse "/grafana/" "https://packagecloud.io/grafana/"
|
|
|
|
# OracleLinux
|
|
CacheEnable disk "/oraclelinux"
|
|
ProxyPass "/oraclelinux/" "http://yum.oracle.com/repo/OracleLinux/" ttl=120 keepalive=On retry=0
|
|
ProxyPassReverse "/oraclelinux/" "http://yum.oracle.com/repo/OracleLinux/"
|
|
|
|
# Percona
|
|
CacheEnable disk "/percona"
|
|
ProxyPass "/percona/" "https://repo.percona.com/" ttl=120 keepalive=On retry=0
|
|
ProxyPassReverse "/percona/" "https://repo.percona.com/"
|
|
|
|
# MariaDB
|
|
CacheEnable disk "/MariaDB"
|
|
ProxyPass "/MariaDB/" "https://downloads.mariadb.com/MariaDB/" ttl=120 keepalive=On retry=0
|
|
ProxyPassReverse "/MariaDB/" "https://downloads.mariadb.com/MariaDB/"
|
|
|
|
# Docker
|
|
CacheEnable disk "/docker"
|
|
ProxyPass "/docker/" "https://download.docker.com/linux/" ttl=120 keepalive=On retry=0
|
|
ProxyPassReverse "/docker/" "https://download.docker.com/linux/"
|
|
|
|
# Alpine
|
|
CacheEnable disk "/alpine"
|
|
ProxyPass "/alpine/" "http://dl-cdn.alpinelinux.org/alpine/" ttl=120 keepalive=On retry=0
|
|
ProxyPassReverse "/alpine/" "http://dl-cdn.alpinelinux.org/alpine/"
|
|
|
|
# LXC (copr)
|
|
CacheEnable disk "/copr-lxc2"
|
|
ProxyPass "/copr-lxc2/" "https://copr-be.cloud.fedoraproject.org/results/thm/lxc2.0/" ttl=120 keepalive=On retry=0
|
|
ProxyPassReverse "/copr-lxc2/" "https://copr-be.cloud.fedoraproject.org/results/thm/lxc2.0/"
|
|
|
|
</VirtualHost>
|
|
|
|
<VirtualHost <%= @vhost_name %>:8081>
|
|
ServerName <%= @srvname %>:8081
|
|
<% if @serveraliases.is_a? Array -%>
|
|
<% @serveraliases.each do |name| -%>
|
|
<%= " ServerAlias #{name}:8081\n" %>
|
|
<% end %>
|
|
<% elsif @serveraliases != nil -%>
|
|
<%= " ServerAlias #{@serveraliases}:8081" -%>
|
|
<% end %>
|
|
|
|
# Disable directory listing by default.
|
|
<Directory />
|
|
Order Deny,Allow
|
|
Deny from all
|
|
Options None
|
|
AllowOverride None
|
|
</Directory>
|
|
|
|
ErrorLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/<%= @name %>_8081_error.log
|
|
LogLevel warn
|
|
CustomLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/<%= @name %>_8081_access.log combined-cache
|
|
ServerSignature Off
|
|
|
|
# Caching reverse proxy for things that don't make sense in AFS
|
|
#
|
|
# General cache rules
|
|
CacheRoot "/var/cache/apache2/proxy"
|
|
CacheDirLevels 5
|
|
CacheDirLength 2
|
|
# SSL support
|
|
SSLProxyEngine on
|
|
# Prevent thundering herds.
|
|
CacheLock on
|
|
CacheLockPath "/tmp/mod_cache-lock"
|
|
CacheLockMaxAge 5
|
|
# 5GiB
|
|
CacheMaxFileSize 5368709120
|
|
# Ignore expire headers as the urls use sha256 hashes.
|
|
CacheIgnoreQueryString On
|
|
# NOTE(pabelanger): In the case of docker, if neither an expiry date nor
|
|
# last-modified date are provided default expire to 1 day. This is up from
|
|
# 1 hour.
|
|
CacheDefaultExpire 86400
|
|
CacheStoreExpired On
|
|
|
|
# registry-1.docker.io
|
|
CacheEnable disk "/registry-1.docker"
|
|
ProxyPass "/registry-1.docker/" "https://registry-1.docker.io/" ttl=120 keepalive=On retry=0
|
|
ProxyPassReverse "/registry-1.docker/" "https://registry-1.docker.io/"
|
|
|
|
# dseasb33srnrn.cloudfront.net
|
|
CacheEnable disk "/cloudfront"
|
|
ProxyPass "/cloudfront/" "https://dseasb33srnrn.cloudfront.net/" ttl=120 keepalive=On retry=0
|
|
ProxyPassReverse "/cloudfront/" "https://dseasb33srnrn.cloudfront.net/"
|
|
|
|
# production.cloudflare.docker.com
|
|
CacheEnable disk "/cloudflare"
|
|
ProxyPass "/cloudflare/" "https://production.cloudflare.docker.com/" ttl=120 keepalive=On retry=0
|
|
ProxyPassReverse "/cloudflare/" "https://production.cloudflare.docker.com/"
|
|
|
|
</VirtualHost>
|
|
|
|
<VirtualHost <%= @vhost_name %>:8082>
|
|
ServerName <%= @srvname %>:8082
|
|
<% if @serveraliases.is_a? Array -%>
|
|
<% @serveraliases.each do |name| -%>
|
|
<%= " ServerAlias #{name}:8082\n" %>
|
|
<% end %>
|
|
<% elsif @serveraliases != nil -%>
|
|
<%= " ServerAlias #{@serveraliases}:8082" -%>
|
|
<% end %>
|
|
|
|
# Disable directory listing by default.
|
|
<Directory />
|
|
Order Deny,Allow
|
|
Deny from all
|
|
Options None
|
|
AllowOverride None
|
|
</Directory>
|
|
|
|
ErrorLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/<%= @name %>_8082_error.log
|
|
LogLevel warn
|
|
CustomLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/<%= @name %>_8082_access.log combined-cache
|
|
ServerSignature Off
|
|
|
|
# Caching reverse proxy for things that don't make sense in AFS
|
|
#
|
|
# General cache rules
|
|
CacheRoot "/var/cache/apache2/proxy"
|
|
CacheDirLevels 5
|
|
CacheDirLength 2
|
|
# SSL support
|
|
SSLProxyEngine on
|
|
# Prevent thundering herds.
|
|
CacheLock on
|
|
CacheLockPath "/tmp/mod_cache-lock"
|
|
CacheLockMaxAge 5
|
|
# 5GiB
|
|
CacheMaxFileSize 5368709120
|
|
# Ignore expire headers as the urls use sha256 hashes.
|
|
CacheIgnoreQueryString On
|
|
# NOTE(pabelanger): In the case of docker, if neither an expiry date nor
|
|
# last-modified date are provided default expire to 1 day. This is up from
|
|
# 1 hour.
|
|
CacheDefaultExpire 86400
|
|
CacheStoreExpired On
|
|
|
|
# NOTE(SamYaple): Docker has historically not allowed to have anything in
|
|
# the URI, but it wasn't enforced until they started validating the config
|
|
# in docker-ce 17.10. Prior to that, it was working fine.
|
|
CacheEnable disk "/"
|
|
ProxyPass "/" "https://registry-1.docker.io/" ttl=120 keepalive=On retry=0
|
|
ProxyPassReverse "/" "https://registry-1.docker.io/"
|
|
|
|
# dseasb33srnrn.cloudfront.net
|
|
CacheEnable disk "/cloudfront"
|
|
ProxyPass "/cloudfront/" "https://dseasb33srnrn.cloudfront.net/" ttl=120 keepalive=On retry=0
|
|
ProxyPassReverse "/cloudfront/" "https://dseasb33srnrn.cloudfront.net/"
|
|
|
|
# production.cloudflare.docker.com
|
|
CacheEnable disk "/cloudflare"
|
|
ProxyPass "/cloudflare/" "https://production.cloudflare.docker.com/" ttl=120 keepalive=On retry=0
|
|
ProxyPassReverse "/cloudflare/" "https://production.cloudflare.docker.com/"
|
|
|
|
</VirtualHost>
|