diff --git a/playbooks/roles/mirror/templates/mirror.vhost.j2 b/playbooks/roles/mirror/templates/mirror.vhost.j2 index 8ed3771f86..87ed9cdd23 100644 --- a/playbooks/roles/mirror/templates/mirror.vhost.j2 +++ b/playbooks/roles/mirror/templates/mirror.vhost.j2 @@ -4,15 +4,23 @@ NameVirtualHost *:443 # Dedicated port for proxy caching, as not to affect afs mirrors. Listen 8080 NameVirtualHost *:8080 +Listen 4443 +NameVirtualHost *:4443 Listen 8081 NameVirtualHost *:8081 +Listen 4444 +NameVirtualHost *:4444 Listen 8082 NameVirtualHost *:8082 +Listen 4445 +NameVirtualHost *:4445 Listen 8083 NameVirtualHost *:8083 +Listen 4446 +NameVirtualHost *:4446 {% raw %} LogFormat "%h %l %u [%{%F %T}t.%{msec_frac}t] \"%r\" %>s %b %{cache-status}e \"%{Referer}i\" \"%{User-agent}i\"" combined-cache @@ -116,6 +124,17 @@ ErrorLogFormat "[%{cu}t] [%-m:%l] [pid %P:tid %T] %7F: %E: [client\ %a] %M% , \ + + SSLEngine On + SSLCertificateFile /etc/letsencrypt-certs/{{ apache_server_name }}/{{ apache_server_name }}.cer + SSLCertificateKeyFile /etc/letsencrypt-certs/{{ apache_server_name }}/{{ apache_server_name }}.key + SSLCertificateChainFile /etc/letsencrypt-certs/{{ apache_server_name }}/ca.cer + SSLProtocol All -SSLv2 -SSLv3 + # Note: this list should ensure ciphers that provide forward secrecy + SSLCipherSuite ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:!AES256:!aNULL:!eNULL:!MD5:!DSS:!PSK:!SRP + SSLHonorCipherOrder on + + ServerName {{ apache_server_name }} ServerAlias {{ apache_server_alias }} @@ -127,21 +146,11 @@ ErrorLogFormat "[%{cu}t] [%-m:%l] [pid %P:tid %T] %7F: %E: [client\ %a] %M% , \ ServerName {{ apache_server_name }} ServerAlias {{ apache_server_alias }} - SSLCertificateFile /etc/letsencrypt-certs/{{ apache_server_name }}/{{ apache_server_name }}.cer - SSLCertificateKeyFile /etc/letsencrypt-certs/{{ apache_server_name }}/{{ apache_server_name }}.key - SSLCertificateChainFile /etc/letsencrypt-certs/{{ apache_server_name }}/ca.cer - SSLProtocol All -SSLv2 -SSLv3 - # Note: this list should ensure ciphers that provide forward secrecy - SSLCipherSuite ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:!AES256:!aNULL:!eNULL:!MD5:!DSS:!PSK:!SRP - SSLHonorCipherOrder on - + Use SSLConfig Use BaseMirror 443 - - ServerName {{ apache_server_name }}:8080 - ServerAlias {{ apache_server_alias }}:8080 - + # Disable directory listing by default. Order Deny,Allow @@ -150,9 +159,9 @@ ErrorLogFormat "[%{cu}t] [%-m:%l] [pid %P:tid %T] %7F: %E: [client\ %a] %M% , \ AllowOverride None - ErrorLog /var/log/apache2/proxy_8080_error.log + ErrorLog /var/log/apache2/proxy_$port_error.log LogLevel warn - CustomLog /var/log/apache2/proxy_8080_access.log combined-cache + CustomLog /var/log/apache2/proxy_$port_access.log combined-cache ServerSignature Off # Let upstreams decide on encoded slash handling. @@ -294,14 +303,25 @@ ErrorLogFormat "[%{cu}t] [%-m:%l] [pid %P:tid %T] %7F: %E: [client\ %a] %M% , \ CacheEnable disk "/copr-lxc2" ProxyPass "/copr-lxc2/" "https://copr-be.cloud.fedoraproject.org/results/thm/lxc2.0/" ttl=120 keepalive=On retry=0 ProxyPassReverse "/copr-lxc2/" "https://copr-be.cloud.fedoraproject.org/results/thm/lxc2.0/" + + + ServerName {{ apache_server_name }}:8080 + ServerAlias {{ apache_server_alias }}:8080 + + Use ProxyMirror 8080 + + + + ServerName {{ apache_server_name }}:4443 + ServerAlias {{ apache_server_alias }}:4443 + + Use SSLConfig + Use ProxyMirror 4443 # Docker registry v1 proxy. - - ServerName {{ apache_server_name }}:8081 - ServerAlias {{ apache_server_alias }}:8081 - + # Disable directory listing by default. Order Deny,Allow @@ -310,9 +330,9 @@ ErrorLogFormat "[%{cu}t] [%-m:%l] [pid %P:tid %T] %7F: %E: [client\ %a] %M% , \ AllowOverride None - ErrorLog /var/log/apache2/proxy_8081_error.log + ErrorLog /var/log/apache2/proxy_$port_error.log LogLevel warn - CustomLog /var/log/apache2/proxy_8081_access.log combined-cache + CustomLog /var/log/apache2/proxy_$port_access.log combined-cache ServerSignature Off # Caching reverse proxy for things that don't make sense in AFS @@ -351,14 +371,25 @@ ErrorLogFormat "[%{cu}t] [%-m:%l] [pid %P:tid %T] %7F: %E: [client\ %a] %M% , \ CacheEnable disk "/cloudflare" ProxyPass "/cloudflare/" "https://production.cloudflare.docker.com/" ttl=120 keepalive=On retry=0 ProxyPassReverse "/cloudflare/" "https://production.cloudflare.docker.com/" + + + ServerName {{ apache_server_name }}:8081 + ServerAlias {{ apache_server_alias }}:8081 + + Use Dockerv1Mirror 8081 + + + + ServerName {{ apache_server_name }}:4444 + ServerAlias {{ apache_server_alias }}:4444 + + Use SSLConfig + Use Dockerv1Mirror 4444 # Docker registry v2 proxy. - - ServerName {{ apache_server_name }}:8082 - ServerAlias {{ apache_server_alias }}:8082 - + # Disable directory listing by default. Order Deny,Allow @@ -367,9 +398,9 @@ ErrorLogFormat "[%{cu}t] [%-m:%l] [pid %P:tid %T] %7F: %E: [client\ %a] %M% , \ AllowOverride None - ErrorLog /var/log/apache2/proxy_8082_error.log + ErrorLog /var/log/apache2/proxy_$port_error.log LogLevel warn - CustomLog /var/log/apache2/proxy_8082_access.log combined-cache + CustomLog /var/log/apache2/proxy_$port_access.log combined-cache ServerSignature Off # Caching reverse proxy for things that don't make sense in AFS @@ -409,13 +440,25 @@ ErrorLogFormat "[%{cu}t] [%-m:%l] [pid %P:tid %T] %7F: %E: [client\ %a] %M% , \ CacheEnable disk "/" ProxyPass "/" "https://registry-1.docker.io/" ttl=120 keepalive=On retry=0 ProxyPassReverse "/" "https://registry-1.docker.io/" + + + + ServerName {{ apache_server_name }}:8082 + ServerAlias {{ apache_server_alias }}:8082 + + Use Dockerv2Mirror 8082 + + + + ServerName {{ apache_server_name }}:4445 + ServerAlias {{ apache_server_alias }}:4445 + + Use SSLConfig + Use Dockerv2Mirror 4445 # Redhat registry proxy. - - ServerName {{ apache_server_name }}:8083 - ServerAlias {{ apache_server_alias }}:8083 - + # Disable directory listing by default. Order Deny,Allow @@ -424,9 +467,9 @@ ErrorLogFormat "[%{cu}t] [%-m:%l] [pid %P:tid %T] %7F: %E: [client\ %a] %M% , \ AllowOverride None - ErrorLog /var/log/apache2/proxy_8083_error.log + ErrorLog /var/log/apache2/proxy_$port_error.log LogLevel warn - CustomLog /var/log/apache2/proxy_8083_access.log combined-cache + CustomLog /var/log/apache2/proxy_$port_access.log combined-cache ServerSignature Off # Caching reverse proxy for things that don't make sense in AFS @@ -462,12 +505,25 @@ ErrorLogFormat "[%{cu}t] [%-m:%l] [pid %P:tid %T] %7F: %E: [client\ %a] %M% , \ CacheEnable disk "/" ProxyPass "/" "https://registry.access.redhat.com/" ttl=120 keepalive=On retry=0 ProxyPassReverse "/" "https://registry.access.redhat.com/" + + + + ServerName {{ apache_server_name }}:8083 + ServerAlias {{ apache_server_alias }}:8083 + + Use RHRegistryMirror 8083 + + + + ServerName {{ apache_server_name }}:4446 + ServerAlias {{ apache_server_alias }}:4446 + + Use SSLConfig + Use RHRegistryMirror 4446 # Quay registry proxy. - - ServerName {{ apache_server_name }}:8084 - ServerAlias {{ apache_server_alias }}:8084 + # Disable directory listing by default. @@ -477,9 +533,9 @@ ErrorLogFormat "[%{cu}t] [%-m:%l] [pid %P:tid %T] %7F: %E: [client\ %a] %M% , \ AllowOverride None - ErrorLog /var/log/apache2/proxy_8083_error.log + ErrorLog /var/log/apache2/proxy_$port_error.log LogLevel warn - CustomLog /var/log/apache2/proxy_8083_access.log combined-cache + CustomLog /var/log/apache2/proxy_$port_access.log combined-cache ServerSignature Off # Caching reverse proxy for things that don't make sense in AFS @@ -510,4 +566,19 @@ ErrorLogFormat "[%{cu}t] [%-m:%l] [pid %P:tid %T] %7F: %E: [client\ %a] %M% , \ CacheEnable disk "/" ProxyPass "/" "https://quay.io/" ttl=120 keepalive=On retry=0 ProxyPassReverse "/" "https://quay.io/" + + + + ServerName {{ apache_server_name }}:8084 + ServerAlias {{ apache_server_alias }}:8084 + + Use QuayRegistryMirror 8084 + + + + ServerName {{ apache_server_name }}:4447 + ServerAlias {{ apache_server_alias }}:4447 + + Use SSLConfig + Use QuayRegistryMirror 4447 diff --git a/testinfra/test_mirror.py b/testinfra/test_mirror.py index e111a647cc..f1e02127c6 100644 --- a/testinfra/test_mirror.py +++ b/testinfra/test_mirror.py @@ -13,21 +13,52 @@ # under the License. -testinfra_hosts = ['mirror01.region.provider.opendev.org', - 'mirror02.region.provider.opendev.org'] +testinfra_hosts = ['mirror01.openafs.provider.opendev.org', + 'mirror02.openafs.provider.opendev.org'] def test_apache(host): apache = host.service('apache2') assert apache.is_running -def test_mirror_indexes(host): +def test_base_mirror(host): + # BaseMirror cmd = host.run("wget --no-check-certificate -qO- https://localhost/") assert '' in cmd.stdout cmd = host.run("wget -qO- http://localhost/") assert '' in cmd.stdout +def test_proxy_mirror(host): + # ProxyMirror + cmd = host.run("wget --no-check-certificate -qO- " + "https://localhost:4443/pypi/simple/setuptools") + assert 'setuptools' in cmd.stdout + + cmd = host.run("wget -qO- http://localhost:8080/pypi/simple/setuptools") + assert 'setuptools' in cmd.stdout + +def test_dockerv1_mirror(host): + # Dockerv1Mirror + cmd = host.run("wget --no-check-certificate -O- " + "https://localhost:4444/registry-1.docker") + # TODO assert that this proxy cache is working more properly + assert '403 Forbidden' in cmd.stderr + + cmd = host.run("wget -O- http://localhost:8081/registry-1.docker") + # TODO assert that this proxy cache is working more properly + assert '403 Forbidden' in cmd.stderr + +def test_dockerv2_mirror(host): + # Dockerv2Mirror + cmd = host.run("wget --no-check-certificate -O- " + "https://localhost:4445/v2/") + assert '401 Unauthorized' in cmd.stderr + + cmd = host.run("wget -O- http://localhost:8082/v2/") + assert '401 Unauthorized' in cmd.stderr + +# TODO test RHRegistryMirror and QuayMirror + # NOTE(ianw): further testing idea for anyone interested; get the -# actual IP address of the mirror node and connect via that, and then -# also poke at the other proxy ports +# actual IP address of the mirror node and connect via that diff --git a/zuul.d/system-config-run.yaml b/zuul.d/system-config-run.yaml index 35715a06e9..3b383428f9 100644 --- a/zuul.d/system-config-run.yaml +++ b/zuul.d/system-config-run.yaml @@ -347,6 +347,12 @@ host_copy_output: '/var/log/apache2/': logs '/var/log/acme.sh': logs + '/etc/apache2/sites-available/mirror.conf': logs + mirror02.openafs.provider.opendev.org: + host_copy_output: + '/var/log/apache2/': logs + '/var/log/acme.sh': logs + '/etc/apache2/sites-available/mirror.conf': logs files: - playbooks/install-ansible.yaml - roles/