From 79ff2afb87e0758efe9e8beab376159c31395c19 Mon Sep 17 00:00:00 2001 From: Clark Boylan Date: Mon, 18 May 2020 11:44:52 -0700 Subject: [PATCH] Enable ssl on all mirror vhosts Previously we had enabled SSL on our main vhost for the mirrors. Do similar for all of the proxy cache vhosts for docker and other external resources. As part of this change we improve the testing to ensure that the new vhosts are working as expected. One testing specific change to note is the testinfra node names did not match our existing system-config-run job nodenames. This has been corrected. Additionally RHRegistryMirror and QuayMirror may not be working and fixing those is left as a followup. Change-Id: I9dbbd4080c3a2cce4acc39d63244f7a645503553 --- .../roles/mirror/templates/mirror.vhost.j2 | 145 +++++++++++++----- testinfra/test_mirror.py | 41 ++++- zuul.d/system-config-run.yaml | 6 + 3 files changed, 150 insertions(+), 42 deletions(-) diff --git a/playbooks/roles/mirror/templates/mirror.vhost.j2 b/playbooks/roles/mirror/templates/mirror.vhost.j2 index 8ed3771f86..87ed9cdd23 100644 --- a/playbooks/roles/mirror/templates/mirror.vhost.j2 +++ b/playbooks/roles/mirror/templates/mirror.vhost.j2 @@ -4,15 +4,23 @@ NameVirtualHost *:443 # Dedicated port for proxy caching, as not to affect afs mirrors. Listen 8080 NameVirtualHost *:8080 +Listen 4443 +NameVirtualHost *:4443 Listen 8081 NameVirtualHost *:8081 +Listen 4444 +NameVirtualHost *:4444 Listen 8082 NameVirtualHost *:8082 +Listen 4445 +NameVirtualHost *:4445 Listen 8083 NameVirtualHost *:8083 +Listen 4446 +NameVirtualHost *:4446 {% raw %} LogFormat "%h %l %u [%{%F %T}t.%{msec_frac}t] \"%r\" %>s %b %{cache-status}e \"%{Referer}i\" \"%{User-agent}i\"" combined-cache @@ -116,6 +124,17 @@ ErrorLogFormat "[%{cu}t] [%-m:%l] [pid %P:tid %T] %7F: %E: [client\ %a] %M% , \ + + SSLEngine On + SSLCertificateFile /etc/letsencrypt-certs/{{ apache_server_name }}/{{ apache_server_name }}.cer + SSLCertificateKeyFile /etc/letsencrypt-certs/{{ apache_server_name }}/{{ apache_server_name }}.key + SSLCertificateChainFile /etc/letsencrypt-certs/{{ apache_server_name }}/ca.cer + SSLProtocol All -SSLv2 -SSLv3 + # Note: this list should ensure ciphers that provide forward secrecy + SSLCipherSuite ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:!AES256:!aNULL:!eNULL:!MD5:!DSS:!PSK:!SRP + SSLHonorCipherOrder on + + ServerName {{ apache_server_name }} ServerAlias {{ apache_server_alias }} @@ -127,21 +146,11 @@ ErrorLogFormat "[%{cu}t] [%-m:%l] [pid %P:tid %T] %7F: %E: [client\ %a] %M% , \ ServerName {{ apache_server_name }} ServerAlias {{ apache_server_alias }} - SSLCertificateFile /etc/letsencrypt-certs/{{ apache_server_name }}/{{ apache_server_name }}.cer - SSLCertificateKeyFile /etc/letsencrypt-certs/{{ apache_server_name }}/{{ apache_server_name }}.key - SSLCertificateChainFile /etc/letsencrypt-certs/{{ apache_server_name }}/ca.cer - SSLProtocol All -SSLv2 -SSLv3 - # Note: this list should ensure ciphers that provide forward secrecy - SSLCipherSuite ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:!AES256:!aNULL:!eNULL:!MD5:!DSS:!PSK:!SRP - SSLHonorCipherOrder on - + Use SSLConfig Use BaseMirror 443 - - ServerName {{ apache_server_name }}:8080 - ServerAlias {{ apache_server_alias }}:8080 - + # Disable directory listing by default. Order Deny,Allow @@ -150,9 +159,9 @@ ErrorLogFormat "[%{cu}t] [%-m:%l] [pid %P:tid %T] %7F: %E: [client\ %a] %M% , \ AllowOverride None - ErrorLog /var/log/apache2/proxy_8080_error.log + ErrorLog /var/log/apache2/proxy_$port_error.log LogLevel warn - CustomLog /var/log/apache2/proxy_8080_access.log combined-cache + CustomLog /var/log/apache2/proxy_$port_access.log combined-cache ServerSignature Off # Let upstreams decide on encoded slash handling. @@ -294,14 +303,25 @@ ErrorLogFormat "[%{cu}t] [%-m:%l] [pid %P:tid %T] %7F: %E: [client\ %a] %M% , \ CacheEnable disk "/copr-lxc2" ProxyPass "/copr-lxc2/" "https://copr-be.cloud.fedoraproject.org/results/thm/lxc2.0/" ttl=120 keepalive=On retry=0 ProxyPassReverse "/copr-lxc2/" "https://copr-be.cloud.fedoraproject.org/results/thm/lxc2.0/" + + + ServerName {{ apache_server_name }}:8080 + ServerAlias {{ apache_server_alias }}:8080 + + Use ProxyMirror 8080 + + + + ServerName {{ apache_server_name }}:4443 + ServerAlias {{ apache_server_alias }}:4443 + + Use SSLConfig + Use ProxyMirror 4443 # Docker registry v1 proxy. - - ServerName {{ apache_server_name }}:8081 - ServerAlias {{ apache_server_alias }}:8081 - + # Disable directory listing by default. Order Deny,Allow @@ -310,9 +330,9 @@ ErrorLogFormat "[%{cu}t] [%-m:%l] [pid %P:tid %T] %7F: %E: [client\ %a] %M% , \ AllowOverride None - ErrorLog /var/log/apache2/proxy_8081_error.log + ErrorLog /var/log/apache2/proxy_$port_error.log LogLevel warn - CustomLog /var/log/apache2/proxy_8081_access.log combined-cache + CustomLog /var/log/apache2/proxy_$port_access.log combined-cache ServerSignature Off # Caching reverse proxy for things that don't make sense in AFS @@ -351,14 +371,25 @@ ErrorLogFormat "[%{cu}t] [%-m:%l] [pid %P:tid %T] %7F: %E: [client\ %a] %M% , \ CacheEnable disk "/cloudflare" ProxyPass "/cloudflare/" "https://production.cloudflare.docker.com/" ttl=120 keepalive=On retry=0 ProxyPassReverse "/cloudflare/" "https://production.cloudflare.docker.com/" + + + ServerName {{ apache_server_name }}:8081 + ServerAlias {{ apache_server_alias }}:8081 + + Use Dockerv1Mirror 8081 + + + + ServerName {{ apache_server_name }}:4444 + ServerAlias {{ apache_server_alias }}:4444 + + Use SSLConfig + Use Dockerv1Mirror 4444 # Docker registry v2 proxy. - - ServerName {{ apache_server_name }}:8082 - ServerAlias {{ apache_server_alias }}:8082 - + # Disable directory listing by default. Order Deny,Allow @@ -367,9 +398,9 @@ ErrorLogFormat "[%{cu}t] [%-m:%l] [pid %P:tid %T] %7F: %E: [client\ %a] %M% , \ AllowOverride None - ErrorLog /var/log/apache2/proxy_8082_error.log + ErrorLog /var/log/apache2/proxy_$port_error.log LogLevel warn - CustomLog /var/log/apache2/proxy_8082_access.log combined-cache + CustomLog /var/log/apache2/proxy_$port_access.log combined-cache ServerSignature Off # Caching reverse proxy for things that don't make sense in AFS @@ -409,13 +440,25 @@ ErrorLogFormat "[%{cu}t] [%-m:%l] [pid %P:tid %T] %7F: %E: [client\ %a] %M% , \ CacheEnable disk "/" ProxyPass "/" "https://registry-1.docker.io/" ttl=120 keepalive=On retry=0 ProxyPassReverse "/" "https://registry-1.docker.io/" + + + + ServerName {{ apache_server_name }}:8082 + ServerAlias {{ apache_server_alias }}:8082 + + Use Dockerv2Mirror 8082 + + + + ServerName {{ apache_server_name }}:4445 + ServerAlias {{ apache_server_alias }}:4445 + + Use SSLConfig + Use Dockerv2Mirror 4445 # Redhat registry proxy. - - ServerName {{ apache_server_name }}:8083 - ServerAlias {{ apache_server_alias }}:8083 - + # Disable directory listing by default. Order Deny,Allow @@ -424,9 +467,9 @@ ErrorLogFormat "[%{cu}t] [%-m:%l] [pid %P:tid %T] %7F: %E: [client\ %a] %M% , \ AllowOverride None - ErrorLog /var/log/apache2/proxy_8083_error.log + ErrorLog /var/log/apache2/proxy_$port_error.log LogLevel warn - CustomLog /var/log/apache2/proxy_8083_access.log combined-cache + CustomLog /var/log/apache2/proxy_$port_access.log combined-cache ServerSignature Off # Caching reverse proxy for things that don't make sense in AFS @@ -462,12 +505,25 @@ ErrorLogFormat "[%{cu}t] [%-m:%l] [pid %P:tid %T] %7F: %E: [client\ %a] %M% , \ CacheEnable disk "/" ProxyPass "/" "https://registry.access.redhat.com/" ttl=120 keepalive=On retry=0 ProxyPassReverse "/" "https://registry.access.redhat.com/" + + + + ServerName {{ apache_server_name }}:8083 + ServerAlias {{ apache_server_alias }}:8083 + + Use RHRegistryMirror 8083 + + + + ServerName {{ apache_server_name }}:4446 + ServerAlias {{ apache_server_alias }}:4446 + + Use SSLConfig + Use RHRegistryMirror 4446 # Quay registry proxy. - - ServerName {{ apache_server_name }}:8084 - ServerAlias {{ apache_server_alias }}:8084 + # Disable directory listing by default. @@ -477,9 +533,9 @@ ErrorLogFormat "[%{cu}t] [%-m:%l] [pid %P:tid %T] %7F: %E: [client\ %a] %M% , \ AllowOverride None - ErrorLog /var/log/apache2/proxy_8083_error.log + ErrorLog /var/log/apache2/proxy_$port_error.log LogLevel warn - CustomLog /var/log/apache2/proxy_8083_access.log combined-cache + CustomLog /var/log/apache2/proxy_$port_access.log combined-cache ServerSignature Off # Caching reverse proxy for things that don't make sense in AFS @@ -510,4 +566,19 @@ ErrorLogFormat "[%{cu}t] [%-m:%l] [pid %P:tid %T] %7F: %E: [client\ %a] %M% , \ CacheEnable disk "/" ProxyPass "/" "https://quay.io/" ttl=120 keepalive=On retry=0 ProxyPassReverse "/" "https://quay.io/" + + + + ServerName {{ apache_server_name }}:8084 + ServerAlias {{ apache_server_alias }}:8084 + + Use QuayRegistryMirror 8084 + + + + ServerName {{ apache_server_name }}:4447 + ServerAlias {{ apache_server_alias }}:4447 + + Use SSLConfig + Use QuayRegistryMirror 4447 diff --git a/testinfra/test_mirror.py b/testinfra/test_mirror.py index e111a647cc..f1e02127c6 100644 --- a/testinfra/test_mirror.py +++ b/testinfra/test_mirror.py @@ -13,21 +13,52 @@ # under the License. -testinfra_hosts = ['mirror01.region.provider.opendev.org', - 'mirror02.region.provider.opendev.org'] +testinfra_hosts = ['mirror01.openafs.provider.opendev.org', + 'mirror02.openafs.provider.opendev.org'] def test_apache(host): apache = host.service('apache2') assert apache.is_running -def test_mirror_indexes(host): +def test_base_mirror(host): + # BaseMirror cmd = host.run("wget --no-check-certificate -qO- https://localhost/") assert '' in cmd.stdout cmd = host.run("wget -qO- http://localhost/") assert '' in cmd.stdout +def test_proxy_mirror(host): + # ProxyMirror + cmd = host.run("wget --no-check-certificate -qO- " + "https://localhost:4443/pypi/simple/setuptools") + assert 'setuptools' in cmd.stdout + + cmd = host.run("wget -qO- http://localhost:8080/pypi/simple/setuptools") + assert 'setuptools' in cmd.stdout + +def test_dockerv1_mirror(host): + # Dockerv1Mirror + cmd = host.run("wget --no-check-certificate -O- " + "https://localhost:4444/registry-1.docker") + # TODO assert that this proxy cache is working more properly + assert '403 Forbidden' in cmd.stderr + + cmd = host.run("wget -O- http://localhost:8081/registry-1.docker") + # TODO assert that this proxy cache is working more properly + assert '403 Forbidden' in cmd.stderr + +def test_dockerv2_mirror(host): + # Dockerv2Mirror + cmd = host.run("wget --no-check-certificate -O- " + "https://localhost:4445/v2/") + assert '401 Unauthorized' in cmd.stderr + + cmd = host.run("wget -O- http://localhost:8082/v2/") + assert '401 Unauthorized' in cmd.stderr + +# TODO test RHRegistryMirror and QuayMirror + # NOTE(ianw): further testing idea for anyone interested; get the -# actual IP address of the mirror node and connect via that, and then -# also poke at the other proxy ports +# actual IP address of the mirror node and connect via that diff --git a/zuul.d/system-config-run.yaml b/zuul.d/system-config-run.yaml index 9dcc1e763a..acfb0f11e9 100644 --- a/zuul.d/system-config-run.yaml +++ b/zuul.d/system-config-run.yaml @@ -347,6 +347,12 @@ host_copy_output: '/var/log/apache2/': logs '/var/log/acme.sh': logs + '/etc/apache2/sites-available/mirror.conf': logs + mirror02.openafs.provider.opendev.org: + host_copy_output: + '/var/log/apache2/': logs + '/var/log/acme.sh': logs + '/etc/apache2/sites-available/mirror.conf': logs files: - playbooks/install-ansible.yaml - roles/