Merge "Correct (again) how ansible-galaxy proxy is configured"

playbooks/roles/mirror/tasks/main.yaml

@@ -54,6 +54,11 @@
     state: present
     name: ssl
 
+- name: Apache headers module
+  apache2_module:
+    state: present
+    name: headers
+
 - name: Apache webroot
   file:
     path: '{{ www_base }}'

playbooks/roles/mirror/templates/mirror.vhost.j2

@@ -567,8 +567,13 @@ ErrorLogFormat "[%{cu}t] [%-m:%l] [pid %P:tid %T] %7F: %E: [client\ %a] %M% , \
     # 5GiB
     CacheMaxFileSize 5368709120
     CacheStoreExpired On
+    CacheIgnoreQueryString On
+    CacheDefaultExpire 86400
+    CacheIgnoreCacheControl On
+    CacheStorePrivate On
 
     <Location "/">
+      CacheEnable disk
       ProxyPass "https://galaxy.ansible.com/" ttl=120 keepalive=On retry=0
       ProxyPassReverse "https://galaxy.ansible.com/"
       SetOutputFilter INFLATE;SUBSTITUTE;DEFLATE
@@ -577,15 +582,19 @@ ErrorLogFormat "[%{cu}t] [%-m:%l] [pid %P:tid %T] %7F: %E: [client\ %a] %M% , \
       # of the REQUEST_SCHEME. Note that mod_substitute can't use parameters...
       <If "-T %{HTTPS}">
         Substitute "s|https://galaxy.ansible.com/|https://{{ apache_server_name }}:$port/|ni"
-        Substitute "s|https://ansible-galaxy.s3.amazonaws.com/|https://{{ apache_server_name }}:$port/galaxy-s3/|ni"
       </If>
       <If "! -T %{HTTPS}">
         Substitute "s|https://galaxy.ansible.com/|http://{{ apache_server_name }}:$port/|ni"
-        Substitute "s|https://ansible-galaxy.s3.amazonaws.com/|http://{{ apache_server_name }}:$port/galaxy-s3/|ni"
       </If>
+      # Substitute doesn't edit headers - ansible-galaxy sets a Location header for the final link
+      # to S3 bucket and content. Let's override it in order to point to our local endpoint
+      Header edit Location "^https://ansible-galaxy.s3.amazonaws.com/" "/galaxy-s3/"
+    </Location>
+    <Location "/galaxy-s3/">
+      CacheEnable disk
+      ProxyPass "https://ansible-galaxy.s3.amazonaws.com/" ttl=120 keepalive=On retry=0
+      ProxyPassReverse "https://ansible-galaxy.s3.amazonaws.com/"
     </Location>
-    ProxyPass "/galaxy-s3/" "https://ansible-galaxy.s3.amazonaws.com/" ttl=120 keepalive=On retry=0
-    ProxyPassReverse "/galaxy-s3/" "https://ansible-galaxy.s3.amazonaws.com/"
 
     ErrorLog /var/log/apache2/proxy_$port_error.log
     LogLevel warn

testinfra/test_mirror.py

@@ -23,9 +23,9 @@ def test_apache(host):
     apache = host.service('apache2')
     assert apache.is_running
 
-def _run_cmd(host, port, scheme='https', url=''):
+def _run_cmd(host, port, scheme='https', url='', curl_opt=''):
     hostname = host.backend.get_hostname()
-    return f'curl --resolve {hostname}:127.0.0.1 {scheme}://{hostname}:{port}{url}'
+    return f'curl {curl_opt} --resolve {hostname}:127.0.0.1 {scheme}://{hostname}:{port}{url}'
 
 def test_base_mirror(host):
     # base mirror
@@ -87,3 +87,8 @@ def test_galaxy_mirror(host):
     answer = json.loads(cmd.stdout)
     download_uri = answer['download_url']
     assert download_uri.startswith('https://{}:4448/download/community-general'.format(hostname))
+    # Download a file and check we get an actual archive
+    download_uri = download_uri.replace('https://{}:4448'.format(hostname), '')
+    host.run(_run_cmd(host, 4448, url=download_uri, curl_opt='-sL --output /tmp/output.tar.gz'))
+    check_file = host.run('file /tmp/output.tar.gz')
+    assert 'gzip compressed data' in check_file.stdout