Merge "Create opendev mirrors"

.zuul.yaml

@@ -482,6 +482,35 @@
       - testinfra/test_adns.py
       - testinfra/test_ns.py
 
+- job:
+    name: system-config-run-mirror
+    parent: system-config-run
+    description: |
+      Run the playbook for a mirror node
+    nodeset:
+      nodes:
+        - name: bridge.openstack.org
+          label: ubuntu-bionic
+        - name: mirror01.region.provider.opendev.org
+          label: ubuntu-bionic
+    vars:
+      run_playbooks:
+        - playbooks/service-letsencrypt.yaml
+        - playbooks/service-mirror.yaml
+    files:
+      - .zuul.yaml
+      - roles/
+      - playbooks/roles/mirror/
+      - playbooks/roles/letsencrypt.*
+      - playbooks/service-letsencrypt.yaml
+      - playbooks/service-mirror.yaml
+      - testinfra/test_mirror.py
+
+    host-vars:
+      mirror.region.provider.opendev.org:
+        host_copy_output:
+          '/var/log/apache2/': logs
+
 - job:
     name: system-config-run-docker-registry
     parent: system-config-run
@@ -616,6 +645,7 @@
         - system-config-run-dns
         - system-config-run-eavesdrop
         - system-config-run-nodepool
+        - system-config-run-mirror
         - system-config-run-docker-registry
         - system-config-run-gitea:
             dependencies:

inventory/groups.yaml

@@ -53,6 +53,7 @@ groups:
     - opendev-k8s*.opendev.org
   letsencrypt:
     - graphite01.opendev.org
+    - mirror[0-9]*.opendev.org
   logstash:
     - logstash[0-9]*.open*.org
   logstash-worker:
@@ -61,7 +62,9 @@ groups:
     - lists*.katacontainers.io
     - lists*.open*.org
   mirror:
-    - mirror[0-9]*.open*.org
+    - mirror[0-9]*.openstack.org
+  mirror_opendev:
+    - mirror[0-9]*.opendev.org
   nodepool:
     - nb[0-9]*.open*.org
     - nl[0-9]*.open*.org
@@ -106,7 +109,7 @@ groups:
     - logstash-worker[0-9]*.open*.org
     - logstash[0-9]*.open*.org
     - mirror-update[0-9]*.open*.org
-    - mirror[0-9]*.open*.org
+    - mirror[0-9]*.openstack.org
     - nb[0-9]*.open*.org
     - nl[0-9]*.open*.org
     - openstackid-dev*.openstack.org
@@ -158,7 +161,7 @@ groups:
     - logstash-worker[0-9]*.open*.org
     - logstash[0-9]*.open*.org
     - mirror-update[0-9]*.open*.org
-    - ^mirror[0-9].*\..*\.(?!linaro|linaro-london|arm64ci).*\.open.*\.org
+    - ^mirror[0-9].*\..*\.(?!linaro|linaro-london|arm64ci).*\.openstack\.org
     - ^nb(?!03)[0-9]*\.open.*\.org
     - nl[0-9]*.open*.org
     - openstackid[0-9]*.openstack.org

playbooks/group_vars/mirror_opendev.yaml

@@ -0,0 +1,6 @@
+iptables_extra_public_tcp_ports:
+  - 80
+  - 443
+  - 8080
+  - 8081
+  - 8082

playbooks/roles/letsencrypt-create-certs/handlers/main.yaml

@@ -30,3 +30,6 @@
   import_tasks: touch_file.yaml
   vars:
     touch_file: '/tmp/letsencrypt02-main-service.stamp'
+
+- name: letsencrypt updated mirror01-region-provider-opendev-org-main
+  import_tasks: restart_apache.yaml

playbooks/roles/letsencrypt-create-certs/handlers/restart_apache.yaml

@@ -0,0 +1,8 @@
+- name: Populate service facts
+  service_facts:
+
+- name: Restart apache
+  service:
+    name: apache2
+    state: restarted
+  when: "'apache2' in ansible_facts.services"

playbooks/roles/mirror/README.rst

@@ -0,0 +1,6 @@
+Configure an opendev mirror
+
+This role installs and configures a mirror node
+
+**Role Variables**
+

playbooks/roles/mirror/defaults/main.yaml

@@ -0,0 +1,3 @@
+mirror_root: '/afs/openstack.org/mirror'
+www_base: '/var/www'
+www_root: '{{ www_base }}/mirror'

playbooks/roles/mirror/files/robots.txt

@@ -0,0 +1,2 @@
+User-agent: *
+Disallow: /

playbooks/roles/mirror/handlers/main.yaml

@@ -0,0 +1,4 @@
+- name: restart apache2
+  service:
+    name: apache2
+    state: restarted

playbooks/roles/mirror/tasks/main.yaml

@@ -0,0 +1,151 @@
+- name: Check AFS mounted
+  stat:
+    path: "/afs/openstack.org/mirror"
+  register: afs_mirror
+- name: Sanity check AFS
+  assert:
+    that:
+      - afs_mirror.stat.exists
+
+- name: Install apache2
+  apt:
+    name:
+      - apache2
+      - apache2-utils
+    state: present
+
+- name: Rewrite module
+  apache2_module:
+    state: present
+    name: rewrite
+
+- name: Substitute module
+  apache2_module:
+    state: present
+    name: substitute
+
+- name: Cache module
+  apache2_module:
+    state: present
+    name: cache
+
+- name: Cache disk module
+  apache2_module:
+    state: present
+    name: cache_disk
+
+- name: Proxy module
+  apache2_module:
+    state: present
+    name: proxy
+
+- name: HTTP Proxy module
+  apache2_module:
+    state: present
+    name: proxy_http
+
+- name: Apache macro module
+  apache2_module:
+    state: present
+    name: macro
+
+- name: Apache 2 ssl module
+  apache2_module:
+    state: present
+    name: ssl
+
+- name: Apache webroot
+  file:
+    path: '{{ www_base }}'
+    state: directory
+    owner: root
+    group: root
+
+- name: Apache www root
+  file:
+    path: '{{ www_root }}'
+    state: directory
+    owner: root
+    group: root
+
+- name: AFS content symlinks
+  file:
+    src: '{{ mirror_root }}/{{ item }}'
+    dest: '{{ www_root }}/{{ item }}'
+    state: link
+    owner: root
+    group: root
+  with_items:
+    - centos
+    - ceph-deb-hammer
+    - ceph-deb-jewel
+    - ceph-deb-luminous
+    - ceph-deb-mimic
+    - deb-docker
+    - debian
+    - debian-security
+    - debian-openstack
+    - epel
+    - fedora
+    - opensuse
+    - ubuntu-ports
+    - ubuntu-cloud-archive
+    - wheel
+    - yum-puppetlabs
+
+- name: Install robots.txt
+  copy:
+    src: robots.txt
+    dest: '{{ www_root }}'
+    owner: root
+    group: root
+    mode: 0444
+
+- name: Apache proxy cache
+  file:
+    path: /var/cache/apache2/proxy
+    owner: www-data
+    group: www-data
+    mode: 0755
+    state: directory
+
+- name: Set mirror servername and alias
+  set_fact:
+    apache_server_name: '{{ inventory_hostname }}'
+    # Strip the numeric host value from mirror01.region.provider.o.o
+    # for the serveralias
+    apache_server_alias: '{{ inventory_hostname | regex_replace("^mirror\d\d\.", "mirror.") }}'
+
+- name: Create mirror virtual host
+  template:
+    src: mirror.vhost.j2
+    dest: /etc/apache2/sites-available/mirror.conf
+
+- name: Make sure default site disabled
+  command: a2dissite 000-default.conf
+  args:
+    removes: /etc/apache2/sites-enabled/000-default.conf
+
+- name: Enable mirror virtual host
+  command: a2ensite mirror
+  args:
+    creates: /etc/apache2/sites-enabled/mirror.conf
+  notify:
+    - restart apache2
+
+- name: Debug config
+  slurp:
+    src: /etc/apache2/sites-available/mirror.conf
+  register: http_config
+- name: Show config
+  debug:
+    msg: '{{ http_config["content"] | b64decode }}'
+
+# Clean apache cache once an hour, keep size down to 70GiB.
+- name: Proxy cleanup cron job
+  cron:
+    name: Apache cache cleanup
+    state: present
+    job: /usr/bin/flock -n /var/run/htcacheclean.lock /usr/bin/htcacheclean -n -p /var/cache/apache2/proxy -t -l 70200M > /dev/null
+    minute: '0'
+    hour: '*'

playbooks/roles/mirror/templates/mirror.vhost.j2

@@ -0,0 +1,404 @@
+NameVirtualHost *:80
+NameVirtualHost *:443
+
+# Dedicated port for proxy caching, as not to affect afs mirrors.
+Listen 8080
+NameVirtualHost *:8080
+
+Listen 8081
+NameVirtualHost *:8081
+
+Listen 8082
+NameVirtualHost *:8082
+
+LogFormat "%h %l %u %t \"%r\" %>s %b %{cache-status}e \"%{Referer}i\" \"%{User-agent}i\"" combined-cache
+
+<Macro BaseProxy $port>
+
+    DocumentRoot /var/www/mirror
+    <Directory /var/www/mirror>
+        Options Indexes FollowSymLinks MultiViews
+        AllowOverride None
+        Order allow,deny
+        allow from all
+        Satisfy any
+        <IfVersion >= 2.4>
+            Require all granted
+        </IfVersion>
+    </Directory>
+
+    # Caching reverse proxy for things that don't make sense in AFS
+    #
+    # General cache rules
+    CacheRoot "/var/cache/apache2/proxy"
+    CacheDirLevels 5
+    CacheDirLength 2
+    # SSL support
+    SSLProxyEngine on
+    # Prevent thundering herds.
+    CacheLock on
+    CacheLockPath "/tmp/mod_cache-lock"
+    CacheLockMaxAge 5
+    # 5GiB
+    CacheMaxFileSize 5368709120
+    CacheStoreExpired On
+    # Pip sets Cache-Control: max-age=0 on requests for pypi index pages.
+    # This means we don't use the cache for those requests. This setting
+    # should force the proxy to ignore cache-control on the request side
+    # but we should still cache things based on the cache-control responses
+    # from the backed servers.
+    CacheIgnoreCacheControl On
+
+    # Added Aug 2017 in an attempt to avoid occasional 502 errors (around
+    # 0.05% of requests) of the type:
+    #
+    #  End of file found: ... AH01102: error reading status line from remote server ...
+    #
+    # Per [1]:
+    #
+    #  This avoids the "proxy: error reading status line from remote
+    #  server" error message caused by the race condition that the backend
+    #  server closed the pooled connection after the connection check by the
+    #  proxy and before data sent by the proxy reached the backend.
+    #
+    # [1] https://httpd.apache.org/docs/2.4/mod/mod_proxy_http.html
+    SetEnv proxy-initial-not-pooled 1
+
+    RewriteEngine On
+    # pypi
+    CacheEnable disk  "/pypi"
+    ProxyPass "/pypi/" "https://pypi.org/" ttl=120 keepalive=On retry=0
+    ProxyPassReverse "/pypi/" "https://pypi.org/
+
+    # files.pythonhosted.org
+    CacheEnable disk  "/pypifiles"
+    ProxyPass "/pypifiles/" "https://files.pythonhosted.org/" ttl=120 keepalive=On retry=0
+    ProxyPassReverse "/pypifiles/" "https://files.pythonhosted.org/"
+
+    # Rewrite the locations of the actual files
+    <Location /pypi>
+      SetOutputFilter INFLATE;SUBSTITUTE;DEFLATE
+      Substitute "s|https://files.pythonhosted.org/|/pypifiles/|ni"
+    </Location>
+
+    # Wheel URL's are:
+    # /wheel/{distro}-{distro-version}/a/a/a-etc.whl
+    # /wheel/{distro}-{distro-version}/a/abcd/abcd-etc.whl
+    # /wheel/{distro}-{distro-version}/a/abcde/abcde-etc.whl
+    RewriteCond %{REQUEST_URI} ^/wheel/([^/]+)/([^/])([^/]*)
+    RewriteCond %{DOCUMENT_ROOT}/wheel/$1/$2/$2$3 -d
+    RewriteRule ^/wheel/([^/]+)/([^/])([^/]*)(/.*)?$ /wheel/$1/$2/$2$3$4 [L]
+
+    # Special cases for openstack.nose_plugin & backports.*
+    RewriteCond %{REQUEST_URI} ^/wheel/
+    RewriteRule ^(.*)/openstack-nose-plugin(.*)$ $1/openstack.nose_plugin$2
+    RewriteCond %{REQUEST_URI} ^/wheel/
+    RewriteRule ^(.*)/backports-(.*)$ $1/backports.$2
+
+    # Try again but replacing -'s with .'s
+    RewriteCond %{REQUEST_URI} ^/wheel/
+    RewriteCond %{DOCUMENT_ROOT}/%{REQUEST_URI} !-f
+    RewriteCond %{DOCUMENT_ROOT}/%{REQUEST_URI} !-d
+    RewriteRule (.*)-(.*) $1.$2 [N]
+
+    ErrorLog /var/log/apache2/proxy_$port_error.log
+    LogLevel warn
+    CustomLog /var/log/apache2/proxy_$port_access.log combined-cache
+    ServerSignature Off
+
+</Macro>
+
+<VirtualHost *:80>
+    ServerName {{ apache_server_name }}
+    ServerAlias {{ apache_server_alias }}
+
+    Use BaseProxy 80
+</VirtualHost>
+
+<VirtualHost *:443>
+    ServerName {{ apache_server_name }}
+    ServerAlias {{ apache_server_alias }}
+
+    SSLCertificateFile      /etc/letsencrypt-certs/{{ apache_server_name }}/{{ apache_server_name }}.cer
+    SSLCertificateKeyFile   /etc/letsencrypt-certs/{{ apache_server_name }}/{{ apache_server_name }}.key
+    SSLCertificateChainFile /etc/letsencrypt-certs/{{ apache_server_name }}/ca.cer
+    SSLProtocol All -SSLv2 -SSLv3
+    # Note: this list should ensure ciphers that provide forward secrecy
+    SSLCipherSuite ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:!AES256:!aNULL:!eNULL:!MD5:!DSS:!PSK:!SRP
+    SSLHonorCipherOrder on
+
+    Use BaseProxy 443
+</VirtualHost>
+
+<VirtualHost *:8080>
+    ServerName {{ apache_server_name }}:8080
+    ServerAlias {{ apache_server_alias }}:8080
+
+    # Disable directory listing by default.
+    <Directory />
+      Order Deny,Allow
+      Deny from all
+      Options None
+      AllowOverride None
+    </Directory>
+
+    ErrorLog /var/log/apache2/proxy_8080_error.log
+    LogLevel warn
+    CustomLog /var/log/apache2/proxy_8080_access.log combined-cache
+    ServerSignature Off
+
+    # Caching reverse proxy for things that don't make sense in AFS
+    #
+    # General cache rules
+    CacheRoot "/var/cache/apache2/proxy"
+    CacheDirLevels 5
+    CacheDirLength 2
+    # SSL support
+    SSLProxyEngine on
+    # Prevent thundering herds.
+    CacheLock on
+    CacheLockPath "/tmp/mod_cache-lock"
+    CacheLockMaxAge 5
+    # 5GiB
+    CacheMaxFileSize 5368709120
+    CacheStoreExpired On
+
+    # Added Aug 2017 in an attempt to avoid occasional 502 errors (around
+    # 0.05% of requests) of the type:
+    #
+    #  End of file found: ... AH01102: error reading status line from remote server ...
+    #
+    # Per [1]:
+    #
+    #  This avoids the "proxy: error reading status line from remote
+    #  server" error message caused by the race condition that the backend
+    #  server closed the pooled connection after the connection check by the
+    #  proxy and before data sent by the proxy reached the backend.
+    #
+    # [1] https://httpd.apache.org/docs/2.4/mod/mod_proxy_http.html
+    SetEnv proxy-initial-not-pooled 1
+
+    # Per site caching reverse proxy rules
+    # Only cache specific backends, rely on afs cache otherwise.
+
+    # buildlogs.centos.org (302 redirects to buildlogs.cdn.centos.org)
+    CacheEnable disk  "/buildlogs.centos"
+    ProxyPass "/buildlogs.centos/" "https://buildlogs.centos.org/" ttl=120 disablereuse=On retry=0
+    ProxyPassReverse "/buildlogs.centos/" "https://buildlogs.centos.org/"
+
+    # buildlogs.cdn.centos.org
+    CacheEnable disk  "/buildlogs.cdn.centos"
+    ProxyPass "/buildlogs.cdn.centos/" "https://buildlogs.cdn.centos.org/" ttl=120 disablereuse=On retry=0
+    ProxyPassReverse "/buildlogs.cdn.centos/" "https://buildlogs.cdn.centos.org/"
+
+    # rdo
+    CacheEnable disk  "/rdo"
+    ProxyPass "/rdo/" "https://trunk.rdoproject.org/" ttl=120 keepalive=On retry=0
+    ProxyPassReverse "/rdo/" "https://trunk.rdoproject.org/"
+
+    # cbs.centos.org
+    CacheEnable disk  "/cbs.centos"
+    ProxyPass "/cbs.centos/" "https://cbs.centos.org/repos/" ttl=120 keepalive=On retry=0
+    ProxyPassReverse "/cbs.centos/" "https://cbs.centos.org/repos/"
+
+    # tarballs
+    CacheEnable disk  "/tarballs"
+    ProxyPass "/tarballs/" "https://tarballs.openstack.org/" ttl=120 keepalive=On retry=0
+    ProxyPassReverse "/tarballs/" "https://tarballs.openstack.org/"
+
+    # pypi
+    CacheEnable disk  "/pypi"
+    ProxyPass "/pypi/" "https://pypi.org/" ttl=120 keepalive=On retry=0
+    ProxyPassReverse "/pypi/" "https://pypi.org/
+
+    # files.pythonhosted.org
+    CacheEnable disk  "/pypifiles"
+    ProxyPass "/pypifiles/" "https://files.pythonhosted.org/" ttl=120 keepalive=On retry=0
+    ProxyPassReverse "/pypifiles/" "https://files.pythonhosted.org/"
+
+    # Rewrite the locations of the actual files
+    <Location /pypi>
+      SetOutputFilter INFLATE;SUBSTITUTE;DEFLATE
+      Substitute "s|https://files.pythonhosted.org/|/pypifiles/|ni"
+    </Location>
+
+    # images.linuxcontainers.org
+    CacheEnable disk "/images.linuxcontainers"
+    ProxyPass "/images.linuxcontainers/" "http://us.images.linuxcontainers.org/" ttl=120 keepalive=On retry=0
+    ProxyPassReverse "/images.linuxcontainers/" "http://us.images.linuxcontainers.org/"
+
+    # registry.npmjs.org
+    CacheEnable disk  "/registry.npmjs"
+    ProxyPass "/registry.npmjs/" "https://registry.npmjs.org/" ttl=120 keepalive=On retry=0
+    ProxyPassReverse "/registry.npmjs/" "https://registry.npmjs.org/"
+
+    # api.rubygems.org
+    CacheEnable disk  "/api.rubygems"
+    ProxyPass "/api.rubygems/" "https://api.rubygems.org/" ttl=120 keepalive=On retry=0
+    ProxyPassReverse "/api.rubygems/" "https://api.rubygems.org/"
+
+    # rubygems.org
+    CacheEnable disk  "/rubygems"
+    ProxyPass "/rubygems/" "https://rubygems.org/" ttl=120 keepalive=On retry=0
+    ProxyPassReverse "/rubygems/" "https://rubygems.org/"
+
+    # opendaylight
+    CacheEnable disk "/opendaylight"
+    ProxyPass "/opendaylight/" "https://nexus.opendaylight.org/" ttl=120 keepalive=On retry=0
+    ProxyPassReverse "/opendaylight/" "https://nexus.opendaylight.org/"
+
+    # elastico
+    CacheEnable disk "/elastic"
+    ProxyPass "/elastic/" "https://packages.elastic.co/" ttl=120 keepalive=On retry=0
+    ProxyPassReverse "/elastic/" "https://packages.elastic.co/"
+
+    # grafana
+    CacheEnable disk "/grafana"
+    ProxyPass "/grafana" "https://packagecloud.io/grafana/" ttl=120 keepalive=On retry=0
+    ProxyPassReverse "/grafana/" "https://packagecloud.io/grafana/"
+
+    # OracleLinux
+    CacheEnable disk "/oraclelinux"
+    ProxyPass "/oraclelinux/" "http://yum.oracle.com/repo/OracleLinux/" ttl=120 keepalive=On retry=0
+    ProxyPassReverse "/oraclelinux/" "http://yum.oracle.com/repo/OracleLinux/"
+
+    # Percona
+    CacheEnable disk "/percona"
+    ProxyPass "/percona/" "https://repo.percona.com/" ttl=120 keepalive=On retry=0
+    ProxyPassReverse "/percona/" "https://repo.percona.com/"
+
+    # MariaDB
+    CacheEnable disk "/MariaDB"
+    ProxyPass "/MariaDB/" "https://downloads.mariadb.com/MariaDB/" ttl=120 keepalive=On retry=0
+    ProxyPassReverse "/MariaDB/" "https://downloads.mariadb.com/MariaDB/"
+
+    # Docker
+    CacheEnable disk "/docker"
+    ProxyPass "/docker/" "https://download.docker.com/linux/" ttl=120 keepalive=On retry=0
+    ProxyPassReverse "/docker/" "https://download.docker.com/linux/"
+
+    # Alpine
+    CacheEnable disk "/alpine"
+    ProxyPass "/alpine/" "http://dl-cdn.alpinelinux.org/alpine/" ttl=120 keepalive=On retry=0
+    ProxyPassReverse "/alpine/" "http://dl-cdn.alpinelinux.org/alpine/"
+
+    # LXC (copr)
+    CacheEnable disk "/copr-lxc2"
+    ProxyPass "/copr-lxc2/" "https://copr-be.cloud.fedoraproject.org/results/thm/lxc2.0/" ttl=120 keepalive=On retry=0
+    ProxyPassReverse "/copr-lxc2/" "https://copr-be.cloud.fedoraproject.org/results/thm/lxc2.0/"
+
+</VirtualHost>
+
+# Docker registry v1 proxy.
+<VirtualHost *:8081>
+    ServerName {{ apache_server_name }}:8081
+    ServerAlias {{ apache_server_alias }}:8081
+
+    # Disable directory listing by default.
+    <Directory />
+      Order Deny,Allow
+      Deny from all
+      Options None
+      AllowOverride None
+    </Directory>
+
+    ErrorLog /var/log/apache2/proxy_8081_error.log
+    LogLevel warn
+    CustomLog /var/log/apache2/proxy_8081_access.log combined-cache
+    ServerSignature Off
+
+    # Caching reverse proxy for things that don't make sense in AFS
+    #
+    # General cache rules
+    CacheRoot "/var/cache/apache2/proxy"
+    CacheDirLevels 5
+    CacheDirLength 2
+    # SSL support
+    SSLProxyEngine on
+    # Prevent thundering herds.
+    CacheLock on
+    CacheLockPath "/tmp/mod_cache-lock"
+    CacheLockMaxAge 5
+    # 5GiB
+    CacheMaxFileSize 5368709120
+    # Ignore expire headers as the urls use sha256 hashes.
+    CacheIgnoreQueryString On
+    # NOTE(pabelanger): In the case of docker, if neither an expiry date nor
+    # last-modified date are provided default expire to 1 day. This is up from
+    # 1 hour.
+    CacheDefaultExpire 86400
+    CacheStoreExpired On
+
+    # registry-1.docker.io
+    CacheEnable disk  "/registry-1.docker"
+    ProxyPass "/registry-1.docker/" "https://registry-1.docker.io/" ttl=120 keepalive=On retry=0
+    ProxyPassReverse "/registry-1.docker/" "https://registry-1.docker.io/"
+
+    # dseasb33srnrn.cloudfront.net
+    CacheEnable disk "/cloudfront"
+    ProxyPass "/cloudfront/" "https://dseasb33srnrn.cloudfront.net/" ttl=120 keepalive=On retry=0
+    ProxyPassReverse "/cloudfront/" "https://dseasb33srnrn.cloudfront.net/"
+
+    # production.cloudflare.docker.com
+    CacheEnable disk "/cloudflare"
+    ProxyPass "/cloudflare/" "https://production.cloudflare.docker.com/" ttl=120 keepalive=On retry=0
+    ProxyPassReverse "/cloudflare/" "https://production.cloudflare.docker.com/"
+
+</VirtualHost>
+
+# Docker registry v2 proxy.
+<VirtualHost *:8082>
+    ServerName {{ apache_server_name }}:8082
+    ServerAlias {{ apache_server_alias }}:8082
+
+    # Disable directory listing by default.
+    <Directory />
+      Order Deny,Allow
+      Deny from all
+      Options None
+      AllowOverride None
+    </Directory>
+
+    ErrorLog /var/log/apache2/proxy_8082_error.log
+    LogLevel warn
+    CustomLog /var/log/apache2/proxy_8082_access.log combined-cache
+    ServerSignature Off
+
+    # Caching reverse proxy for things that don't make sense in AFS
+    #
+    # General cache rules
+    CacheRoot "/var/cache/apache2/proxy"
+    CacheDirLevels 5
+    CacheDirLength 2
+    # SSL support
+    SSLProxyEngine on
+    # Prevent thundering herds.
+    CacheLock on
+    CacheLockPath "/tmp/mod_cache-lock"
+    CacheLockMaxAge 5
+    # 5GiB
+    CacheMaxFileSize 5368709120
+    # Ignore expire headers as the urls use sha256 hashes.
+    CacheIgnoreQueryString On
+    # NOTE(pabelanger): In the case of docker, if neither an expiry date nor
+    # last-modified date are provided default expire to 1 day. This is up from
+    # 1 hour.
+    CacheDefaultExpire 86400
+    CacheStoreExpired On
+
+    # dseasb33srnrn.cloudfront.net
+    CacheEnable disk "/cloudfront"
+    ProxyPass "/cloudfront/" "https://dseasb33srnrn.cloudfront.net/" ttl=120 keepalive=On retry=0
+    ProxyPassReverse "/cloudfront/" "https://dseasb33srnrn.cloudfront.net/"
+
+    # production.cloudflare.docker.com
+    CacheEnable disk "/cloudflare"
+    ProxyPass "/cloudflare/" "https://production.cloudflare.docker.com/" ttl=120 keepalive=On retry=0
+    ProxyPassReverse "/cloudflare/" "https://production.cloudflare.docker.com/"
+
+    # NOTE(corvus): Ensure this stanza is last since it's the most
+    # greedy match.
+    CacheEnable disk  "/"
+    ProxyPass "/" "https://registry-1.docker.io/" ttl=120 keepalive=On retry=0
+    ProxyPassReverse "/" "https://registry-1.docker.io/"
+</VirtualHost>

playbooks/service-mirror.yaml

@@ -0,0 +1,11 @@
+- hosts: "mirror_opendev:!disabled"
+  name: "Configure per region opendev mirrors"
+  roles:
+    - role: kerberos-client
+      kerberos_realm: 'OPENSTACK.ORG'
+      kerberos_admin_server: 'kdc.openstack.org'
+      kerberos_kdcs:
+        - kdc03.openstack.org
+        - kdc04.openstack.org
+    - role: openafs-client
+    - role: mirror

playbooks/zuul/run-base.yaml

@@ -81,6 +81,7 @@
         - host_vars/bridge.openstack.org.yaml
         - host_vars/letsencrypt01.opendev.org.yaml
         - host_vars/letsencrypt02.opendev.org.yaml
+        - host_vars/mirror01.region.provider.opendev.org.yaml
     - name: Display group membership
       command: ansible localhost -m debug -a 'var=groups'
     - name: Run base.yaml

playbooks/zuul/templates/gate-groups.yaml.j2

@@ -8,3 +8,4 @@ groups:
   letsencrypt:
     - letsencrypt01.opendev.org
     - letsencrypt02.opendev.org
+    - mirror01.region.provider.opendev.org

playbooks/zuul/templates/host_vars/mirror01.region.provider.opendev.org.yaml.j2

@@ -0,0 +1,4 @@
+letsencrypt_certs:
+  mirror01-region-provider-opendev-org-main:
+    - mirror01.region.provider.opendev.org
+    - mirror.region.provider.opendev.org

run_all.sh

@@ -100,6 +100,10 @@ start_timer
 timeout -k 2m 30m ansible-playbook -f 50 ${ANSIBLE_PLAYBOOKS}/service-nodepool.yaml
 send_timer nodepool
 
+start_timer
+timeout -k 2m 30m ansible-playbook -f 50 ${ANSIBLE_PLAYBOOKS}/service-mirror.yaml
+send_timer nodepool
+
 start_timer
 timeout -k 2m 30m ansible-playbook -f 50 ${ANSIBLE_PLAYBOOKS}/service-registry.yaml
 send_timer registry

testinfra/test_mirror.py

@@ -0,0 +1,32 @@
+# Copyright 2019 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+testinfra_hosts = ['mirror01.region.provider.opendev.org']
+
+
+def test_apache(host):
+    apache = host.service('apache2')
+    assert apache.is_running
+
+def test_mirror_indexes(host):
+    cmd = host.run("wget --no-check-certificate -qO- https://localhost/")
+    assert '<a href="debian/">' in cmd.stdout
+
+    cmd = host.run("wget -qO- http://localhost/")
+    assert '<a href="debian/">' in cmd.stdout
+
+# NOTE(ianw): further testing idea for anyone interested; get the
+# actual IP address of the mirror node and connect via that, and then
+# also poke at the other proxy ports