master-nameserver: Add unmanaged domains; add acme.opendev.org

This adds the concept of an unmanaged domain; for unmanaged domains we
will write out the zone file only if it doesn't already exist.

acme.opendev.org is added as an unmanaged domain.  It will be managed
by other ansible roles which add TXT records for ACME authentication.
The initial template comes from the dependent change, and this ensures
the bind configuration is always valid.

For flexibility and testing purposes, we allow passing an extra
refspec and version to the git checkout.  This is one way to pull in
changes for speculative CI runs (I looked into having the hosts under
test checkout from Zuul; but by the time we're 3-ansible call's deep
on the DNS hosts-under-test it's a real pain.  For the amount of times
we update this, it's easier to just allow a speculative change that
can take a gerrit URL; for an example see [1])

[1] https://review.openstack.org/#/c/641155/10/playbooks/group_vars/dns.yaml

Testing is enhanced to check for zone files and correct configuration
stanzas.

Depends-On: https://review.openstack.org/641154
Depends-On: https://review.openstack.org/641168
Change-Id: I9ef5cfc850c3458c63aff46cfaa0d49a5d194e87

.zuul.yaml

@@ -482,6 +482,9 @@
     parent: system-config-run
     description: |
       Run the playbook for dns.
+    required-projects:
+      - openstack-infra/zone-opendev.org
+      - openstack-infra/zone-zuul-ci.org
     nodeset:
       nodes:
         - name: bridge.openstack.org
@@ -490,6 +493,11 @@
           label: ubuntu-bionic
         - name: ns1.opendev.org
           label: ubuntu-bionic
+    host-vars:
+      adns1.opendev.org:
+        host_copy_output:
+          '/etc/bind/named.conf': logs
+          '/var/lib/bind/zones': logs
     files:
       - .zuul.yaml
       - playbooks/group_vars/adns.yaml

playbooks/group_vars/dns.yaml

@@ -6,6 +6,9 @@ dns_repos:
 dns_zones:
   - name: opendev.org
     source: zone-opendev.org/zones/opendev.org/
+  - name: acme.opendev.org
+    source: zone-opendev.org/zones/acme.opendev.org/
+    unmanaged: True
   - name: zuul-ci.org
     source: zone-zuul-ci.org/zones/zuul-ci.org/
   - name: zuulci.org

playbooks/roles/master-nameserver/README.rst

@@ -51,6 +51,14 @@ nameserver.
 
       The URL of the git repository.
 
+   .. zuul:rolevar:: refspec
+
+      Add an additional refspec passed to the git checkout
+
+   .. zuul:rolevar:: version
+
+      An additional version passed to the git checkout
+
 .. zuul:rolevar:: dns_zones
    :type: list
 
@@ -70,6 +78,14 @@ nameserver.
       located at ``zones/example_com/zone.db``, then the value here
       should be ``example.com/zones/example_com``.
 
+   .. zuul:rolevar:: unmanaged
+      :type: bool
+      :default: False
+
+      If ``True`` the zone is considered unmanaged.  The ``source``
+      file will be put in place if it does not exist, but will
+      otherwise be left alone.
+
 .. zuul:rolevar:: dns_notify
    :type: list
 

playbooks/roles/master-nameserver/tasks/main.yaml

@@ -12,16 +12,21 @@
 - name: Clone zone repos
   git:
     repo: "{{ item.url }}"
+    refspec: "{{ item.refspec | default(omit) }}"
+    version: "{{ item.version | default(omit) }}"
     dest: "/opt/source/{{ item.name }}"
   loop: "{{ dns_repos }}"
+- name: Set base rsync options
+  set_fact:
+    _rsync_options:
+      - "--chmod=u+rwX,g+rX,o+rX"
+      - "--chown=bind:bind"
 - name: Synchronize zone repos to zone directories
   delegate_to: "{{ inventory_hostname }}"
   synchronize:
     src: "/opt/source/{{ item.source }}"
     dest: "/var/lib/bind/zones/{{ item.name }}"
-    rsync_opts:
+    rsync_opts: '{{ _rsync_options + ["--ignore-existing"] if item.unmanaged|default(False) else _rsync_options }}'
-      - "--chmod=u+rwX,g+rX,o+rX"
-      - "--chown=bind:bind"
   loop: "{{ dns_zones }}"
   notify: Reload named
 - name: Install tsig key

playbooks/zuul/templates/group_vars/adns.yaml.j2

@@ -134,3 +134,47 @@ dnssec_keys:
       Created: 20190326230948
       Publish: 20190326230948
       Activate: 20190326230948
+  '32631':
+    zone: acme.opendev.org
+    public: |
+      ; This is a zone-signing key, keyid 32631, for acme.opendev.org.
+      ; Created: 20190326051524 (Tue Mar 26 05:15:24 2019)
+      ; Publish: 20190326051524 (Tue Mar 26 05:15:24 2019)
+      ; Activate: 20190326051524 (Tue Mar 26 05:15:24 2019)
+      acme.opendev.org. IN DNSKEY 256 3 8 AwEAAcUE5JwzrD69s2SoTlCr1xyfw/9iX9IJKPBwRE0YCMe5GtSxjB71 aeFhvELg8xVuCVBJ8Af9x5GrbpSYP37GI5zNe3WGr+7YX9LsVOGnR4L6 GF096qEwcMLaEDUOMShcN8N0qV2/Cj6a8GaBxTDGavcq35mnmFtKXfrt VXchI0crf2Pl34rOBop8VcjQBepivmMA46hVzlJxQDek93XKP4EAi7Tw 8NN0PAT69XS4oHaoBCYzG6I3PcsStnhgdLDn8ppI3ZuxCzpNbWV94CBr K6/Stz+8ec0eHUXuh8EGfO3Xwd2+LV0WGMeahHzz8fPYyWvmPDprKiDF nUeVEWqVzLk=
+    private: |
+      Private-key-format: v1.3
+      Algorithm: 8 (RSASHA256)
+      Modulus: xQTknDOsPr2zZKhOUKvXHJ/D/2Jf0gko8HBETRgIx7ka1LGMHvVp4WG8QuDzFW4JUEnwB/3HkatulJg/fsYjnM17dYav7thf0uxU4adHgvoYXT3qoTBwwtoQNQ4xKFw3w3SpXb8KPprwZoHFMMZq9yrfmaeYW0pd+u1VdyEjRyt/Y+Xfis4GinxVyNAF6mK+YwDjqFXOUnFAN6T3dco/gQCLtPDw03Q8BPr1dLigdqgEJjMbojc9yxK2eGB0sOfymkjdm7ELOk1tZX3gIGsrr9K3P7x5zR4dRe6HwQZ87dfB3b4tXRYYx5qEfPPx89jJa+Y8OmsqIMWdR5URapXMuQ==
+      PublicExponent: AQAB
+      PrivateExponent: mn42wmImvGBHTzRHjSzjFvgVWqsKlopGRxzSAl5JbEwzxPug9BnfuDPKy+rX00MhHIuOJMYVe54hrXYhvEilXm0nVcaTKUkVAzH9caGaCxQQjPVjipiQo8sZkHEbjRmbRLKzqOaIowUeZFN4jMHa2Q0On8/zQgrz3TPEpBEhN8l8IZxpkciAHpiFffBhM98bkLBGWJS7hRc7QpNINpNR866RQNxvXqOgiEbS42ej28BkfpTc4QKzoZQck9Wu7UVjV9Udg5/tna0ZQTuPNbwoD6tTycu9J1P9ZKEBB3e3D3X9ZGMA6A2nmAAImRqURL8Nt1f5OdrodDlgoA1yJFOtAQ==
+      Prime1: 8KT+jPQfVPk6/PtruBJpSOa4V9Pbnl9AuL6tfyN2953gnrNl4od4QpN6dFq4kU/a8qF0GOI/MpcVQWP2BRvdkxwh02EDD65A9hmK3zbl7MKwW5hWtzsVMwINru/zRww6lHk5wzlE6MfqN0Mq9U8g0rprxcPMEN7xNjS/ghGZxZk=
+      Prime2: 0ZdDhdOUcm/7LuV2cNJonfhw5ocBgxDXF1EfYxyF+qKoWOLtz7CjiJCfxFCPHoMmeUL8E10QokIX/1/F+b87Rwr619VhW3TNRae7lowpdEnBueliOnzeOcpW988Ir+UvdlvK9cD5GvgN1GuysXUQlKwFMT9XjxoULjLW52pKdCE=
+      Exponent1: x0I3rIsvrnK4j2W36jEEkOLKXZ8FSPviYZcxngbFqX9G0OIHSS2XPLlVOicskNYom6NouHoOjltftEeLHOvX6snukFLR8Bf/nkfEH9QbSpJi6VUY6Ju5kATxQ5tYO8o6b1p5o9c14fI3VA7/8SPWL+dA+f6IaKfR32qJ8K+WPnE=
+      Exponent2: ryXYQIq6gBOCdgM9wjSjRnfqaUsjAVNeW9boAtxAPl4Vjwo8r5YuYx5w1Q55O4df7HAE1W2tS9st0LRJblbXg5vyWdGwZUwrim0MP1fsAIjugp09ACF/WA32NWpnGQ7OZft5lXto8JegfwZtMwzgCU3jnO8RDb4+ZQkJPCRACeE=
+      Coefficient: m3u9O/Xl/bRMBMxxiBN7K2fJnhIjXYb9gpL6kKDi6fCXUrh7SF5LBRUtAH65OFUZ8N9St55UrnuZwwTw3sE3ikf1I6aNu0rwdNg0h+Fos3Q4yj6cYHSydiXe2e0NWIRTqEUcEscbCAJ53IdPbdxHFupp8elR6VmAsS25e9f0fPw=
+      Created: 20190326051524
+      Publish: 20190326051524
+      Activate: 20190326051524
+  '62692':
+    zone: acme.opendev.org
+    public: |
+      ; This is a key-signing key, keyid 62692, for acme.opendev.org.
+      ; Created: 20190326051559 (Tue Mar 26 05:15:59 2019)
+      ; Publish: 20190326051559 (Tue Mar 26 05:15:59 2019)
+      ; Activate: 20190326051559 (Tue Mar 26 05:15:59 2019)
+      acme.opendev.org. IN DNSKEY 257 3 8 AwEAAbjAUwmuDM9qaw9moFESZy5mTMb5QJtOs5VU/5aWuwezJwlR4RO+ xw1yIoxunIlU2i7Vjr4Vn/jgbOwlGEYEg28qbQt8GH0R5pA4IbrV++3Q BvPJbbGLTIm2/yvWIwk8hLXzl3oeAESjjH0DNb3rEmINX8LXstIm8XWw /HIZ3gbRjzhjluE86/enf9gn3kVCpwD/rjwNPcVsdhEsOevjgPZ7iOv7 FnMIRFeN8eICMzi3LaL1dyRrLUBkf/yW1QIy3NFE80Ub4OykVeGDbIO6 zgYcB1r3/X/6hee82ck9nHHf8xsDQqZ54gqbte0a/TXb5D8hEUmXnWne ORvLM/Lyb60=
+    private: |
+      Private-key-format: v1.3
+      Algorithm: 8 (RSASHA256)
+      Modulus: uMBTCa4Mz2prD2agURJnLmZMxvlAm06zlVT/lpa7B7MnCVHhE77HDXIijG6ciVTaLtWOvhWf+OBs7CUYRgSDbyptC3wYfRHmkDghutX77dAG88ltsYtMibb/K9YjCTyEtfOXeh4ARKOMfQM1vesSYg1fwtey0ibxdbD8chneBtGPOGOW4Tzr96d/2CfeRUKnAP+uPA09xWx2ESw56+OA9nuI6/sWcwhEV43x4gIzOLctovV3JGstQGR//JbVAjLc0UTzRRvg7KRV4YNsg7rOBhwHWvf9f/qF57zZyT2ccd/zGwNCpnniCpu17Rr9NdvkPyERSZedad45G8sz8vJvrQ==
+      PublicExponent: AQAB
+      PrivateExponent: E2UdUobTEXM6igNcESa9bkGPDdRc0/EPKT4jFsv8FnLYRkIyPsBoZSD2P4fdJw2hWglRUuMySA5HYQMD6VXP9nudtvbwGzEl4z4BTHvqVqzgDfe3bEwTXOG5KADy7KVNyUwpOsirfoks1nLf0XA8Hc8JnorGWwl7j79kwRW2GUD483e45XvfGQjTnYC4f3RZmrhYiIaKDxA5uhVuILkqV1WN7dPLphQJhQGJEEI1r3rktg5rNwFwpVEHMapzuFj3st/G9COmCKMuemeNjbVPnxLH3iOmj4x82vDzNEnWjnssXSzzQvGranIOc7GB0wVpF/SqpBc6qJtEGqEYqOQIAQ==
+      Prime1: 4zXtaHG4VKGLQZX/Yi8alhsJGphyaRs61AmFD9AnmRL1M82Gl3WkPSTBlpCZsB4CT0wUFldteLlEVSC4Bw1rIdYGSxMzj37tIOdqQTBZ91qVQFTxH0EmS3TnKKVTsW+/3o8dmOIO0v+kBdsvE3RR/ARJchSppx9goVM6gXCRDt0=
+      Prime2: 0CkiX1uxqszinngsbcqqHD6Y/GNXdcu+/7YfHpFXebsLfqrkqhU3ZFTqypTbyeNRSg/q2z2i7W4PCDp4NECDQ3iVzr80vVMtaqXuAg0FQRMHHVCcuJ6RFnODAemt+sXuQ0S0O6G0WQK6CSiL20yUxJtfQ8rjStYtV9ydE8ZfjxE=
+      Exponent1: eXPiK+pd9h9EKRLdKMa1F3fsLeM/hR+hGqbcEc/a2uBfYgmC4INp/6UeNjWlcZcY9Ppd4nNpeRbPiBGtTVfG5JdbVdY1wYa/is8o5R/Ld4VcMr81BNf2eG9NAVUen8J0dataztZHxlIQg3DegS+0g1pnSCvzY/pJ1PKAW6CoaaE=
+      Exponent2: LLsaIsmudRiP/iOu0G0DfwxIjbu/OJXu1j5Jk6UB2ivCfZa1ioMCozHIPn4ceNa7SiH/gttM3p6O5mLCH+BZFK+d6Y6XA7QTB17etVwc6+3t0nPXKakRXnS2Czwu4buUxqnF3SaTfakjVwJ6g0aClXkZ0JSRoSxDFCVZL72qHTE=
+      Coefficient: Z7OL0bH9l2uNwYRECyEFuq7omma9DxA4XhCVeh8inhq1wBkzoH/4QmpIQAL8hY2eZQCNimhkMHOj41a2mqnFX5+/PQMEUXRopsueIRjRbHQ27wA1kmFiK+cybC7UyaN4yxVe/UUrtf/NDn4vhv0C/Q3cRlpVqAmDhUKIQsCEHac=
+      Created: 20190326051559
+      Publish: 20190326051559
+      Activate: 20190326051559

testinfra/test_adns.py

@@ -19,3 +19,22 @@ testinfra_hosts = ['adns1.opendev.org']
 def test_bind(host):
     named = host.service('bind9')
     assert named.is_running
+
+def test_zone_files(host):
+    opendev_zone = host.file('/var/lib/bind/zones/opendev.org')
+    assert opendev_zone.exists
+
+    acme_opendev_zone = host.file('/var/lib/bind/zones/acme.opendev.org')
+    assert acme_opendev_zone.exists
+
+    zuul_ci_zone = host.file('/var/lib/bind/zones/zuul-ci.org')
+    assert zuul_ci_zone.exists
+
+    zuulci_zone = host.file('/var/lib/bind/zones/zuulci.org')
+    assert zuulci_zone.exists
+
+    bind_config = host.file('/etc/bind/named.conf')
+    assert b'zone opendev.org {' in bind_config.content
+    assert b'zone acme.opendev.org {' in bind_config.content
+    assert b'zone zuul-ci.org {' in bind_config.content
+    assert b'zone zuulci.org {' in bind_config.content