Gate: Add support for testing fqdn over-rides in zuul

This PS adds support for testing fqdn over-rides in zuul gates.
When enabled it will direct requests to a configurable domain to
the default ip of the primary node.

Change-Id: I3d9a4a0bf06532caf0f544d44027493622f4ae5b
Signed-off-by: Pete Birley <pete@port.direct>

.zuul.yaml

@@ -73,6 +73,10 @@
     nodes:
       - name: primary
         label: ubuntu-xenial
+    groups:
+      - name: primary
+        nodes:
+          - primary
 
 - nodeset:
     name: openstack-helm-ubuntu
@@ -260,6 +264,7 @@
     vars:
       zuul_osh_relative_path: ../openstack-helm/
       kubernetes_keystone_auth: true
+      gate_fqdn_test: true
     parent: openstack-helm-infra
     nodeset: openstack-helm-single-node
     run: playbooks/osh-infra-keystone-k8s-auth.yaml

roles/deploy-kubeadm-aio-common/defaults/main.yml

@@ -50,3 +50,6 @@ nodes:
         value: enabled
       - name: ceph-mgr
         value: enabled
+
+gate_fqdn_test: false
+gate_fqdn_tld: openstackhelm.test

roles/deploy-kubeadm-aio-common/tasks/main.yaml

@@ -19,6 +19,7 @@
     playbook_user_dir: "{{ ansible_user_dir }}"
     kubernetes_default_device: "{{ ansible_default_ipv4.alias }}"
     kubernetes_default_address: null
+    primary_node_default_ip: "{{ hostvars[(groups['primary'][0])]['ansible_default_ipv4']['address'] }}"
 
 - name: if we have defined a custom interface for kubernetes use that
   when: kubernetes_network_default_device is defined and kubernetes_network_default_device

roles/deploy-kubeadm-aio-common/tasks/util-kubeadm-aio-run.yaml

@@ -52,6 +52,9 @@
           KUBELET_NODE_LABELS="{{ kubeadm_kubelet_labels }}"
           KUBE_SELF_HOSTED="{{ kubernetes_selfhosted }}"
           KUBE_KEYSTONE_AUTH="{{ kubernetes_keystone_auth }}"
+          GATE_FQDN_TEST="{{ gate_fqdn_test }}"
+          GATE_FQDN_TLD="{{ gate_fqdn_tld }}"
+          GATE_INGRESS_IP="{{ primary_node_default_ip }}"
       register: kubeadm_master_deploy
   rescue:
     - name: "getting logs for {{ kubeadm_aio_action }} action"

tools/gate/devel/local-vars.yaml

@@ -13,3 +13,4 @@
 # limitations under the License.
 
 kubernetes_network_default_device: docker0
+gate_fqdn_test: true

tools/images/kubeadm-aio/assets/entrypoint.sh

@@ -54,6 +54,9 @@ fi
 : ${KUBE_SELF_HOSTED:="false"}
 : ${KUBE_KEYSTONE_AUTH:="false"}
 : ${KUBELET_NODE_LABELS:=""}
+: ${GATE_FQDN_TEST:="false"}
+: ${GATE_INGRESS_IP:="127.0.0.1"}
+: ${GATE_FQDN_TLD:="openstackhelm.test"}
 
 PLAYBOOK_VARS="{
   \"my_container_name\": \"${CONTAINER_NAME}\",
@@ -88,6 +91,11 @@ PLAYBOOK_VARS="{
       \"podSubnet\": \"${KUBE_NET_POD_SUBNET}\",
       \"serviceSubnet\": \"${KUBE_NET_SUBNET_SUBNET}\"
     }
+  },
+  \"gate\": {
+    \"fqdn_testing\": \"${GATE_FQDN_TEST}\",
+    \"ingress_ip\": \"${GATE_INGRESS_IP}\",
+    \"fqdn_tld\": \"${GATE_FQDN_TLD}\"
   }
 }"
 

tools/images/kubeadm-aio/assets/opt/playbooks/roles/deploy-kubelet/tasks/kubelet.yaml

@@ -147,6 +147,31 @@
         enabled: yes
         masked: no
 
+- name: Setup DNS redirector for fqdn testing
+  # NOTE(portdirect): This must be done before the K8S DNS pods attempt to
+  # start, so they use the dnsmasq instance to resolve upstream hostnames
+  when: gate.fqdn_testing|bool == true
+  block:
+    - name: Setup DNS redirector | Remove std kubelet resolv.conf
+      file:
+        path: "/etc/kubernetes/kubelet-resolv.conf"
+        state: absent
+    - name: Setup DNS redirector | Populating new kubelet resolv.conf
+      copy:
+        dest: "/etc/kubernetes/kubelet-resolv.conf"
+        mode: 0640
+        content: |
+          nameserver 172.17.0.1
+    - name: Setup DNS redirector | Ensuring static manifests dir exists
+      file:
+        path: "/etc/kubernetes/manifests/"
+        state: directory
+    - name: Setup DNS redirector | Placing pod manifest on host
+      template:
+        src: osh-dns-redirector.yaml.j2
+        dest: /etc/kubernetes/manifests/osh-dns-redirector.yaml
+        mode: 0640
+
 - name: docker | ensure service is started and enabled
   when: kubelet.container_runtime == 'docker'
   systemd:

tools/images/kubeadm-aio/assets/opt/playbooks/roles/deploy-kubelet/templates/osh-dns-redirector.yaml.j2

@@ -0,0 +1,30 @@
+#jinja2: trim_blocks:False
+apiVersion: v1
+kind: Pod
+metadata:
+  name: osh-dns-redirector
+  namespace: kube-system
+spec:
+  hostNetwork: true
+  containers:
+    - name: osh-dns-redirector
+      image: docker.io/openstackhelm/neutron:newton
+      securityContext:
+        capabilities:
+          add:
+            - NET_ADMIN
+        runAsUser: 0
+      command:
+        - dnsmasq
+        - --keep-in-foreground
+        - --no-hosts
+        - --bind-interfaces
+        - --all-servers
+        {% for nameserver in external_dns_nameservers %}
+        - --server={{ nameserver }}
+        {% endfor %}
+        - --address
+        - /{{ gate.fqdn_tld }}/{{ gate.ingress_ip }}
+        # NOTE(portdirect): just listen on the docker0 interface
+        - --listen-address
+        - 172.17.0.1

tools/images/kubeadm-aio/assets/opt/playbooks/vars.yaml

@@ -47,3 +47,7 @@ all:
         dnsDomain: cluster.local
         podSubnet: 192.168.0.0/16
         serviceSubnet: 10.96.0.0/12
+    gate:
+      fqdn_testing: false
+      ingress_ip: 127.0.0.1
+      fqdn_tld: openstackhelm.test