From 5c089bf03214239b50da04365cca2b08039d8172 Mon Sep 17 00:00:00 2001
From: Clark Boylan <clark.boylan@gmail.com>
Date: Wed, 21 Aug 2019 09:36:27 -0700
Subject: [PATCH] Allow for overriding dns resolvers in install-kubernetes

By default kubernetes wants to use the local resolvers on the host
system. But in many cases the local resolver may be 127.0.0.1 which is a
different localhost on the host system then in the container network
namespaces. To address this disconnect you can supply a different
resolv.conf with a list of nameservers to use instead. Add support for
this.

Note that the test currently only uses ipv4 resolvers because by default
k8s has no support for ipv6.

Change-Id: I3c0d7d88613e2c9112e809f036aef3dfeaf7ab72
---
 roles/install-kubernetes/README.rst           |  6 ++++++
 roles/install-kubernetes/defaults/main.yaml   |  1 +
 roles/install-kubernetes/tasks/minikube.yaml  | 18 +++++++++++++++++-
 .../templates/resolv.conf.j2                  |  3 +++
 test-playbooks/install-kubernetes.yaml        |  8 ++++++++
 zuul-tests.d/container-roles-jobs.yaml        | 19 +++++++++++++++++++
 6 files changed, 54 insertions(+), 1 deletion(-)
 create mode 100644 roles/install-kubernetes/templates/resolv.conf.j2
 create mode 100644 test-playbooks/install-kubernetes.yaml

diff --git a/roles/install-kubernetes/README.rst b/roles/install-kubernetes/README.rst
index a48798923..b5567959e 100644
--- a/roles/install-kubernetes/README.rst
+++ b/roles/install-kubernetes/README.rst
@@ -11,3 +11,9 @@ An ansible role to install kubernetes.
    :default: latest
 
    The version of Minikube to install.
+
+.. zuul:rolevar:: minikube_dns_resolvers
+   :default: []
+
+   List of dns resolvers to configure in k8s. Use this to override the
+   resolvers that are found by default.
diff --git a/roles/install-kubernetes/defaults/main.yaml b/roles/install-kubernetes/defaults/main.yaml
index 6040df0d3..158b030e1 100644
--- a/roles/install-kubernetes/defaults/main.yaml
+++ b/roles/install-kubernetes/defaults/main.yaml
@@ -1,2 +1,3 @@
 install_kubernetes_with_cluster: True
 minikube_version: latest
+minikube_dns_resolvers: []
diff --git a/roles/install-kubernetes/tasks/minikube.yaml b/roles/install-kubernetes/tasks/minikube.yaml
index 425c001c8..75672618b 100644
--- a/roles/install-kubernetes/tasks/minikube.yaml
+++ b/roles/install-kubernetes/tasks/minikube.yaml
@@ -32,9 +32,25 @@
     state: directory
     mode: 0755
 
+- name: Default args
+  set_fact:
+    minikube_args: ""
+
+- name: Configure dns options if set
+  block:
+    - name: Write resolv.conf
+      template:
+        src: resolv.conf.j2
+        dest: "{{ ansible_user_dir }}/.minikube/k8s_resolv.conf"
+        mode: "0444"
+    - name: Set extra kube setttings
+      set_fact:
+        minikube_args: "--extra-config=kubelet.resolv-conf={{ ansible_user_dir }}/.minikube/k8s_resolv.conf"
+  when: minikube_dns_resolvers
+
 - name: Start Minikube
   become: yes
-  command: /tmp/minikube --vm-driver=none start
+  command: "/tmp/minikube --vm-driver=none start {{ minikube_args }}"
   environment:
     MINIKUBE_WANTUPDATENOTIFICATION: false
     MINIKUBE_WANTREPORTERRORPROMPT: false
diff --git a/roles/install-kubernetes/templates/resolv.conf.j2 b/roles/install-kubernetes/templates/resolv.conf.j2
new file mode 100644
index 000000000..657fc0dd7
--- /dev/null
+++ b/roles/install-kubernetes/templates/resolv.conf.j2
@@ -0,0 +1,3 @@
+{% for x in minikube_dns_resolvers %}
+nameserver {{ x }}
+{% endfor %}
diff --git a/test-playbooks/install-kubernetes.yaml b/test-playbooks/install-kubernetes.yaml
new file mode 100644
index 000000000..a348350a2
--- /dev/null
+++ b/test-playbooks/install-kubernetes.yaml
@@ -0,0 +1,8 @@
+- hosts: primary
+  name: Install kubernetes with minikube
+  roles:
+    - role: install-kubernetes
+      vars:
+        minikube_dns_resolvers:
+          - '1.1.1.1'
+          - '8.8.8.8'
diff --git a/zuul-tests.d/container-roles-jobs.yaml b/zuul-tests.d/container-roles-jobs.yaml
index 729be52c3..0937d3bf6 100644
--- a/zuul-tests.d/container-roles-jobs.yaml
+++ b/zuul-tests.d/container-roles-jobs.yaml
@@ -24,11 +24,30 @@
         - name: builder
           label: ubuntu-bionic
 
+- job:
+    name: zuul-jobs-test-install-kubernetes
+    description: |
+      Test the install-kubernetes role
+
+      This job tests changes to the install-kubernetes roles.  It
+      is not meant to be used directly but rather run on changes to
+      roles in the zuul-jobs repo.
+    files:
+      - roles/install-docker/.*
+      - roles/install-kubernetes/.*
+      - test-playbooks/install-kubernetes.yaml
+    run: test-playbooks/install-kubernetes.yaml
+    nodeset:
+      nodes:
+        - name: primary
+          label: ubuntu-bionic
+
 # List all the jobs in this file.
 
 - project:
     check:
       jobs: &id001
         - zuul-jobs-test-registry
+        - zuul-jobs-test-install-kubernetes
     gate:
       jobs: *id001