From ce21f6e96d237720c95239da701d1f245383581e Mon Sep 17 00:00:00 2001 From: Pete Birley Date: Sun, 24 Jun 2018 16:21:59 -0500 Subject: [PATCH] Gate: Add support for testing fqdn over-rides in zuul This PS adds support for testing fqdn over-rides in zuul gates. When enabled it will direct requests to a configurable domain to the default ip of the primary node. Change-Id: I3d9a4a0bf06532caf0f544d44027493622f4ae5b Signed-off-by: Pete Birley --- .zuul.yaml | 5 ++++ .../defaults/main.yml | 3 ++ .../deploy-kubeadm-aio-common/tasks/main.yaml | 1 + .../tasks/util-kubeadm-aio-run.yaml | 3 ++ tools/gate/devel/local-vars.yaml | 1 + tools/images/kubeadm-aio/assets/entrypoint.sh | 8 +++++ .../roles/deploy-kubelet/tasks/kubelet.yaml | 25 ++++++++++++++++ .../templates/osh-dns-redirector.yaml.j2 | 30 +++++++++++++++++++ .../assets/opt/playbooks/vars.yaml | 4 +++ 9 files changed, 80 insertions(+) create mode 100644 tools/images/kubeadm-aio/assets/opt/playbooks/roles/deploy-kubelet/templates/osh-dns-redirector.yaml.j2 diff --git a/.zuul.yaml b/.zuul.yaml index 5397f639f..c40de931f 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -73,6 +73,10 @@ nodes: - name: primary label: ubuntu-xenial + groups: + - name: primary + nodes: + - primary - nodeset: name: openstack-helm-ubuntu @@ -260,6 +264,7 @@ vars: zuul_osh_relative_path: ../openstack-helm/ kubernetes_keystone_auth: true + gate_fqdn_test: true parent: openstack-helm-infra nodeset: openstack-helm-single-node run: playbooks/osh-infra-keystone-k8s-auth.yaml diff --git a/roles/deploy-kubeadm-aio-common/defaults/main.yml b/roles/deploy-kubeadm-aio-common/defaults/main.yml index fd1cbf07b..dc5121ef8 100644 --- a/roles/deploy-kubeadm-aio-common/defaults/main.yml +++ b/roles/deploy-kubeadm-aio-common/defaults/main.yml @@ -50,3 +50,6 @@ nodes: value: enabled - name: ceph-mgr value: enabled + +gate_fqdn_test: false +gate_fqdn_tld: openstackhelm.test diff --git a/roles/deploy-kubeadm-aio-common/tasks/main.yaml b/roles/deploy-kubeadm-aio-common/tasks/main.yaml index ed9a9d26c..9a75dc55e 100644 --- a/roles/deploy-kubeadm-aio-common/tasks/main.yaml +++ b/roles/deploy-kubeadm-aio-common/tasks/main.yaml @@ -19,6 +19,7 @@ playbook_user_dir: "{{ ansible_user_dir }}" kubernetes_default_device: "{{ ansible_default_ipv4.alias }}" kubernetes_default_address: null + primary_node_default_ip: "{{ hostvars[(groups['primary'][0])]['ansible_default_ipv4']['address'] }}" - name: if we have defined a custom interface for kubernetes use that when: kubernetes_network_default_device is defined and kubernetes_network_default_device diff --git a/roles/deploy-kubeadm-aio-common/tasks/util-kubeadm-aio-run.yaml b/roles/deploy-kubeadm-aio-common/tasks/util-kubeadm-aio-run.yaml index 8b1296ffc..af4819d4c 100644 --- a/roles/deploy-kubeadm-aio-common/tasks/util-kubeadm-aio-run.yaml +++ b/roles/deploy-kubeadm-aio-common/tasks/util-kubeadm-aio-run.yaml @@ -52,6 +52,9 @@ KUBELET_NODE_LABELS="{{ kubeadm_kubelet_labels }}" KUBE_SELF_HOSTED="{{ kubernetes_selfhosted }}" KUBE_KEYSTONE_AUTH="{{ kubernetes_keystone_auth }}" + GATE_FQDN_TEST="{{ gate_fqdn_test }}" + GATE_FQDN_TLD="{{ gate_fqdn_tld }}" + GATE_INGRESS_IP="{{ primary_node_default_ip }}" register: kubeadm_master_deploy rescue: - name: "getting logs for {{ kubeadm_aio_action }} action" diff --git a/tools/gate/devel/local-vars.yaml b/tools/gate/devel/local-vars.yaml index efdbfaeeb..cc94aff20 100644 --- a/tools/gate/devel/local-vars.yaml +++ b/tools/gate/devel/local-vars.yaml @@ -13,3 +13,4 @@ # limitations under the License. kubernetes_network_default_device: docker0 +gate_fqdn_test: true diff --git a/tools/images/kubeadm-aio/assets/entrypoint.sh b/tools/images/kubeadm-aio/assets/entrypoint.sh index 5fbcbb073..5c4a1047d 100755 --- a/tools/images/kubeadm-aio/assets/entrypoint.sh +++ b/tools/images/kubeadm-aio/assets/entrypoint.sh @@ -54,6 +54,9 @@ fi : ${KUBE_SELF_HOSTED:="false"} : ${KUBE_KEYSTONE_AUTH:="false"} : ${KUBELET_NODE_LABELS:=""} +: ${GATE_FQDN_TEST:="false"} +: ${GATE_INGRESS_IP:="127.0.0.1"} +: ${GATE_FQDN_TLD:="openstackhelm.test"} PLAYBOOK_VARS="{ \"my_container_name\": \"${CONTAINER_NAME}\", @@ -88,6 +91,11 @@ PLAYBOOK_VARS="{ \"podSubnet\": \"${KUBE_NET_POD_SUBNET}\", \"serviceSubnet\": \"${KUBE_NET_SUBNET_SUBNET}\" } + }, + \"gate\": { + \"fqdn_testing\": \"${GATE_FQDN_TEST}\", + \"ingress_ip\": \"${GATE_INGRESS_IP}\", + \"fqdn_tld\": \"${GATE_FQDN_TLD}\" } }" diff --git a/tools/images/kubeadm-aio/assets/opt/playbooks/roles/deploy-kubelet/tasks/kubelet.yaml b/tools/images/kubeadm-aio/assets/opt/playbooks/roles/deploy-kubelet/tasks/kubelet.yaml index caa550378..05f21e729 100644 --- a/tools/images/kubeadm-aio/assets/opt/playbooks/roles/deploy-kubelet/tasks/kubelet.yaml +++ b/tools/images/kubeadm-aio/assets/opt/playbooks/roles/deploy-kubelet/tasks/kubelet.yaml @@ -147,6 +147,31 @@ enabled: yes masked: no +- name: Setup DNS redirector for fqdn testing + # NOTE(portdirect): This must be done before the K8S DNS pods attempt to + # start, so they use the dnsmasq instance to resolve upstream hostnames + when: gate.fqdn_testing|bool == true + block: + - name: Setup DNS redirector | Remove std kubelet resolv.conf + file: + path: "/etc/kubernetes/kubelet-resolv.conf" + state: absent + - name: Setup DNS redirector | Populating new kubelet resolv.conf + copy: + dest: "/etc/kubernetes/kubelet-resolv.conf" + mode: 0640 + content: | + nameserver 172.17.0.1 + - name: Setup DNS redirector | Ensuring static manifests dir exists + file: + path: "/etc/kubernetes/manifests/" + state: directory + - name: Setup DNS redirector | Placing pod manifest on host + template: + src: osh-dns-redirector.yaml.j2 + dest: /etc/kubernetes/manifests/osh-dns-redirector.yaml + mode: 0640 + - name: docker | ensure service is started and enabled when: kubelet.container_runtime == 'docker' systemd: diff --git a/tools/images/kubeadm-aio/assets/opt/playbooks/roles/deploy-kubelet/templates/osh-dns-redirector.yaml.j2 b/tools/images/kubeadm-aio/assets/opt/playbooks/roles/deploy-kubelet/templates/osh-dns-redirector.yaml.j2 new file mode 100644 index 000000000..e3a7b7c61 --- /dev/null +++ b/tools/images/kubeadm-aio/assets/opt/playbooks/roles/deploy-kubelet/templates/osh-dns-redirector.yaml.j2 @@ -0,0 +1,30 @@ +#jinja2: trim_blocks:False +apiVersion: v1 +kind: Pod +metadata: + name: osh-dns-redirector + namespace: kube-system +spec: + hostNetwork: true + containers: + - name: osh-dns-redirector + image: docker.io/openstackhelm/neutron:newton + securityContext: + capabilities: + add: + - NET_ADMIN + runAsUser: 0 + command: + - dnsmasq + - --keep-in-foreground + - --no-hosts + - --bind-interfaces + - --all-servers + {% for nameserver in external_dns_nameservers %} + - --server={{ nameserver }} + {% endfor %} + - --address + - /{{ gate.fqdn_tld }}/{{ gate.ingress_ip }} + # NOTE(portdirect): just listen on the docker0 interface + - --listen-address + - 172.17.0.1 diff --git a/tools/images/kubeadm-aio/assets/opt/playbooks/vars.yaml b/tools/images/kubeadm-aio/assets/opt/playbooks/vars.yaml index 814150776..c50424165 100644 --- a/tools/images/kubeadm-aio/assets/opt/playbooks/vars.yaml +++ b/tools/images/kubeadm-aio/assets/opt/playbooks/vars.yaml @@ -47,3 +47,7 @@ all: dnsDomain: cluster.local podSubnet: 192.168.0.0/16 serviceSubnet: 10.96.0.0/12 + gate: + fqdn_testing: false + ingress_ip: 127.0.0.1 + fqdn_tld: openstackhelm.test