openstack-helm/tools/gate/playbooks/deploy-k8s.yaml

- hosts: all
  become: true
  gather_facts: true
  roles:
    - clear-firewall
  tasks:
    - name: Load necessary modules
      modprobe:
        name: "{{ item }}"
        state: present
      with_items:
        - overlay
        - br_netfilter

    - name: Configure sysctl
      sysctl:
        name: "{{ item }}"
        value: "1"
        state: present
      loop:
        - net.ipv6.conf.default.disable_ipv6
        - net.ipv6.conf.all.disable_ipv6
        - net.ipv6.conf.lo.disable_ipv6
        - net.bridge.bridge-nf-call-iptables
        - net.bridge.bridge-nf-call-ip6tables
        - net.ipv4.ip_forward
      ignore_errors: true

    - name: Remove swapfile from /etc/fstab
      mount:
        name: "{{ item }}"
        fstype: swap
        state: absent
      with_items:
        - swap
        - none

    - name: Disable swap
      command: swapoff -a
      when: ansible_swaptotal_mb > 0

    - name: Ensure dependencies are installed
      apt:
        name:
          - apt-transport-https
          - ca-certificates
          - gnupg2
          - ipvsadm
          - jq
        state: present

    - name: Add Kubernetes apt repository key
      apt_key:
        url: https://packages.cloud.google.com/apt/doc/apt-key.gpg
        state: present

    - name: Add Kubernetes apt repository
      apt_repository:
        repo: deb https://apt.kubernetes.io/ kubernetes-xenial main
        state: present
        filename: kubernetes.list

    - name: Install Kubernetes binaries
      apt:
        state: present
        update_cache: true
        allow_downgrade: true
        pkg:
          - "kubelet={{ kube_version }}"
          - "kubeadm={{ kube_version }}"
          - "kubectl={{ kube_version }}"

    - name: Restart kubelet
      service:
        name: kubelet
        daemon_reload: yes
        state: restarted

    - name: Disable systemd-resolved
      service:
        name: systemd-resolved
        enabled: false
        state: stopped

    - name: Configure resolv.conf
      copy:
        src: files/resolv.conf
        dest: "{{ item }}"
      loop:
        - /etc/resolv.conf
        - /run/systemd/resolve/resolv.conf

- hosts: primary
  become: true
  tasks:
    - name: Mount tmpfs to /var/lib/etcd
      mount:
        path: /var/lib/etcd
        src: tmpfs
        fstype: tmpfs
        opts: size=1g
        state: mounted

    - name: Prepare kubeadm config
      copy:
        src: files/kubeadm_config.yaml
        dest: /tmp/kubeadm_config.yaml

    - name: Initialize the Kubernetes cluster using kubeadm
      command: kubeadm init --config /tmp/kubeadm_config.yaml

    - name: Setup kubeconfig for zuul user
      shell: |
        mkdir -p /home/zuul/.kube
        cp -i /etc/kubernetes/admin.conf /home/zuul/.kube/config
        chown zuul:zuul /home/zuul/.kube/config        
      args:
        executable: /bin/bash

- hosts: all
  become: true
  tasks:
    # We download Calico manifest on all nodes because we then want to download
    # Calico images BEFORE deploying it
    - name: Download Calico manifest
      shell: |
        curl -LSs https://docs.projectcalico.org/archive/{{ calico_version }}/manifests/calico.yaml -o /tmp/calico.yaml
        sed -i -e 's#docker.io/calico/#quay.io/calico/#g' /tmp/calico.yaml        
      args:
        executable: /bin/bash

    # Download images needed for calico before applying manifests, so that `kubectl wait` timeout
    # for `k8s-app=kube-dns` isn't reached by slow download speeds
    - name: Download Calico images
      shell: |
        export CONTAINER_RUNTIME_ENDPOINT=unix:///run/containerd/containerd.sock
        export IMAGE_SERVICE_ENDPOINT=unix:///run/containerd/containerd.sock
        awk '/image:/ { print $2 }' /tmp/calico.yaml | xargs -I{} crictl pull {}        
      args:
        executable: /bin/bash

- hosts: primary
  tasks:
    - name: Deploy Calico
      command: kubectl apply -f /tmp/calico.yaml

    - name: Wait for Calico pods ready
      command: kubectl -n kube-system wait --timeout=240s --for=condition=Ready pods -l k8s-app=calico-node

    - name: Prepare Calico patch
      copy:
        src: files/calico_patch.yaml
        dest: /tmp/calico_patch.yaml

    - name: Patch Calico
      command: kubectl -n kube-system patch daemonset calico-node --patch-file /tmp/calico_patch.yaml

    - name: Wait for Calico pods ready
      command: kubectl -n kube-system wait --timeout=240s --for=condition=Ready pods -l k8s-app=calico-node

    - name: Generate join command
      command: kubeadm token create --print-join-command
      register: join_command

- hosts: nodes
  become: true
  tasks:
    - name: Join node to cluster
      command: "{{ hostvars['primary']['join_command'].stdout_lines[0] }}"

- hosts: primary
  tasks:
    - name: Wait for Calico pods ready
      command: kubectl -n kube-system wait --timeout=240s --for=condition=Ready pods -l k8s-app=calico-node

    - name: Wait for Coredns pods ready
      command: kubectl -n kube-system wait --timeout=240s --for=condition=Ready pods -l k8s-app=kube-dns

    - name: Untaint Kubernetes control plane node
      command: kubectl taint nodes -l 'node-role.kubernetes.io/control-plane' node-role.kubernetes.io/control-plane-

- hosts: all
  become: true
  tasks:
    - name: Add coredns to /etc/resolv.conf
      lineinfile:
        line: nameserver 10.96.0.10
        path: /etc/resolv.conf
        state: present
        insertbefore: "BOF"

- hosts: primary
  tasks:
    - name: Enable recursive queries for coredns
      shell: |
        PATCH=$(mktemp)
        kubectl get configmap coredns -n kube-system -o json | jq -r "{data: .data}"  | sed 's/ready\\n/header \{\\n        response set ra\\n    \}\\n    ready\\n/g' > "${PATCH}"
        kubectl patch configmap coredns -n kube-system --patch-file "${PATCH}"
        kubectl set image deployment coredns -n kube-system "coredns=registry.k8s.io/coredns/coredns:v1.9.4"
        kubectl rollout restart -n kube-system deployment/coredns
        rm -f "${PATCH}"        
      args:
        executable: /bin/bash