ensure-kubernetes: add microk8s support

Add microk8s support to the ensure-kubernetes role.  This installs via
a snap, and is currently only implemented for Ubuntu Jammy.

Mostly this is a straight-forward installation.  I did notice though
it needs a little more time to be stable in the test, so the timeout
is bumped slightly.

microk8s is the Ubuntu "blessed" way of doing things.  This should be
a better choice for Ubuntu platforms, because minikube is tightly tied
to cri-o, which is only packaged through kubic, which is currently in
some sort of deprecated but best-effort supported mode [1].  This was
inspired by an outage where the kubic gpg expired.  This appears fixed
now.

[1] https://kubic.opensuse.org/blog/2022-06-10-kubic-retired/

Change-Id: Id3e31c70a35dde218e35e7c50964f8a3c0348150

roles/ensure-kubernetes/README.rst

@@ -2,6 +2,25 @@ An ansible role to install kubernetes.
 
 **Role Variables**
 
+.. zuul:rolevar:: ensure_kubernetes_type
+   :default: minikube
+
+   The kubernetes distribution to use.  Currently ```minikube`` or
+   ```microk8s```.  Note that ```microk8s``` is only implemented for
+   Ubuntu Jammy distributions currently.
+
+.. zuul:rolevar:: ensure_kubernetes_microk8s_channel
+   :default: latest/stable
+
+   The ``snap`` channel to use for ```microk8s```.  See
+   `<https://microk8s.io/docs/setting-snap-channel>`__.
+
+.. zuul:rolevar:: ensure_kubernetes_microk8s_addons
+   :default: ['dns', 'storage']
+
+   The addons for ``microk8s```.  See
+   `<https://microk8s.io/docs/addons>`__
+
 .. zuul:rolevar:: install_kubernetes_with_cluster
    :default: True
 
@@ -21,10 +40,10 @@ An ansible role to install kubernetes.
 .. zuul:rolevar:: kubernetes_runtime
    :default: docker
 
-   Which kubernetes runtime to use; values are ``docker`` or
+   Which kubernetes runtime to use for minikube; values are ``docker`` or
    ``cri-o``.
 
 .. zuul:rolevar:: ensure_kubernetes_minikube_addons
    :default: []
 
-   List of addons to configure in k8s. Use this to enable the addons.
+   List of addons to configure in k8s. Use this to enable the addons.

roles/ensure-kubernetes/defaults/main.yaml

@@ -1,3 +1,12 @@
+ensure_kubernetes_type: minikube
+ensure_kubernetes_microk8s_channel: 'latest/stable'
+# NOTE(ianw) : 2022-12-13
+#  - "storage" is deprecated and has become "hostpath-storage" in
+#     >1.24, but we still need to support 1.23.  If it really goes away
+#     we might need to switch on the channel name
+ensure_kubernetes_microk8s_addons:
+  - dns
+  - storage
 install_kubernetes_with_cluster: True
 minikube_version: latest
 minikube_dns_resolvers: []

roles/ensure-kubernetes/tasks/main.yaml

@@ -1,21 +1,28 @@
-- name: Add all repositories
+- name: Install microk8s
-  include_role:
+  when: ensure_kubernetes_type == 'microk8s'
-    name: ensure-package-repositories
+  include_tasks: microk8s.yaml
-  vars:
-    repositories_keys:
-      - url: https://packages.cloud.google.com/apt/doc/apt-key.gpg
-    repositories_list:
-      - repo: deb http://apt.kubernetes.io/ kubernetes-xenial main
 
-- name: Install kubernetes packages and dependencies
+- name: Install minikube
-  become: yes
+  when: ensure_kubernetes_type == 'minikube'
-  apt:
+  block:
-    name:
+    - name: Add all repositories
-      - conntrack
+      include_role:
-      - kubectl
+        name: ensure-package-repositories
-      - liblz4-tool
+      vars:
-    update_cache: yes
+        repositories_keys:
+          - url: https://packages.cloud.google.com/apt/doc/apt-key.gpg
+        repositories_list:
+          - repo: deb http://apt.kubernetes.io/ kubernetes-xenial main
 
-- name: Run cluster tasks
+    - name: Install kubernetes packages and dependencies
-  include_tasks: minikube.yaml
+      become: yes
-  when: install_kubernetes_with_cluster
+      apt:
+        name:
+          - conntrack
+          - kubectl
+          - liblz4-tool
+        update_cache: yes
+
+    - name: Run cluster tasks
+      include_tasks: minikube.yaml
+      when: install_kubernetes_with_cluster

roles/ensure-kubernetes/tasks/microk8s.yaml

@@ -0,0 +1,58 @@
+- name: Check distro
+  assert:
+    that: ansible_distribution_release == 'jammy'
+    msg: 'This role only supported on Jammy'
+
+- name: Install snapd
+  become: yes
+  package:
+    name: snapd
+    state: present
+
+- name: Install microk8s snap
+  become: yes
+  command: 'snap install microk8s --classic --channel={{ ensure_kubernetes_microk8s_channel }}'
+
+- name: Install add-ons
+  become: yes
+  command: '/snap/bin/microk8s enable {{ ensure_kubernetes_microk8s_addons | join(" ") }}'
+
+- name: Install kubectl wrapper
+  become: yes
+  copy:
+    content: |
+      #!/bin/bash
+      /snap/bin/microk8s.kubectl "$@"
+    dest: /usr/local/bin/kubectl
+    owner: root
+    group: root
+    mode: 0755
+
+- name: Allow zuul user kubectl access
+  become: yes
+  user:
+    name: 'zuul'
+    groups: 'microk8s'
+    append: yes
+
+- name: Create .kube directory
+  file:
+    path: "{{ ansible_user_dir }}/.kube"
+    state: directory
+    mode: 0755
+
+- name: Populate kube config
+  become: yes
+  shell: /snap/bin/microk8s config > {{ ansible_user_dir }}/.kube/config
+
+- name: Ensure we can read config
+  become: yes
+  file:
+    path: "{{ ansible_user_dir }}/.kube/config"
+    owner: zuul
+    group: zuul
+    state: touch
+    mode: 0644
+
+- name: Reset connection to pickup group
+  meta: reset_connection

test-playbooks/ensure-kubernetes/microk8s.yaml

@@ -0,0 +1,6 @@
+- hosts: all
+  name: Install kubernetes with minikube
+  roles:
+    - role: ensure-kubernetes
+      vars:
+        ensure_kubernetes_type: microk8s

test-playbooks/ensure-kubernetes/post.yaml

@@ -29,12 +29,19 @@
         - name: Start pod
           command: kubectl apply -f test-pod.yaml
 
+        - name: Wait a bit
+          pause:
+            seconds: 30
+
+        - name: Describe pod
+          shell: sleep 5; kubectl describe pods test
+
         - name: Ensure pod is running
           shell: sleep 5; kubectl get pods
           register: _get_pods_output
           until: "'Running' in _get_pods_output.stdout"
           retries: 3
-          delay: 5
+          delay: 10
 
       always:
         - name: Collect container logs
@@ -47,6 +54,6 @@
 
         - name: Get minikube logs
           become: true
-          shell: "/tmp/minikube logs > {{ ansible_user_dir }}/zuul-output/logs/minikube.txt"
+          shell: "/tmp/minikube logs > {{ ansible_user_dir }}/zuul-output/logs/minikube.txt || true"
           environment:
             MINIKUBE_HOME: "{{ ansible_user_dir }}"

zuul-tests.d/container-roles-jobs.yaml

@@ -392,6 +392,34 @@
         - name: ubuntu-jammy
           label: ubuntu-jammy
 
+- job:
+    name: zuul-jobs-test-ensure-kubernetes-microk8s
+    description: |
+      Test the ensure-kubernetes role with microk8s
+
+      This job tests changes to the ensure-kubernetes roles.  It
+      is not meant to be used directly but rather run on changes to
+      roles in the zuul-jobs repo.
+    abstract: true
+    files:
+      - roles/ensure-docker/.*
+      - roles/ensure-kubernetes/.*
+      - roles/ensure-package-repositories/.*
+      - test-playbooks/ensure-kubernetes/.*
+    run: test-playbooks/ensure-kubernetes/microk8s.yaml
+    post-run: test-playbooks/ensure-kubernetes/post.yaml
+
+# NOTE(ianw) 2022-12-08 : only supported on jammy
+
+- job:
+    name: zuul-jobs-test-ensure-kubernetes-microk8s-ubuntu-jammy
+    description: Test the ensure-kubernetes role with docker on ubuntu-jammy
+    parent: zuul-jobs-test-ensure-kubernetes-microk8s
+    nodeset:
+      nodes:
+        - name: ubuntu-jammy
+          label: ubuntu-jammy
+
 - job:
     name: zuul-jobs-test-ensure-podman
     description: |
@@ -532,6 +560,7 @@
         - zuul-jobs-test-ensure-kubernetes-crio-ubuntu-bionic
         - zuul-jobs-test-ensure-kubernetes-crio-ubuntu-focal
         - zuul-jobs-test-ensure-kubernetes-crio-ubuntu-jammy
+        - zuul-jobs-test-ensure-kubernetes-microk8s-ubuntu-jammy
         - zuul-jobs-test-ensure-skopeo-ubuntu-bionic
         - zuul-jobs-test-ensure-skopeo-ubuntu-focal
         - zuul-jobs-test-ensure-skopeo-ubuntu-jammy
@@ -563,6 +592,7 @@
         - zuul-jobs-test-ensure-kubernetes-crio-ubuntu-bionic
         - zuul-jobs-test-ensure-kubernetes-crio-ubuntu-focal
         - zuul-jobs-test-ensure-kubernetes-crio-ubuntu-jammy
+        - zuul-jobs-test-ensure-kubernetes-microk8s-ubuntu-jammy
         - zuul-jobs-test-ensure-skopeo-ubuntu-bionic
         - zuul-jobs-test-ensure-skopeo-ubuntu-focal
         - zuul-jobs-test-ensure-skopeo-ubuntu-jammy