Add static node to functional test

So that we can better test functions related to ssh connectivity,
add a static node to the functional test.

Change-Id: I3dfdaca0b4e5dbfb35e4a645c7bcb56e0eec88c2

playbooks/zuul-operator-functional/run.yaml

@@ -5,6 +5,7 @@
     # locally for development.
     install_operator: true
     zuul_work_dir: "{{ zuul.projects['opendev.org/zuul/zuul-operator'].src_dir }}"
+    runtime: minikube
   tasks:
     - name: Setup CRD
       command: kubectl apply -f deploy/crds/zuul-ci_v1alpha2_zuul_crd.yaml -f deploy/rbac.yaml
@@ -17,6 +18,9 @@
       args:
         chdir: "{{ zuul_work_dir }}"
 
+    - name: Create static node
+      include_tasks: ./tasks/create_static_node.yaml
+
     - name: Create required secret
       include_tasks: ./tasks/create_test_secrets.yaml
 

playbooks/zuul-operator-functional/static-node/Dockerfile

@@ -0,0 +1,29 @@
+FROM ubuntu:20.04
+
+RUN apt-get update \
+    && DEBIAN_FRONTEND="noninteractive" apt-get -y install \
+        git \
+        openssh-server \
+        rsync \
+    && rm -rf /var/lib/apt/lists/*
+
+# If proxy environment variables supplied during build allow pass through via
+# ~/.ssh/environment, as the test env likely will require these set for any
+# job launched on the node to be capable of internet access.
+RUN set -e ; \
+    mkdir /var/run/sshd ; \
+    mkdir -p -m 0700 ~/.ssh; \
+    if [ -n "${http_proxy}" ]; then \
+        sed -ri 's/#PermitUserEnvironment no/PermitUserEnvironment yes/g' /etc/ssh/sshd_config; \
+        echo "http_proxy=${http_proxy}" > ~/.ssh/environment; \
+        echo "https_proxy=${https_proxy}" >> ~/.ssh/environment; \
+        echo "no_proxy=${no_proxy}" >> ~/.ssh/environment; \
+    fi \
+    ;
+
+COPY --chown=root:root ./ssh_host_ed25519_key /etc/ssh/ssh_host_ed25519_key
+RUN chmod 0600 /etc/ssh/ssh_host_ed25519_key
+
+EXPOSE 22
+
+ENTRYPOINT ["/usr/sbin/sshd", "-D" ]

playbooks/zuul-operator-functional/static-node/ssh_host_ed25519_key

@@ -0,0 +1,7 @@
+-----BEGIN OPENSSH PRIVATE KEY-----
+b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW
+QyNTUxOQAAACDoByWHoyDSCs1IrkidjDfLj3PnXxa2fSR9YS1kvICPoAAAAJiZWqEimVqh
+IgAAAAtzc2gtZWQyNTUxOQAAACDoByWHoyDSCs1IrkidjDfLj3PnXxa2fSR9YS1kvICPoA
+AAAEA9aXkLh3eloH1HMQ2RR3DQ2bzIMROVxkvKKDmeYsDlLegHJYejINIKzUiuSJ2MN8uP
+c+dfFrZ9JH1hLWS8gI+gAAAAEXJvb3RAMzI5NmRjMDg4ODQ2AQIDBA==
+-----END OPENSSH PRIVATE KEY-----

playbooks/zuul-operator-functional/static-node/ssh_host_ed25519_key.pub

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOgHJYejINIKzUiuSJ2MN8uPc+dfFrZ9JH1hLWS8gI+g root@3296dc088846

playbooks/zuul-operator-functional/tasks/create_config.yaml

@@ -23,32 +23,60 @@
               - name: container
                 label: pod-fedora-32
 
+        - nodeset:
+            name: ubuntu-focal
+            nodes:
+              - name: ubuntu-focal
+                label: ubuntu-focal
+
         - job:
-            name: test-job
-            run: test.yaml
+            name: test-pod
+            run: test-pod.yaml
             # skip zuul-base-jobs for now as it doesn't work with kubectl
             roles:
               - zuul: zuul/zuul-jobs
             parent: null
             nodeset: pod-fedora
 
+        - job:
+            name: test-static-node
+            run: test-static-node.yaml
+            roles:
+              - zuul: zuul/zuul-jobs
+            parent: null
+            nodeset: ubuntu-focal
+
         - project:
             periodic:
               jobs:
-                - test-job
+                - test-pod
+                - test-static-node
 
-    - name: test.yaml
+    - name: test-pod.yaml
       content: |
         - hosts: localhost
           tasks:
-            - include_role:
-                name: start-zuul-console
-            - debug: msg='Demo job is running'
             - name: check job volume
               stat:
                 path: /system-dbus
               register: _job_volume
               failed_when: not _job_volume.stat.isdir
+        - hosts: container
+          tasks:
+            - include_role:
+                name: start-zuul-console
+            - debug: msg='Demo job is running on container'
+            - shell: id
+            - pause: seconds=30
+
+    - name: test-static-node.yaml
+      content: |
+        - hosts: all
+          tasks:
+            - include_role:
+                name: start-zuul-console
+            - debug: msg='Demo job is running on static node'
+            - shell: id
             - pause: seconds=30
 
 - name: commit config

playbooks/zuul-operator-functional/tasks/create_static_node.yaml

@@ -0,0 +1,74 @@
+- name: Create static node image
+  when: "runtime == 'minikube'"
+  args:
+    chdir: "{{ zuul_work_dir }}/playbooks/zuul-operator-functional/static-node"
+  shell: |
+    /tmp/minikube image build . -t static-node
+
+- name: Create static node image
+  when: "runtime == 'kind'"
+  args:
+    chdir: "{{ zuul_work_dir }}/playbooks/zuul-operator-functional/static-node"
+  shell: |
+    docker build . -t static-node
+    kind load docker-image static-node
+
+- name: Run static node
+  k8s:
+    namespace: "default"
+    definition:
+      apiVersion: v1
+      kind: Pod
+      metadata:
+        name: node
+        labels:
+          app: static-node
+      spec:
+        containers:
+          - name: node
+            image: static-node
+            imagePullPolicy: Never
+            lifecycle:
+              postStart:
+                exec:
+                  command:
+                    - bash
+                    - -c
+                    - mkdir -p /root/.ssh && chmod 0600 /root/.ssh && cp /sshkeys/authorized_keys /root/.ssh
+            ports:
+              - name: ssh
+                containerPort: 22
+                protocol: TCP
+              - name: zuul
+                containerPort: 19885
+                protocol: TCP
+            volumeMounts:
+              - name: authorized-keys
+                mountPath: /sshkeys
+                readOnly: true
+        volumes:
+          - name: authorized-keys
+            secret:
+              secretName: static-node-authorized-keys
+              defaultMode: 0600
+
+- name: Create static node service
+  k8s:
+    namespace: "default"
+    definition:
+      apiVersion: v1
+      kind: Service
+      metadata:
+        name: node
+      spec:
+        selector:
+          app: static-node
+        ports:
+          - name: ssh
+            port: 22
+            targetPort: 22
+            protocol: TCP
+          - name: zuul
+            port: 19885
+            targetPort: 19885
+            protocol: TCP

playbooks/zuul-operator-functional/tasks/create_test_secrets.yaml

@@ -8,6 +8,10 @@
   command: cat id_rsa
   register: _ssh_key
 
+- name: Read generated ssh public key
+  command: cat id_rsa.pub
+  register: _ssh_key_pub
+
 - name: Read kube server address
   command: kubectl config view -o jsonpath='{.clusters[0].cluster.server}'
   register: _kube_config
@@ -30,6 +34,10 @@
         name: "{{ item.name }}"
       stringData: "{{ item.data }}"
   loop:
+    - name: static-node-authorized-keys
+      data:
+        authorized_keys: "{{ _ssh_key_pub.stdout }}"
+
     - name: executor-ssh-key
       data:
         sshkey: "{{ _ssh_key.stdout }}"
@@ -52,6 +60,8 @@
           labels:
             - name: pod-fedora-32
               min-ready: 1
+            - name: ubuntu-focal
+              min-ready: 1
           providers:
             - name: kube-cluster
               driver: kubernetes
@@ -64,6 +74,16 @@
                     type: pod
                     image: docker.io/fedora:32
                     python-path: /bin/python3
+            - name: static-vms
+              driver: static
+              pools:
+                - name: main
+                  nodes:
+                    - name: node.default
+                      labels: ubuntu-focal
+                      host-key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOgHJYejINIKzUiuSJ2MN8uPc+dfFrZ9JH1hLWS8gI+g"
+                      python-path: /usr/bin/python3
+                      username: root
 
     - name: nodepool-kube-config
       data:

playbooks/zuul-operator-functional/tasks/wait_services.yaml

@@ -19,9 +19,16 @@
 - name: Wait for executor deployment
   command: timeout 10m kubectl rollout status statefulset/zuul-executor
 
-- name: Wait 8 minutes for launcher to settle
+- name: Wait 8 minutes for kube launcher to settle
   command: kubectl logs deployment/nodepool-launcher-kube-cluster
   register: _launcher_log
   until: "'Starting PoolWorker' in _launcher_log.stdout"
   delay: 10
   retries: 48
+
+- name: Wait 8 minutes for static launcher to settle
+  command: kubectl logs deployment/nodepool-launcher-static-vms
+  register: _launcher_log
+  until: "'Starting PoolWorker' in _launcher_log.stdout"
+  delay: 10
+  retries: 48

playbooks/zuul-operator-functional/test.yaml

@@ -1,5 +1,8 @@
 - name: run functional tst
   hosts: all
+  vars:
+    zuul_work_dir: "{{ zuul.projects['opendev.org/zuul/zuul-operator'].src_dir }}"
+    runtime: minikube
   tasks:
     - name: Install ingress
       include_tasks: tasks/ingress.yaml
@@ -10,13 +13,24 @@
       register: git_root
 
     - name: get cluster ip
+      when: runtime == 'minikube'
       command: /tmp/minikube ip
-      register: minikube_ip
-      failed_when: false
+      register: _cluster_ip
 
     - name: set cluster ip
+      when: runtime == 'minikube'
       set_fact:
-        cluster_ip: "{{ cluster_ip | default(minikube_ip.stdout_lines[0]) }}"
+        cluster_ip: "{{ _cluster_ip.stdout_lines[0] }}"
+
+    - name: get cluster ip
+      when: runtime == 'kind'
+      command: docker inspect -f "{% raw %}{{ .NetworkSettings.IPAddress }}{% endraw %}" kind-control-plane
+      register: _cluster_ip
+
+    - name: set cluster ip
+      when: runtime == 'kind'
+      set_fact:
+        cluster_ip: "{{ _cluster_ip.stdout_lines[0] }}"
 
     - name: set fact zuul_web_url
       set_fact:
@@ -121,6 +135,14 @@
             # 1 queue means a job is running
             queue: 1
 
+    - name: get build results
+      uri:
+        url: "{{ zuul_web_url }}/api/tenant/local/builds?complete=true"
+      register: result
+      until: "result.json is defined and result.json and (result.json|length) > 1"
+      retries: 600
+      delay: 1
+
     - name: get buillds results
       include_tasks: tasks/zuul_web_check.yaml
       vars:
@@ -130,13 +152,14 @@
       assert:
         that:
           - result.json[0].result == 'SUCCESS'
+          - result.json[1].result == 'SUCCESS'
 
     - name: grab job uuid
       shell: |
         curl -s {{ zuul_web_url }}/api/tenant/local/status | jq -r '.pipelines[].change_queues[].heads[][].jobs[].uuid'
       register: _job_uuid
       # Wait until the executor start the job
-      until: _job_uuid.stdout != "" and _job_uuid.stdout != "null"
+      until: _job_uuid.stdout != "" and "null" not in _job_uuid.stdout
       retries: 60
       delay: 1
 

tools/vars.yaml

@@ -1,3 +1,4 @@
 namespace: default
-cluster_ip: 172.17.0.2
 install_operator: false
+zuul_work_dir: "{{ playbook_dir }}/../../"
+runtime: kind