Run a git server in k8s for functional tests

To make the functional tests easier to run locally, run a git server
in k8s instead of on the local machine.

Also, update the secret creation to work with kind as well as minikube.

Change-Id: I4c00129c7105405b0f6256e769486e79c0b173fe

playbooks/zuul-operator-functional/pre-k8s.yaml

@@ -9,11 +9,19 @@
         minikube_dns_resolvers:
           - '1.1.1.1'
           - '8.8.8.8'
+        ensure_kubernetes_minikube_addons:
+          - ingress
     - role: use-buildset-registry
       buildset_registry_docker_user: root
   post_tasks:
     - name: Install openshift client for k8s tasks
       command: python3 -m pip install --user openshift
+    - name: Install websocket
+      become: true
+      package:
+        name:
+          - python3-websocket
+          - jq
     - name: check kubernetes connection
       command: timeout 10s kubectl get pods
       register: _api_ready

playbooks/zuul-operator-functional/tasks/create_config.yaml

@@ -1,13 +1,11 @@
 - name: create config directory
-  become: yes
+  command: "git clone git://{{ cluster_ip }}:{{ git_server_port }}/config"
-  file:
+  args:
-    path: /srv/git/config/
+    chdir: "{{ git_root.path }}"
-    state: directory
-    mode: 0777
 
 - name: copy config content
   copy:
-    dest: "/srv/git/config/{{ item.name }}"
+    dest: "{{ git_root.path }}/config/{{ item.name }}"
     content: "{{ item.content }}"
   loop:
     - name: zuul.yaml
@@ -55,7 +53,10 @@
 
 - name: commit config
   shell: |
-    cd /srv/git/config/
+    set -x
-    test -d .git || git init .
+
     git add *.yaml
     git commit -m 'Setup config project' || true
+    git push --set-upstream git://{{ cluster_ip }}:{{ git_server_port }}/config master
+  args:
+    chdir: "{{ git_root.path }}/config"

playbooks/zuul-operator-functional/tasks/create_test_secrets.yaml

@@ -8,24 +8,13 @@
   command: cat id_rsa
   register: _ssh_key
 
-- name: Read generated kubectl configuration
+- name: Read kube server address
-  command: |
+  command: kubectl config view -o jsonpath='{.clusters[0].cluster.server}'
-    sed -e 's#/home/zuul/.minikube/profiles/minikube/#/etc/kubernetes/#g'
-        -e 's#/home/zuul/.minikube/#/etc/kubernetes/#g'
-        ~/.kube/config
   register: _kube_config
 
-- name: Read client cert
+- name: Read generated kubectl configuration
-  command: cat ~/.minikube/profiles/minikube/client.crt
+  shell: "kubectl config view --flatten | sed -e 's#{{ _kube_config.stdout }}#https://kubernetes/#g'"
-  register: _kube_cert
+  register: _kube_config
-
-- name: Read client key
-  command: cat ~/.minikube/profiles/minikube/client.key
-  register: _kube_key
-
-- name: Read ca cert
-  command: cat ~/.minikube/ca.crt
-  register: _kube_ca
 
 - name: Read current context name
   command: kubectl config current-context
@@ -79,6 +68,3 @@
     - name: nodepool-kube-config
       data:
         kube.config: "{{ _kube_config.stdout }}"
-        client.crt: "{{ _kube_cert.stdout }}"
-        client.key: "{{ _kube_key.stdout }}"
-        ca.crt: "{{ _kube_ca.stdout }}"

playbooks/zuul-operator-functional/tasks/git_setup.yaml

@@ -1,34 +1,61 @@
-
 - name: setup gitconfig
   shell: |
     if ! test -f ~/.gitconfig && ! test -d ~/.config/git ; then
       git config --global user.email "you@example.com"
       git config --global user.name "Your Name"
     fi
-    sudo mkdir -p /srv/git
 
-- name: setup local-git service
+- k8s:
-  become: yes
+    namespace: default
-  copy:
+    definition:
-    dest: /etc/systemd/system/local-git.service
+      apiVersion: apps/v1
-    content: |
+      kind: Deployment
-      [Unit]
+      metadata:
-      Description=local git service
+        name: git-server
-      After=syslogp.target network.target
+      spec:
+        replicas: 1
+        selector:
+          matchLabels:
+            app: git-server
+        template:
+          metadata:
+            labels:
+              app: git-server
+          spec:
+            volumes:
+            - name: git-repos
+              emptyDir: {}
+            containers:
+            - name: git-server
+              image: docker.io/sdelrio/git-daemon:latest
+              ports:
+              - containerPort: 9418
+              volumeMounts:
+              - name: git-repos
+                mountPath: /git
+            initContainers:
+            - name: init-config
+              image: docker.io/sdelrio/git-daemon:latest
+              command: ['/usr/bin/git', 'init', '--bare', '/git/config']
+              volumeMounts:
+              - name: git-repos
+                mountPath: /git
 
-      [Service]
+- k8s:
-      Type=simple
+    namespace: default
-      SyslogIdentifier=local-git
+    definition:
-      ExecStart=/usr/bin/git daemon --export-all --reuseaddr --verbose --base-path=/srv/git/ /srv/git/
+      apiVersion: v1
-      Restart=always
+      kind: Service
-      RestartSec=3s
+      metadata:
+        name: git-server
+        labels:
+          app: git-server
+      spec:
+        type: NodePort
+        ports:
+          - port: 9418
+        selector:
+          app: git-server
 
-      [Install]
+- name: Wait for scheduler deployment
-      WantedBy=multi-user.target
+  command: timeout 10m kubectl rollout status deployment/git-server
-
-- name: start local-git service
-  become: yes
-  systemd:
-    daemon_reload: true
-    name: "local-git"
-    state: started

playbooks/zuul-operator-functional/tasks/zuul_web_check.yaml

@@ -2,6 +2,6 @@
   uri:
     url: "{{ zuul_web_url }}{{ endpoint }}"
   register: result
-  until: "result.json is defined and result.json{% if expected is defined %} and result.json == expected{% endif %}"
+  until: "result.json is defined and result.json and (expected is not defined or result.json == expected)"
   retries: 600
   delay: 1

playbooks/zuul-operator-functional/test.yaml

@@ -1,26 +1,24 @@
 - name: run functional tst
   hosts: all
-  pre_tasks:
-    - name: install git
-      become: yes
-      package:
-        name:
-          - git
-          - jq
-
-    - name: install websocket client
-      become: yes
-      command: python3 -m pip install websocket-client
-
   tasks:
-    - name: get rest api url
+    - name: Create temp git root directory
-      command: kubectl get svc zuul-web -o jsonpath='{.spec.clusterIP}'
+      tempfile:
-      register: zuul_web_ip
+        state: directory
+      register: git_root
+
+    - name: get cluster ip
+      command: /tmp/minikube ip
+      register: minikube_ip
+      failed_when: false
+
+    - name: set cluster ip
+      set_fact:
+        cluster_ip: "{{ cluster_ip | default(minikube_ip.stdout_lines[0]) }}"
 
     - name: set fact zuul_web_url
       set_fact:
-        zuul_web_url: "http://{{ zuul_web_ip.stdout_lines[0] }}:9000"
+        zuul_web_url: "http://{{ cluster_ip }}"
-        zuul_ws_url: "ws://{{ zuul_web_ip.stdout_lines[0] }}:9000"
+        zuul_ws_url: "ws://{{ cluster_ip }}"
 
     - name: ensure zuul web api is working
       when: skip_check is not defined
@@ -35,6 +33,14 @@
     - name: setup git service
       include_tasks: tasks/git_setup.yaml
 
+    - name: get git-server port
+      command: kubectl get svc git-server -o jsonpath='{.spec.ports[0].nodePort}'
+      register: git_server_port
+
+    - name: set git-server port
+      set_fact:
+        git_server_port: '{{ git_server_port.stdout_lines[0] }}'
+
     - name: create a config project
       include_tasks: tasks/create_config.yaml
 
@@ -83,7 +89,7 @@
                   baseurl: https://opendev.org
                   driver: git
                 local-git:
-                  baseurl: "git://{{ ansible_all_ipv4_addresses[0] }}/"
+                  baseurl: "git://git-server.default/"
                   driver: git
               externalConfig:
                 kubernetes:
@@ -131,7 +137,7 @@
 
     - name: connect to console-stream
       command: |
-        python3 /usr/local/bin/wsdump.py -r --eof-wait 5 -t '{"uuid":"{{ _job_uuid.stdout_lines[0] }}","logfile":"console.log"}' {{ zuul_ws_url }}/api/tenant/local/console-stream
+        wsdump -r --eof-wait 5 -t '{"uuid":"{{ _job_uuid.stdout_lines[0] }}","logfile":"console.log"}' {{ zuul_ws_url }}/api/tenant/local/console-stream
       register: console_stream
 
     - name: show console stream

zuul_operator/templates/zuul.yaml

@@ -76,7 +76,7 @@ metadata:
     app.kubernetes.io/part-of: zuul
     app.kubernetes.io/component: zuul-web
 spec:
-  #type: NodePort
+  type: NodePort
   ports:
   - name: zuul-web
     port: 9000