Increase scheduler wait timeout and improve logs collection

In some case, the test node needs more time to pull the
zuul image. This change increases the wait time to 8 minutes
to prevent false positive failure.

Thsi change also:
* adds a build artifact with the generated kubernetes resources.
* redirects post commands output to logfiles to unclutter the
  job-output console.
* replaces kubectl wait by rollout status.
* wait for operator and nodepool-launcher deployment.

Change-Id: I1c499bd11576f92b98511cd1ff180026b8aa70d8

.gitignore

@@ -0,0 +1,9 @@
+# Auto generated files
+*.pem
+*.crt
+*.key
+*.srl
+*.csr
+id_rsa
+id_rsa.pub
+*.patch

.zuul.yaml

@@ -10,6 +10,7 @@
       # We disable userland-proxy to enable scheduler deployement to connect to the gearman service
       # see: https://github.com/eclipse/che/issues/8134
       docker_userland_proxy: false
+      container_runtime: docker
 
 - job:
     description: Operator integration tests with Kubernetes

playbooks/zuul-operator-functional/post.yaml

@@ -3,7 +3,8 @@
     - collect-container-logs
   post_tasks:
     - name: Describe resources
-      command: "kubectl describe {{ item }}"
+      command: "bash -c 'kubectl describe {{ item }} > ~/zuul-output/logs/describe-{{ item }}.txt'"
+      ignore_errors: yes
       loop:
         - pods
         - deployments
@@ -12,10 +13,6 @@
         - secrets
         - configmaps
 
-    - name: Grab scheduler logs
+    - name: Delete empty container logs
-      command: "kubectl logs statefulset/zuul-scheduler"
+      command: "find {{ ansible_user_dir }}/zuul-output/logs/ -type f -empty -delete"
-      ignore_errors: yes
-
-    - name: Grab executor logs
-      command: "kubectl logs statefulset/zuul-executor"
       ignore_errors: yes

playbooks/zuul-operator-functional/pre-k8s.yaml

@@ -14,4 +14,8 @@
       buildset_registry_docker_user: root
   post_tasks:
     - name: check kubernetes connection
-      command: kubectl get pods
+      command: timeout 10s kubectl get pods
+      register: _api_ready
+      until: _api_ready.rc == 0
+      retries: 6
+      delay: 10

playbooks/zuul-operator-functional/run.yaml

@@ -1,11 +1,28 @@
 - name: install and start zuul operator
   hosts: all
   tasks:
+    - name: Render default crd
+      when:
+        - not use_local_role | default(false) | bool
+      shell: |
+        set -e
+        JSON_TO_DHALL="{{ container_runtime }} run -v $(pwd)/conf:/conf:Z --rm --entrypoint json-to-dhall -i docker.io/zuul/zuul-operator"
+        DHALL_TO_YAML="{{ container_runtime }} run -v $(pwd)/conf:/conf:Z --rm --entrypoint dhall-to-yaml -i docker.io/zuul/zuul-operator"
+        JSON=$(python3 -c 'import yaml, json; print(json.dumps(yaml.safe_load(open("playbooks/files/cr_spec.yaml"))))')
+        INPUT=$(echo $JSON | $JSON_TO_DHALL '(/conf/zuul/input.dhall).Input.Type')
+        echo '(/conf/zuul/resources.dhall ('$INPUT')).List' | $DHALL_TO_YAML > ~/zuul-output/logs/cr_spec-resources.yaml
+      args:
+        executable: /bin/bash
+        chdir: "{{ zuul.projects['opendev.org/zuul/zuul-operator'].src_dir }}"
+
     - name: Setup CRD
       command: make install
       args:
         chdir: "{{ zuul.projects['opendev.org/zuul/zuul-operator'].src_dir }}"
 
+    - name: Wait for operator deployment
+      command: timeout 8m kubectl rollout status deployment/zuul-operator
+
     - name: Generate executor ssh key
       command: ssh-keygen -t rsa -m PEM -N '' -f 'id_rsa' -q -C 'zuul-executor'
       args:
@@ -116,28 +133,33 @@
               secretName: nodepool-kube-config
               key: kube.config
 
-
+    - name: Wait maximum 4 minutes for the scheduler deployment
-    - name: Wait maximum 4 minutes for the scheduler pod
       shell: |
         for idx in $(seq 24); do
           date;
-          for res in statefulsets deployments pods; do echo == $res ==; kubectl get $res; done
+          kubectl get statefulset zuul-scheduler 2> /dev/null && break || :
-          kubectl get pod zuul-scheduler-0 2> /dev/null && break || :
           sleep 10;
         done
 
-    - name: Wait 2 minutes for the scheduler pod to be ready
+    - name: Wait for scheduler deployment
-      command: kubectl wait --for=condition=Ready --timeout=120s pod/zuul-scheduler-0
+      command: timeout 10m kubectl rollout status statefulset/zuul-scheduler
 
-    - name: Wait 4 minutes for scheduler to settle
+    - name: Wait 8 minutes for scheduler to settle
       command: kubectl logs pod/zuul-scheduler-0
       register: _scheduler_log
       until: "'Full reconfiguration complete' in _scheduler_log.stdout"
       delay: 10
-      retries: 24
+      retries: 48
 
-    - name: Wait 2 minutes for the executor pod to be ready
+    - name: Wait for executor deployment
-      command: kubectl wait --for=condition=Ready --timeout=120s pod/zuul-executor-0
+      command: timeout 10m kubectl rollout status statefulset/zuul-executor
+
+    - name: Wait 8 minutes for launcher to settle
+      command: kubectl logs deployment/zuul-launcher
+      register: _launcher_log
+      until: "'Active requests' in _launcher_log.stdout"
+      delay: 10
+      retries: 48
 
     - name: Wait an extra 2 minutes for the services to settle
       pause: