Handle service restart when connections are changed

This change adds tasks to reload tenant queues when the configuration
changes.

Change-Id: Ia404d85d57e76202ac7381ae7cdc51aaf809e07b

build/Dockerfile

@@ -4,7 +4,8 @@ FROM quay.io/operator-framework/ansible-operator:v0.13.0
 USER root
 
 # See: https://github.com/operator-framework/operator-sdk/issues/2384
-RUN pip3 install --upgrade openshift
+# Install gear to connect to the scheduler gearman
+RUN pip3 install --upgrade openshift gear
 
 # unarchive: bzip2 and tar
 # generate zuul ssh-keys or certificate: openssh and openssl

roles/zuul-lookup-conf/tasks/main.yaml

@@ -0,0 +1,3 @@
+- name: Lookup zuul conf secret
+  set_fact:
+    zuul_conf_secret: "{{ lookup('k8s', api_version='v1', kind='Secret', namespace=namespace, resource_name=zuul_name + '-secret-zuul') }}"

roles/zuul-restart-when-zuul-conf-changed/library/dump_zuul_changes.py

@@ -0,0 +1,41 @@
+#!/usr/bin/env python3
+# Copyright 2020 Red Hat
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from ansible.module_utils.basic import AnsibleModule
+from ansible.module_utils import gearlib
+
+
+def gearman_dump():
+    client = gearlib.connect("scheduler")
+    queues = dict()
+    for tenant in gearlib.run(client, "zuul:tenant_list"):
+        name = tenant['name']
+        queues[name] = gearlib.run(client, "zuul:status_get", {"tenant": name})
+    return queues
+
+
+def ansible_main():
+    module = AnsibleModule(
+        argument_spec=dict()
+    )
+
+    try:
+        module.exit_json(changed=False, changes=gearman_dump())
+    except Exception as e:
+        module.fail_json(msg="Couldn't get gearman status: %s" % e)
+
+
+if __name__ == '__main__':
+    ansible_main()

roles/zuul-restart-when-zuul-conf-changed/library/load_zuul_changes.py

@@ -0,0 +1,60 @@
+#!/usr/bin/env python3
+# Copyright 2020 Red Hat
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import time
+from ansible.module_utils.basic import AnsibleModule
+from ansible.module_utils import gearlib
+
+
+def gearman_load(changes):
+    for retry in range(120):
+        try:
+            client = gearlib.connect("scheduler")
+        except Exception:
+            time.sleep(1)
+    for tenant, status in changes.items():
+        for pipeline in status['pipelines']:
+            for queue in pipeline['change_queues']:
+                for head in queue['heads']:
+                    for change in head:
+                        if (not change['live'] or
+                                not change.get('id') or
+                                ',' not in change['id']):
+                            continue
+                        cid, cps = change['id'].split(',')
+                        gearlib.run(client, "zuul:enqueue", dict(
+                            tenant=tenant,
+                            pipeline=pipeline['name'],
+                            project=change['project_canonical'],
+                            trigger='gerrit',
+                            change=cid + ',' + cps
+                        ))
+
+
+def ansible_main():
+    module = AnsibleModule(
+        argument_spec=dict(
+            changes=dict(required=True)
+        )
+    )
+
+    try:
+        module.exit_json(changed=False, changes=gearman_load(module.params['changes']))
+    except Exception as e:
+        module.fail_json(msg="Couldn't get gearman status: %s" % e)
+
+
+if __name__ == '__main__':
+    ansible_main()

roles/zuul-restart-when-zuul-conf-changed/module_utils/gearlib.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python3
+# Copyright 2020 Red Hat
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import json
+import time
+from typing import Any
+import gear  # type: ignore
+
+
+def connect(host : str) -> Any:
+    client = gear.Client()
+    client.addServer(host, 4730, 'client.key', 'client.pem', 'ca.pem')
+    client.waitForServer(timeout=10)
+    return client
+
+
+def run(client : Any, job_name : str, args : Any = dict()) -> Any:
+    job = gear.Job(job_name.encode('utf-8'), json.dumps(args).encode('utf-8'))
+    client.submitJob(job, timeout=300)
+    while not job.complete:
+        time.sleep(0.1)
+    return json.loads(job.data[0])
+
+
+if __name__ == '__main__':
+    print(run(connect("scheduler"), "status"))

roles/zuul-restart-when-zuul-conf-changed/tasks/main.yaml

@@ -0,0 +1,49 @@
+- name: Lookup zuul conf secret
+  set_fact:
+    old_zuul_conf: "{{ zuul_conf_secret.data['zuul.conf'] | checksum }}"
+    new_zuul_conf: "{{ lookup('k8s', api_version='v1', kind='Secret', namespace=namespace, resource_name=zuul_name + '-secret-zuul').data['zuul.conf'] | checksum }}"
+    scheduler: "{{ lookup('k8s', api_version='v1', kind='StatefulSet', namespace=namespace, resource_name=zuul_name + '-scheduler') }}"
+
+- name: Restart zuul
+  when: >
+    new_zuul_conf != old_zuul_conf or (
+      scheduler.spec.template.metadata.labels.version is defined and
+      scheduler.spec.template.metadata.labels.version != new_zuul_conf )
+  vars:
+    services:
+        - kind: StatefulSet
+          name: "{{ zuul_name }}-scheduler"
+        - kind: StatefulSet
+          name: "{{ zuul_name }}-executor"
+        - kind: Deployment
+          name: "{{ zuul_name }}-web"
+    extra_services:
+        - kind: Deployment
+          name: "{{ zuul_name }}-merger"
+
+  block:
+    - name: Dump pipelines qeues
+      dump_zuul_changes:
+      register: zuul_changes
+
+    - name: Patch service
+      k8s:
+        state: present
+        namespace: "{{ namespace }}"
+        merge_type: merge
+        wait: true
+        definition:
+          apiVersion: v1
+          kind: "{{ item.kind }}"
+          metadata:
+            name: "{{ item.name }}"
+          spec:
+            template:
+              metadata:
+                labels:
+                  version: "{{ new_zuul_conf }}"
+      loop: "{% if merger.count is defined and merger.count > 0 %}{{ services | union(extra_services) }}{% else %}{{ services }}{% endif %}"
+
+    - name: Reload pipeline queues
+      load_zuul_changes:
+        changes: "{{ zuul_changes }}"

roles/zuul/tasks/main.yaml

@@ -1,5 +1,8 @@
 - include_role:
-    name: zuul-ensure-gearman-tls
+    name: "{{ item }}"
+  loop:
+    - zuul-lookup-conf
+    - zuul-ensure-gearman-tls
 
 - name: Convert spec to template input
   json_to_dhall:
@@ -30,4 +33,6 @@
     apply: yes
   loop: "{{ _json.result['List']['items'] }}"
 
-# TODO: Patch services when their configuration changed
+- include_role:
+    name: zuul-restart-when-zuul-conf-changed
+  when: zuul_conf_secret.data is defined