From 3c0f9ea989321d6fd594154eeaa6db66124a25cf Mon Sep 17 00:00:00 2001 From: Tristan Cacqueray Date: Tue, 21 Jan 2020 14:17:47 +0000 Subject: [PATCH] Handle service restart when connections are changed This change adds tasks to reload tenant queues when the configuration changes. Change-Id: Ia404d85d57e76202ac7381ae7cdc51aaf809e07b --- build/Dockerfile | 3 +- roles/zuul-lookup-conf/tasks/main.yaml | 3 + .../library/dump_zuul_changes.py | 41 +++++++++++++ .../library/load_zuul_changes.py | 60 +++++++++++++++++++ .../module_utils/gearlib.py | 38 ++++++++++++ .../tasks/main.yaml | 49 +++++++++++++++ roles/zuul/tasks/main.yaml | 9 ++- 7 files changed, 200 insertions(+), 3 deletions(-) create mode 100644 roles/zuul-lookup-conf/tasks/main.yaml create mode 100755 roles/zuul-restart-when-zuul-conf-changed/library/dump_zuul_changes.py create mode 100755 roles/zuul-restart-when-zuul-conf-changed/library/load_zuul_changes.py create mode 100755 roles/zuul-restart-when-zuul-conf-changed/module_utils/gearlib.py create mode 100644 roles/zuul-restart-when-zuul-conf-changed/tasks/main.yaml diff --git a/build/Dockerfile b/build/Dockerfile index e3188bb..5c79248 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -4,7 +4,8 @@ FROM quay.io/operator-framework/ansible-operator:v0.13.0 USER root # See: https://github.com/operator-framework/operator-sdk/issues/2384 -RUN pip3 install --upgrade openshift +# Install gear to connect to the scheduler gearman +RUN pip3 install --upgrade openshift gear # unarchive: bzip2 and tar # generate zuul ssh-keys or certificate: openssh and openssl diff --git a/roles/zuul-lookup-conf/tasks/main.yaml b/roles/zuul-lookup-conf/tasks/main.yaml new file mode 100644 index 0000000..da4e927 --- /dev/null +++ b/roles/zuul-lookup-conf/tasks/main.yaml @@ -0,0 +1,3 @@ +- name: Lookup zuul conf secret + set_fact: + zuul_conf_secret: "{{ lookup('k8s', api_version='v1', kind='Secret', namespace=namespace, resource_name=zuul_name + '-secret-zuul') }}" diff --git a/roles/zuul-restart-when-zuul-conf-changed/library/dump_zuul_changes.py b/roles/zuul-restart-when-zuul-conf-changed/library/dump_zuul_changes.py new file mode 100755 index 0000000..1682200 --- /dev/null +++ b/roles/zuul-restart-when-zuul-conf-changed/library/dump_zuul_changes.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +# Copyright 2020 Red Hat +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from ansible.module_utils.basic import AnsibleModule +from ansible.module_utils import gearlib + + +def gearman_dump(): + client = gearlib.connect("scheduler") + queues = dict() + for tenant in gearlib.run(client, "zuul:tenant_list"): + name = tenant['name'] + queues[name] = gearlib.run(client, "zuul:status_get", {"tenant": name}) + return queues + + +def ansible_main(): + module = AnsibleModule( + argument_spec=dict() + ) + + try: + module.exit_json(changed=False, changes=gearman_dump()) + except Exception as e: + module.fail_json(msg="Couldn't get gearman status: %s" % e) + + +if __name__ == '__main__': + ansible_main() diff --git a/roles/zuul-restart-when-zuul-conf-changed/library/load_zuul_changes.py b/roles/zuul-restart-when-zuul-conf-changed/library/load_zuul_changes.py new file mode 100755 index 0000000..2c4cc00 --- /dev/null +++ b/roles/zuul-restart-when-zuul-conf-changed/library/load_zuul_changes.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +# Copyright 2020 Red Hat +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import time +from ansible.module_utils.basic import AnsibleModule +from ansible.module_utils import gearlib + + +def gearman_load(changes): + for retry in range(120): + try: + client = gearlib.connect("scheduler") + except Exception: + time.sleep(1) + for tenant, status in changes.items(): + for pipeline in status['pipelines']: + for queue in pipeline['change_queues']: + for head in queue['heads']: + for change in head: + if (not change['live'] or + not change.get('id') or + ',' not in change['id']): + continue + cid, cps = change['id'].split(',') + gearlib.run(client, "zuul:enqueue", dict( + tenant=tenant, + pipeline=pipeline['name'], + project=change['project_canonical'], + trigger='gerrit', + change=cid + ',' + cps + )) + + +def ansible_main(): + module = AnsibleModule( + argument_spec=dict( + changes=dict(required=True) + ) + ) + + try: + module.exit_json(changed=False, changes=gearman_load(module.params['changes'])) + except Exception as e: + module.fail_json(msg="Couldn't get gearman status: %s" % e) + + +if __name__ == '__main__': + ansible_main() diff --git a/roles/zuul-restart-when-zuul-conf-changed/module_utils/gearlib.py b/roles/zuul-restart-when-zuul-conf-changed/module_utils/gearlib.py new file mode 100755 index 0000000..3c94f9f --- /dev/null +++ b/roles/zuul-restart-when-zuul-conf-changed/module_utils/gearlib.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +# Copyright 2020 Red Hat +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import json +import time +from typing import Any +import gear # type: ignore + + +def connect(host : str) -> Any: + client = gear.Client() + client.addServer(host, 4730, 'client.key', 'client.pem', 'ca.pem') + client.waitForServer(timeout=10) + return client + + +def run(client : Any, job_name : str, args : Any = dict()) -> Any: + job = gear.Job(job_name.encode('utf-8'), json.dumps(args).encode('utf-8')) + client.submitJob(job, timeout=300) + while not job.complete: + time.sleep(0.1) + return json.loads(job.data[0]) + + +if __name__ == '__main__': + print(run(connect("scheduler"), "status")) diff --git a/roles/zuul-restart-when-zuul-conf-changed/tasks/main.yaml b/roles/zuul-restart-when-zuul-conf-changed/tasks/main.yaml new file mode 100644 index 0000000..32101b9 --- /dev/null +++ b/roles/zuul-restart-when-zuul-conf-changed/tasks/main.yaml @@ -0,0 +1,49 @@ +- name: Lookup zuul conf secret + set_fact: + old_zuul_conf: "{{ zuul_conf_secret.data['zuul.conf'] | checksum }}" + new_zuul_conf: "{{ lookup('k8s', api_version='v1', kind='Secret', namespace=namespace, resource_name=zuul_name + '-secret-zuul').data['zuul.conf'] | checksum }}" + scheduler: "{{ lookup('k8s', api_version='v1', kind='StatefulSet', namespace=namespace, resource_name=zuul_name + '-scheduler') }}" + +- name: Restart zuul + when: > + new_zuul_conf != old_zuul_conf or ( + scheduler.spec.template.metadata.labels.version is defined and + scheduler.spec.template.metadata.labels.version != new_zuul_conf ) + vars: + services: + - kind: StatefulSet + name: "{{ zuul_name }}-scheduler" + - kind: StatefulSet + name: "{{ zuul_name }}-executor" + - kind: Deployment + name: "{{ zuul_name }}-web" + extra_services: + - kind: Deployment + name: "{{ zuul_name }}-merger" + + block: + - name: Dump pipelines qeues + dump_zuul_changes: + register: zuul_changes + + - name: Patch service + k8s: + state: present + namespace: "{{ namespace }}" + merge_type: merge + wait: true + definition: + apiVersion: v1 + kind: "{{ item.kind }}" + metadata: + name: "{{ item.name }}" + spec: + template: + metadata: + labels: + version: "{{ new_zuul_conf }}" + loop: "{% if merger.count is defined and merger.count > 0 %}{{ services | union(extra_services) }}{% else %}{{ services }}{% endif %}" + + - name: Reload pipeline queues + load_zuul_changes: + changes: "{{ zuul_changes }}" diff --git a/roles/zuul/tasks/main.yaml b/roles/zuul/tasks/main.yaml index b0dbff3..c703a52 100644 --- a/roles/zuul/tasks/main.yaml +++ b/roles/zuul/tasks/main.yaml @@ -1,5 +1,8 @@ - include_role: - name: zuul-ensure-gearman-tls + name: "{{ item }}" + loop: + - zuul-lookup-conf + - zuul-ensure-gearman-tls - name: Convert spec to template input json_to_dhall: @@ -30,4 +33,6 @@ apply: yes loop: "{{ _json.result['List']['items'] }}" -# TODO: Patch services when their configuration changed +- include_role: + name: zuul-restart-when-zuul-conf-changed + when: zuul_conf_secret.data is defined