From d87ea64c063beeec7d6a99a1c93de57f869dead2 Mon Sep 17 00:00:00 2001 From: "James E. Blair" Date: Sat, 10 Apr 2021 18:21:39 -0700 Subject: [PATCH] Add support for zuul-registry Change-Id: Ie206f8f7c48da3cfaabb33d16f2a339b35db4e5e --- deploy/crds/zuul-ci_v1alpha2_zuul_cr.yaml | 2 - doc/source/index.rst | 60 ++++++++++ playbooks/zuul-operator-functional/run.yaml | 6 +- .../tasks/create_test_secrets.yaml | 9 ++ .../tasks/test_registry.yaml | 44 +++---- playbooks/zuul-operator-functional/test.yaml | 3 + zuul_operator/templates/zuul-registry.yaml | 107 ++++++++++++++++++ zuul_operator/zuul.py | 56 +++++++++ 8 files changed, 260 insertions(+), 27 deletions(-) create mode 100644 zuul_operator/templates/zuul-registry.yaml diff --git a/deploy/crds/zuul-ci_v1alpha2_zuul_cr.yaml b/deploy/crds/zuul-ci_v1alpha2_zuul_cr.yaml index 9d115f7..e572926 100644 --- a/deploy/crds/zuul-ci_v1alpha2_zuul_cr.yaml +++ b/deploy/crds/zuul-ci_v1alpha2_zuul_cr.yaml @@ -13,8 +13,6 @@ spec: scheduler: config: secretName: zuul-yaml-conf - registry: - count: 1 launcher: config: secretName: nodepool-yaml-conf diff --git a/doc/source/index.rst b/doc/source/index.rst index 2d3ef6b..28cec80 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -192,6 +192,33 @@ static HTML/Javascript sites). If you enable this, the operator will configure a ``zuul-preview`` service to which you may route an Ingress or LoadBalancer. +Zuul Registry +------------- + +The operator has optional support for deploying a zuul-registry +service. This is an experimental add-on for Zuul to act as an +intermediate registry for the container image jobs in `zuul-jobs`. + +If you enable this, the operator will, by default, configure a +``zuul-registry`` service in a manner appropriate for access from +within the cluster only. If you need to access the registry from +outside the cluster, you will need to additionally add an Ingress or +LoadBalancer, as well as provide TLS certs with the appropriate +hostname. Currently, zuul-registry performs its own TLS termination. + +If you usue this, you will also need to provide a ``registry.yaml`` +config file in a secret. You only need to provide the ``users`` and, +if you are accessing the registry outside the cluster, the +``public-url`` setting (omit it if you are accessing it from within +the cluster only). + +.. code-block:: yaml + + registry: + users: + - name: testuser + pass: testpass + access: write Specification Reference ----------------------- @@ -481,3 +508,36 @@ verbatim): :default: 0 How many Zuul Preview servers to manage. + + .. attr:: registry + + .. attr:: count + :default: 0 + + How many Zuul Registry servers to manage. + + .. attr:: volumeSize + :default: 80Gi + + The requested size of the registry storage volume. + + .. attr:: tls + + .. attr:: secretName + + The name of a secret containing a TLS client certificate + and key for Zuul Registry. This should be (or the format + should match) a standard Kubernetes TLS secret. + + If you omit this, the operator will create a secret for + you. + + .. attr:: config + + .. attr:: secretName + + The name of a secret containing a registry + configuration file. The key in the secret should be + ``registry.yaml``. Only provide the ``users`` and, if + exposing the registry outside the cluster, the + ``public-url`` entries. diff --git a/playbooks/zuul-operator-functional/run.yaml b/playbooks/zuul-operator-functional/run.yaml index 28d5e8f..5a58480 100644 --- a/playbooks/zuul-operator-functional/run.yaml +++ b/playbooks/zuul-operator-functional/run.yaml @@ -37,6 +37,8 @@ secretName: nodepool-kube-config registry: count: 1 + config: + secretName: zuul-registry-conf preview: count: 1 @@ -45,7 +47,3 @@ - name: Test the cert-manager include_tasks: ./tasks/test_cert_manager.yaml - - # TODO: implement - # - name: Test the registry - # include_tasks: ./tasks/test_registry.yaml diff --git a/playbooks/zuul-operator-functional/tasks/create_test_secrets.yaml b/playbooks/zuul-operator-functional/tasks/create_test_secrets.yaml index 292685f..45d9d25 100644 --- a/playbooks/zuul-operator-functional/tasks/create_test_secrets.yaml +++ b/playbooks/zuul-operator-functional/tasks/create_test_secrets.yaml @@ -68,3 +68,12 @@ - name: nodepool-kube-config data: kube.config: "{{ _kube_config.stdout }}" + + - name: zuul-registry-conf + data: + registry.yaml: | + registry: + users: + - name: testuser + pass: testpass + access: write diff --git a/playbooks/zuul-operator-functional/tasks/test_registry.yaml b/playbooks/zuul-operator-functional/tasks/test_registry.yaml index 2fd12e9..32da8ea 100644 --- a/playbooks/zuul-operator-functional/tasks/test_registry.yaml +++ b/playbooks/zuul-operator-functional/tasks/test_registry.yaml @@ -1,21 +1,23 @@ -- name: Get registry service ip - command: kubectl get svc registry -o "jsonpath={.spec.clusterIP}" - register: _registry_ip - -- name: Add registry to /etc/hosts - become: yes - lineinfile: - path: /etc/hosts - regexp: "^.* registry$" - line: "{{ _registry_ip.stdout_lines[0] }} registry" - -- name: Get registry password - command: kubectl get secret zuul-registry-user-rw -o "jsonpath={.data.password}" - register: _registry_password - -- name: Test registry login - command: > - podman login - --tls-verify=false registry:9000 - -u zuul - -p "{{ _registry_password.stdout_lines[0] | b64decode }}" +- k8s: + namespace: default + definition: + apiVersion: batch/v1 + kind: Job + metadata: + name: test-registry + spec: + template: + spec: + containers: + - name: test-registry + image: quay.io/containers/podman:latest + command: ['podman', 'login', '--tls-verify=false', 'https://zuul-registry/', '-u', 'testuser', '-p', 'testpass'] + securityContext: + privileged: true + restartPolicy: Never + backoffLimit: 4 + wait: yes + wait_timeout: 300 + wait_condition: + type: Complete + status: "True" diff --git a/playbooks/zuul-operator-functional/test.yaml b/playbooks/zuul-operator-functional/test.yaml index ff5ce0e..526f649 100644 --- a/playbooks/zuul-operator-functional/test.yaml +++ b/playbooks/zuul-operator-functional/test.yaml @@ -156,3 +156,6 @@ - name: Test the preview include_tasks: ./tasks/test_preview.yaml + + - name: Test the registry + include_tasks: ./tasks/test_registry.yaml diff --git a/zuul_operator/templates/zuul-registry.yaml b/zuul_operator/templates/zuul-registry.yaml new file mode 100644 index 0000000..7ff9be7 --- /dev/null +++ b/zuul_operator/templates/zuul-registry.yaml @@ -0,0 +1,107 @@ +{%- if manage_registry_cert %} +--- +apiVersion: cert-manager.io/v1alpha2 +kind: Certificate +metadata: + name: zuul-registry-tls + labels: + app.kubernetes.io/name: zuul + app.kubernetes.io/instance: {{ instance_name }} + app.kubernetes.io/part-of: zuul + app.kubernetes.io/component: zuul-registry-tls +spec: + keyEncoding: pkcs8 + secretName: zuul-registry-tls + commonName: client + usages: + - digital signature + - key encipherment + - server auth + - client auth + issuerRef: + name: ca-issuer + kind: Issuer +{%- endif %} +--- +apiVersion: v1 +kind: Service +metadata: + name: zuul-registry + labels: + app.kubernetes.io/name: zuul + app.kubernetes.io/instance: {{ instance_name }} + app.kubernetes.io/part-of: zuul + app.kubernetes.io/component: zuul-registry +spec: + type: NodePort + ports: + - name: zuul-registry + port: 443 + protocol: TCP + targetPort: registry + selector: + app.kubernetes.io/name: zuul + app.kubernetes.io/instance: {{ instance_name }} + app.kubernetes.io/part-of: zuul + app.kubernetes.io/component: zuul-registry +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: zuul-registry + labels: + app.kubernetes.io/name: zuul + app.kubernetes.io/instance: {{ instance_name }} + app.kubernetes.io/part-of: zuul + app.kubernetes.io/component: zuul-registry +spec: + replicas: {{ spec.registry.count }} + serviceName: zuul-registry + selector: + matchLabels: + app.kubernetes.io/name: zuul + app.kubernetes.io/instance: {{ instance_name }} + app.kubernetes.io/part-of: zuul + app.kubernetes.io/component: zuul-registry + template: + metadata: + labels: + app.kubernetes.io/name: zuul + app.kubernetes.io/instance: {{ instance_name }} + app.kubernetes.io/part-of: zuul + app.kubernetes.io/component: zuul-registry + spec: + containers: + - name: registry + image: {{ spec.imagePrefix }}/zuul-registry:{{ spec.zuulImageVersion }} + env: + - name: DEBUG + value: '1' + ports: + - name: registry + containerPort: 9000 + volumeMounts: + - name: zuul-registry-config + mountPath: /conf + readOnly: true + - name: zuul-registry-tls + mountPath: /tls + readOnly: true + - name: zuul-registry + mountPath: /storage + volumes: + - name: zuul-registry-config + secret: + secretName: zuul-registry-generated-config + - name: zuul-registry-tls + secret: + secretName: {{ spec.registry.tls.secretName }} + volumeClaimTemplates: + - metadata: + name: zuul-registry + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ spec.registry.volumeSize }} #80Gi diff --git a/zuul_operator/zuul.py b/zuul_operator/zuul.py index 19bcf3e..b3f88bc 100644 --- a/zuul_operator/zuul.py +++ b/zuul_operator/zuul.py @@ -12,6 +12,7 @@ # License for the specific language governing permissions and limitations # under the License. +import kopf import copy import base64 import hashlib @@ -72,10 +73,17 @@ class Zuul: self.spec.setdefault('web', {}).setdefault('count', 1) self.spec.setdefault('fingergw', {}).setdefault('count', 1) self.spec.setdefault('preview', {}).setdefault('count', 0) + registry = self.spec.setdefault('registry', {}) + registry.setdefault('count', 0) + registry.setdefault('volumeSize', '80Gi') + registry_tls = registry.setdefault('tls', {}) + self.manage_registry_cert = ('secretName' not in registry_tls) + registry_tls.setdefault('secretName', 'zuul-registry-tls') self.spec.setdefault('imagePrefix', 'docker.io/zuul') self.spec.setdefault('zuulImageVersion', 'latest') self.spec.setdefault('zuulPreviewImageVersion', 'latest') + self.spec.setdefault('zuulRegistryImageVersion', 'latest') self.spec.setdefault('nodepoolImageVersion', 'latest') self.cert_manager = certmanager.CertManager( @@ -314,7 +322,55 @@ class Zuul: except pykube.exceptions.ObjectDoesNotExist: pass + def write_registry_conf(self): + config_secret = self.spec['registry'].get('config', {}).get('secretName') + if not config_secret: + raise kopf.PermanentError("No registry config secret found") + + try: + obj = objects.Secret.objects(self.api).\ + filter(namespace=self.namespace).\ + get(name=config_secret) + except pykube.exceptions.ObjectDoesNotExist: + raise kopf.TemporaryError("Registry config secret not found") + + # Shard the config so we can create a deployment + secret for + # each provider. + registry_yaml = yaml.safe_load(base64.b64decode( + obj.obj['data']['registry.yaml'])) + + reg = registry_yaml['registry'] + if 'public-url' not in reg: + reg['public-url'] = 'https://zuul-registry' + reg['address'] = '0.0.0.0' + reg['port'] = 9000 + reg['tls-cert'] = '/tls/tls.crt' + reg['tls-key'] = '/tls/tls.key' + reg['secret'] = utils.generate_password(56) + reg['storage'] = { + 'driver': 'filesystem', + 'root': '/storage', + } + + text = yaml.dump(registry_yaml) + utils.update_secret(self.api, self.namespace, + 'zuul-registry-generated-config', + string_data={'registry.yaml': text}) + + def create_registry(self): + self.write_registry_conf() + kw = { + 'instance_name': self.name, + 'spec': self.spec, + 'manage_registry_cert': self.manage_registry_cert, + } + utils.apply_file(self.api, 'zuul-registry.yaml', + namespace=self.namespace, **kw) + def create_zuul(self): + if self.spec['registry']['count']: + self.create_registry() + kw = { 'zuul_conf_sha': self.zuul_conf_sha, 'zuul_tenant_secret': self.tenant_secret,