Add support for zuul-registry

Change-Id: Ie206f8f7c48da3cfaabb33d16f2a339b35db4e5e

deploy/crds/zuul-ci_v1alpha2_zuul_cr.yaml

@@ -13,8 +13,6 @@ spec:
   scheduler:
     config:
       secretName: zuul-yaml-conf
-  registry:
-    count: 1
   launcher:
     config:
       secretName: nodepool-yaml-conf

doc/source/index.rst

@@ -192,6 +192,33 @@ static HTML/Javascript sites).  If you enable this, the operator will
 configure a ``zuul-preview`` service to which you may route an Ingress
 or LoadBalancer.
 
+Zuul Registry
+-------------
+
+The operator has optional support for deploying a zuul-registry
+service.  This is an experimental add-on for Zuul to act as an
+intermediate registry for the container image jobs in `zuul-jobs`.
+
+If you enable this, the operator will, by default, configure a
+``zuul-registry`` service in a manner appropriate for access from
+within the cluster only.  If you need to access the registry from
+outside the cluster, you will need to additionally add an Ingress or
+LoadBalancer, as well as provide TLS certs with the appropriate
+hostname.  Currently, zuul-registry performs its own TLS termination.
+
+If you usue this, you will also need to provide a ``registry.yaml``
+config file in a secret.  You only need to provide the ``users`` and,
+if you are accessing the registry outside the cluster, the
+``public-url`` setting (omit it if you are accessing it from within
+the cluster only).
+
+.. code-block:: yaml
+
+   registry:
+     users:
+       - name: testuser
+         pass: testpass
+         access: write
 
 Specification Reference
 -----------------------
@@ -481,3 +508,36 @@ verbatim):
             :default: 0
 
             How many Zuul Preview servers to manage.
+
+      .. attr:: registry
+
+         .. attr:: count
+            :default: 0
+
+            How many Zuul Registry servers to manage.
+
+         .. attr:: volumeSize
+            :default: 80Gi
+
+            The requested size of the registry storage volume.
+
+         .. attr:: tls
+
+            .. attr:: secretName
+
+               The name of a secret containing a TLS client certificate
+               and key for Zuul Registry.  This should be (or the format
+               should match) a standard Kubernetes TLS secret.
+
+               If you omit this, the operator will create a secret for
+               you.
+
+         .. attr:: config
+
+            .. attr:: secretName
+
+               The name of a secret containing a registry
+               configuration file.  The key in the secret should be
+               ``registry.yaml``.  Only provide the ``users`` and, if
+               exposing the registry outside the cluster, the
+               ``public-url`` entries.

playbooks/zuul-operator-functional/run.yaml

@@ -37,6 +37,8 @@
               secretName: nodepool-kube-config
           registry:
             count: 1
+            config:
+              secretName: zuul-registry-conf
           preview:
             count: 1
 
@@ -45,7 +47,3 @@
 
     - name: Test the cert-manager
       include_tasks: ./tasks/test_cert_manager.yaml
-
-    # TODO: implement
-    # - name: Test the registry
-    #   include_tasks: ./tasks/test_registry.yaml

playbooks/zuul-operator-functional/tasks/create_test_secrets.yaml

@@ -68,3 +68,12 @@
     - name: nodepool-kube-config
       data:
         kube.config: "{{ _kube_config.stdout }}"
+
+    - name: zuul-registry-conf
+      data:
+        registry.yaml: |
+          registry:
+            users:
+              - name: testuser
+                pass: testpass
+                access: write

playbooks/zuul-operator-functional/tasks/test_registry.yaml

@@ -1,21 +1,23 @@
-- name: Get registry service ip
-  command: kubectl get svc registry -o "jsonpath={.spec.clusterIP}"
-  register: _registry_ip
-
-- name: Add registry to /etc/hosts
-  become: yes
-  lineinfile:
-    path: /etc/hosts
-    regexp: "^.* registry$"
-    line: "{{ _registry_ip.stdout_lines[0] }} registry"
-
-- name: Get registry password
-  command: kubectl get secret zuul-registry-user-rw -o "jsonpath={.data.password}"
-  register: _registry_password
-
-- name: Test registry login
-  command: >
-    podman login
-      --tls-verify=false registry:9000
-      -u zuul
-      -p "{{ _registry_password.stdout_lines[0] | b64decode }}"
+- k8s:
+    namespace: default
+    definition:
+      apiVersion: batch/v1
+      kind: Job
+      metadata:
+        name: test-registry
+      spec:
+        template:
+          spec:
+            containers:
+            - name: test-registry
+              image: quay.io/containers/podman:latest
+              command: ['podman', 'login', '--tls-verify=false', 'https://zuul-registry/', '-u', 'testuser', '-p', 'testpass']
+              securityContext:
+                privileged: true
+            restartPolicy: Never
+        backoffLimit: 4
+    wait: yes
+    wait_timeout: 300
+    wait_condition:
+      type: Complete
+      status: "True"

playbooks/zuul-operator-functional/test.yaml

@@ -156,3 +156,6 @@
 
     - name: Test the preview
       include_tasks: ./tasks/test_preview.yaml
+
+    - name: Test the registry
+      include_tasks: ./tasks/test_registry.yaml

zuul_operator/templates/zuul-registry.yaml

@@ -0,0 +1,107 @@
+{%- if manage_registry_cert %}
+---
+apiVersion: cert-manager.io/v1alpha2
+kind: Certificate
+metadata:
+  name: zuul-registry-tls
+  labels:
+    app.kubernetes.io/name: zuul
+    app.kubernetes.io/instance: {{ instance_name }}
+    app.kubernetes.io/part-of: zuul
+    app.kubernetes.io/component: zuul-registry-tls
+spec:
+  keyEncoding: pkcs8
+  secretName: zuul-registry-tls
+  commonName: client
+  usages:
+    - digital signature
+    - key encipherment
+    - server auth
+    - client auth
+  issuerRef:
+    name: ca-issuer
+    kind: Issuer
+{%- endif %}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: zuul-registry
+  labels:
+    app.kubernetes.io/name: zuul
+    app.kubernetes.io/instance: {{ instance_name }}
+    app.kubernetes.io/part-of: zuul
+    app.kubernetes.io/component: zuul-registry
+spec:
+  type: NodePort
+  ports:
+  - name: zuul-registry
+    port: 443
+    protocol: TCP
+    targetPort: registry
+  selector:
+    app.kubernetes.io/name: zuul
+    app.kubernetes.io/instance: {{ instance_name }}
+    app.kubernetes.io/part-of: zuul
+    app.kubernetes.io/component: zuul-registry
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: zuul-registry
+  labels:
+    app.kubernetes.io/name: zuul
+    app.kubernetes.io/instance: {{ instance_name }}
+    app.kubernetes.io/part-of: zuul
+    app.kubernetes.io/component: zuul-registry
+spec:
+  replicas: {{ spec.registry.count }}
+  serviceName: zuul-registry
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: zuul
+      app.kubernetes.io/instance: {{ instance_name }}
+      app.kubernetes.io/part-of: zuul
+      app.kubernetes.io/component: zuul-registry
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: zuul
+        app.kubernetes.io/instance: {{ instance_name }}
+        app.kubernetes.io/part-of: zuul
+        app.kubernetes.io/component: zuul-registry
+    spec:
+      containers:
+      - name: registry
+        image: {{ spec.imagePrefix }}/zuul-registry:{{ spec.zuulImageVersion }}
+        env:
+        - name: DEBUG
+          value: '1'
+        ports:
+        - name: registry
+          containerPort: 9000
+        volumeMounts:
+        - name: zuul-registry-config
+          mountPath: /conf
+          readOnly: true
+        - name: zuul-registry-tls
+          mountPath: /tls
+          readOnly: true
+        - name: zuul-registry
+          mountPath: /storage
+      volumes:
+      - name: zuul-registry-config
+        secret:
+          secretName: zuul-registry-generated-config
+      - name: zuul-registry-tls
+        secret:
+          secretName: {{ spec.registry.tls.secretName }}
+  volumeClaimTemplates:
+  - metadata:
+      name: zuul-registry
+    spec:
+      accessModes:
+      - ReadWriteOnce
+      resources:
+        requests:
+          storage: {{ spec.registry.volumeSize }} #80Gi

zuul_operator/zuul.py

@@ -12,6 +12,7 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+import kopf
 import copy
 import base64
 import hashlib
@@ -72,10 +73,17 @@ class Zuul:
         self.spec.setdefault('web', {}).setdefault('count', 1)
         self.spec.setdefault('fingergw', {}).setdefault('count', 1)
         self.spec.setdefault('preview', {}).setdefault('count', 0)
+        registry = self.spec.setdefault('registry', {})
+        registry.setdefault('count', 0)
+        registry.setdefault('volumeSize', '80Gi')
+        registry_tls = registry.setdefault('tls', {})
+        self.manage_registry_cert = ('secretName' not in registry_tls)
+        registry_tls.setdefault('secretName', 'zuul-registry-tls')
 
         self.spec.setdefault('imagePrefix', 'docker.io/zuul')
         self.spec.setdefault('zuulImageVersion', 'latest')
         self.spec.setdefault('zuulPreviewImageVersion', 'latest')
+        self.spec.setdefault('zuulRegistryImageVersion', 'latest')
         self.spec.setdefault('nodepoolImageVersion', 'latest')
 
         self.cert_manager = certmanager.CertManager(
@@ -314,7 +322,55 @@ class Zuul:
             except pykube.exceptions.ObjectDoesNotExist:
                 pass
 
+    def write_registry_conf(self):
+        config_secret = self.spec['registry'].get('config', {}).get('secretName')
+        if not config_secret:
+            raise kopf.PermanentError("No registry config secret found")
+
+        try:
+            obj = objects.Secret.objects(self.api).\
+                filter(namespace=self.namespace).\
+                get(name=config_secret)
+        except pykube.exceptions.ObjectDoesNotExist:
+            raise kopf.TemporaryError("Registry config secret not found")
+
+        # Shard the config so we can create a deployment + secret for
+        # each provider.
+        registry_yaml = yaml.safe_load(base64.b64decode(
+            obj.obj['data']['registry.yaml']))
+
+        reg = registry_yaml['registry']
+        if 'public-url' not in reg:
+            reg['public-url'] = 'https://zuul-registry'
+        reg['address'] = '0.0.0.0'
+        reg['port'] = 9000
+        reg['tls-cert'] = '/tls/tls.crt'
+        reg['tls-key'] = '/tls/tls.key'
+        reg['secret'] = utils.generate_password(56)
+        reg['storage'] = {
+            'driver': 'filesystem',
+            'root': '/storage',
+        }
+
+        text = yaml.dump(registry_yaml)
+        utils.update_secret(self.api, self.namespace,
+                            'zuul-registry-generated-config',
+                            string_data={'registry.yaml': text})
+
+    def create_registry(self):
+        self.write_registry_conf()
+        kw = {
+            'instance_name': self.name,
+            'spec': self.spec,
+            'manage_registry_cert': self.manage_registry_cert,
+        }
+        utils.apply_file(self.api, 'zuul-registry.yaml',
+                         namespace=self.namespace, **kw)
+
     def create_zuul(self):
+        if self.spec['registry']['count']:
+            self.create_registry()
+
         kw = {
             'zuul_conf_sha': self.zuul_conf_sha,
             'zuul_tenant_secret': self.tenant_secret,