From 0f759638f388b948f0f0ccb6f5d8ae4834755aac Mon Sep 17 00:00:00 2001 From: Tristan Cacqueray Date: Fri, 3 Apr 2020 16:13:44 +0000 Subject: [PATCH] Update dhall-kubernetes version to v4.0.0 This change update the kubernetes binding to use the new Optional types. The main consequence is that all the fields that are optional needs to be prefixed with Some. This let us remove the `--omit-empty` parameter resulting in cleaner resources where we don't need to set a dummy emptyDir medium value. See this issue for the details: https://github.com/dhall-lang/dhall-kubernetes/issues/86 Change-Id: I23a0a028909208cd58f57a6f07ee93090b3f3a1a --- CONTRIBUTE.md | 4 +- build/Dockerfile | 2 +- conf/Kubernetes.dhall | 2 +- conf/zuul/resources.dhall | 160 +++++++++++++++------------- playbooks/files/cr_spec.yaml | 4 +- roles/zuul/library/dhall_to_json.py | 2 +- 6 files changed, 92 insertions(+), 82 deletions(-) diff --git a/CONTRIBUTE.md b/CONTRIBUTE.md index 07f9321..c329871 100644 --- a/CONTRIBUTE.md +++ b/CONTRIBUTE.md @@ -42,13 +42,13 @@ INPUT=$(yaml-to-dhall "(./conf/zuul/input.dhall).Input.Type" < playbooks/files/c Then you can evaluate the resources function, for example to get the scheduler service: ```bash -dhall-to-yaml --omit-empty --explain <<< "(./conf/zuul/resources.dhall ($INPUT)).Components.Zuul.Scheduler" +dhall-to-yaml --explain <<< "(./conf/zuul/resources.dhall ($INPUT)).Components.Zuul.Scheduler" ``` Or get all the kubernetes resources: ```bash -dhall-to-yaml --omit-empty <<< "(./conf/zuul/resources.dhall ($INPUT)).List" +dhall-to-yaml <<< "(./conf/zuul/resources.dhall ($INPUT)).List" ``` ## Run the ansible roles locally diff --git a/build/Dockerfile b/build/Dockerfile index 4656d29..bb816e1 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -5,7 +5,7 @@ ARG DHALL_VERSION=1.30.0 ARG DHALL_JSON_VERSION=1.6.2 ARG DHALL_JSON_DIGEST=ea37627c4e19789af33def099d4cb145b874c03b4d5b98cb33ce06be1debf4f3 ARG DHALL_LANG_REF=v14.0.0 -ARG DHALL_KUBE_REF=v3.0.0 +ARG DHALL_KUBE_REF=v4.0.0 # kubectl versions and digests ARG KUBECTL_VERSION=v1.17.0 diff --git a/conf/Kubernetes.dhall b/conf/Kubernetes.dhall index 9ccdba2..4d5c30b 100644 --- a/conf/Kubernetes.dhall +++ b/conf/Kubernetes.dhall @@ -1,3 +1,3 @@ {- Import the kubernetes types, see the ./Prelude.dhall file for documentation -} env:DHALL_KUBERNETES -? https://raw.githubusercontent.com/dhall-lang/dhall-kubernetes/3c6d09a9409977cdde58a091d76a6d20509ca4b0/package.dhall sha256:e9c55c7ff71f901314129e7ef100c3af5ec7a918dce25e06d83fa8c5472cb680 +? https://raw.githubusercontent.com/dhall-lang/dhall-kubernetes/v4.0.0/package.dhall sha256:d9eac5668d5ed9cb3364c0a39721d4694e4247dad16d8a82827e4619ee1d6188 diff --git a/conf/zuul/resources.dhall b/conf/zuul/resources.dhall index 946efcd..5dc1c45 100644 --- a/conf/zuul/resources.dhall +++ b/conf/zuul/resources.dhall @@ -264,11 +264,11 @@ in \(input : Input) let mkObjectMeta = \(name : Text) -> \(labels : Labels) - -> Kubernetes.ObjectMeta::{ name = name, labels = labels } + -> Kubernetes.ObjectMeta::{ name = name, labels = Some labels } let mkSelector = \(labels : Labels) - -> Kubernetes.LabelSelector::{ matchLabels = labels } + -> Kubernetes.LabelSelector::{ matchLabels = Some labels } let mkService = \(name : Text) @@ -280,8 +280,8 @@ in \(input : Input) , metadata = mkObjectMeta name labels , spec = Some Kubernetes.ServiceSpec::{ , type = Some "ClusterIP" - , selector = labels - , ports = + , selector = Some labels + , ports = Some [ Kubernetes.ServicePort::{ , name = Some port-name , protocol = Some "TCP" @@ -300,9 +300,7 @@ in \(input : Input) ( \(volume : Volume.Type) -> Kubernetes.Volume::{ , name = volume.name - , emptyDir = Kubernetes.EmptyDirVolumeSource::{ - , medium = Some "" - } + , emptyDir = Some Kubernetes.EmptyDirVolumeSource::{=} } ) @@ -326,9 +324,10 @@ in \(input : Input) -> Kubernetes.PodTemplateSpec::{ , metadata = mkObjectMeta component.name labels , spec = Some Kubernetes.PodSpec::{ - , volumes = - mkVolumeSecret component.volumes - # mkVolumeEmptyDir component.data-dir + , volumes = Some + ( mkVolumeSecret component.volumes + # mkVolumeEmptyDir component.data-dir + ) , containers = [ component.container ] , automountServiceAccountToken = Some False } @@ -351,14 +350,16 @@ in \(input : Input) , metadata = mkObjectMeta component-name ([] : Labels) , spec = Some Kubernetes.PersistentVolumeClaimSpec::{ - , accessModes = [ "ReadWriteOnce" ] + , accessModes = Some [ "ReadWriteOnce" ] , resources = Some Kubernetes.ResourceRequirements::{ - , requests = toMap - { storage = - Natural/show - component.claim-size - ++ "Gi" - } + , requests = Some + ( toMap + { storage = + Natural/show + component.claim-size + ++ "Gi" + } + ) } } } @@ -371,7 +372,7 @@ in \(input : Input) , replicas = Some component.count , selector = mkSelector labels , template = mkPodTemplateSpec component labels - , volumeClaimTemplates = claim + , volumeClaimTemplates = Some claim } } @@ -408,7 +409,7 @@ in \(input : Input) ( \(env : EnvSecret) -> Kubernetes.EnvVar::{ , name = env.name - , valueFrom = Kubernetes.EnvVarSource::{ + , valueFrom = Some Kubernetes.EnvVarSource::{ , secretKeyRef = Some Kubernetes.SecretKeySelector::{ , key = env.key , name = Some env.secret @@ -433,16 +434,17 @@ in \(input : Input) -> Kubernetes.Resource.Secret Kubernetes.Secret::{ , metadata = Kubernetes.ObjectMeta::{ name = volume.name } - , stringData = - Prelude.List.map - File - { mapKey : Text, mapValue : Text } - ( \(config : File) - -> { mapKey = config.path - , mapValue = config.content - } - ) - volume.files + , stringData = Some + ( Prelude.List.map + File + { mapKey : Text, mapValue : Text } + ( \(config : File) + -> { mapKey = config.path + , mapValue = config.content + } + ) + volume.files + ) } let zk-hosts = @@ -546,23 +548,25 @@ in \(input : Input) , image = Some "docker.io/library/postgres:12.1" , imagePullPolicy = Some "IfNotPresent" - , ports = + , ports = Some [ Kubernetes.ContainerPort::{ , name = Some "pg" , containerPort = 5432 } ] - , env = - mkEnvVarValue - ( toMap - { POSTGRES_USER = "zuul" - , POSTGRES_PASSWORD = - default-db-password - , PGDATA = "/var/lib/pg/data" - } - ) - , volumeMounts = - mkVolumeMount db-volumes + , env = Some + ( mkEnvVarValue + ( toMap + { POSTGRES_USER = "zuul" + , POSTGRES_PASSWORD = + default-db-password + , PGDATA = + "/var/lib/pg/data" + } + ) + ) + , volumeMounts = Some + (mkVolumeMount db-volumes) } } ) @@ -588,14 +592,14 @@ in \(input : Input) , image = Some "docker.io/library/zookeeper" , imagePullPolicy = Some "IfNotPresent" - , ports = + , ports = Some [ Kubernetes.ContainerPort::{ , name = Some "zk" , containerPort = 2181 } ] - , volumeMounts = - mkVolumeMount zk-volumes + , volumeMounts = Some + (mkVolumeMount zk-volumes) } } ) @@ -672,21 +676,23 @@ in \(input : Input) , container = Kubernetes.Container::{ , name = "scheduler" , image = zuul-image "scheduler" - , args = [ "zuul-scheduler", "-d" ] + , args = Some [ "zuul-scheduler", "-d" ] , imagePullPolicy = Some "IfNotPresent" - , ports = + , ports = Some [ Kubernetes.ContainerPort::{ , name = Some "gearman" , containerPort = 4730 } ] - , env = - zuul-env - # db-uri-secret-env - # zk-hosts-secret-env - , volumeMounts = - mkVolumeMount - (scheduler-volumes # zuul-data-dir) + , env = Some + ( zuul-env + # db-uri-secret-env + # zk-hosts-secret-env + ) + , volumeMounts = Some + ( mkVolumeMount + (scheduler-volumes # zuul-data-dir) + ) } } ) @@ -704,18 +710,19 @@ in \(input : Input) , container = Kubernetes.Container::{ , name = "executor" , image = zuul-image "executor" - , args = [ "zuul-executor", "-d" ] + , args = Some [ "zuul-executor", "-d" ] , imagePullPolicy = Some "IfNotPresent" - , ports = + , ports = Some [ Kubernetes.ContainerPort::{ , name = Some "finger" , containerPort = 7900 } ] - , env = zuul-env - , volumeMounts = - mkVolumeMount - (executor-volumes # zuul-data-dir) + , env = Some zuul-env + , volumeMounts = Some + ( mkVolumeMount + (executor-volumes # zuul-data-dir) + ) , securityContext = Some Kubernetes.SecurityContext::{ , privileged = Some True } @@ -735,18 +742,19 @@ in \(input : Input) , container = Kubernetes.Container::{ , name = "web" , image = zuul-image "web" - , args = [ "zuul-web", "-d" ] + , args = Some [ "zuul-web", "-d" ] , imagePullPolicy = Some "IfNotPresent" - , ports = + , ports = Some [ Kubernetes.ContainerPort::{ , name = Some "api" , containerPort = 9000 } ] - , env = zuul-env - , volumeMounts = - mkVolumeMount - (web-volumes # zuul-data-dir) + , env = Some zuul-env + , volumeMounts = Some + ( mkVolumeMount + (web-volumes # zuul-data-dir) + ) } } ) @@ -762,12 +770,13 @@ in \(input : Input) , container = Kubernetes.Container::{ , name = "merger" , image = zuul-image "merger" - , args = [ "zuul-merger", "-d" ] + , args = Some [ "zuul-merger", "-d" ] , imagePullPolicy = Some "IfNotPresent" - , env = zuul-env - , volumeMounts = - mkVolumeMount - (merger-volumes # zuul-data-dir) + , env = Some zuul-env + , volumeMounts = Some + ( mkVolumeMount + (merger-volumes # zuul-data-dir) + ) } } ) @@ -852,17 +861,18 @@ in \(input : Input) , container = Kubernetes.Container::{ , name = "launcher" , image = nodepool-image "launcher" - , args = + , args = Some [ "sh" , "-c" , shard-config ++ "nodepool-launcher -d -c /var/lib/nodepool/config.yaml" ] , imagePullPolicy = Some "IfNotPresent" - , env = nodepool-env - , volumeMounts = - mkVolumeMount - (nodepool-volumes # nodepool-data-dir) + , env = Some nodepool-env + , volumeMounts = Some + ( mkVolumeMount + (nodepool-volumes # nodepool-data-dir) + ) } } ) diff --git a/playbooks/files/cr_spec.yaml b/playbooks/files/cr_spec.yaml index 65941f0..3b05643 100644 --- a/playbooks/files/cr_spec.yaml +++ b/playbooks/files/cr_spec.yaml @@ -1,8 +1,8 @@ # Render kubernetes resources using: # INPUT=$(yaml-to-dhall "(./conf/zuul/input.dhall).Input.Type" < playbooks/files/cr_spec.yaml) -# dhall-to-yaml --omit-empty --explain <<< "(./conf/zuul/resources.dhall ($INPUT)).Components.Zuul.Scheduler" +# dhall-to-yaml --explain <<< "(./conf/zuul/resources.dhall ($INPUT)).Components.Zuul.Scheduler" # Or -# dhall-to-yaml --omit-empty --explain <<< "(./conf/zuul/resources.dhall ($INPUT)).List" +# dhall-to-yaml --explain <<< "(./conf/zuul/resources.dhall ($INPUT)).List" executor: count: 1 diff --git a/roles/zuul/library/dhall_to_json.py b/roles/zuul/library/dhall_to_json.py index f4e7947..a0b16c8 100755 --- a/roles/zuul/library/dhall_to_json.py +++ b/roles/zuul/library/dhall_to_json.py @@ -23,7 +23,7 @@ from ansible.module_utils.basic import AnsibleModule # type: ignore def run(expression: str) -> Any: proc = subprocess.Popen( - ['dhall-to-json', '--omit-empty', '--explain'], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + ['dhall-to-json', '--explain'], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = proc.communicate(expression.encode('utf-8')) if stderr: return dict(failed=True, msg=stderr.decode('utf-8'))