diff --git a/CONTRIBUTE.md b/CONTRIBUTE.md index 07f9321..c329871 100644 --- a/CONTRIBUTE.md +++ b/CONTRIBUTE.md @@ -42,13 +42,13 @@ INPUT=$(yaml-to-dhall "(./conf/zuul/input.dhall).Input.Type" < playbooks/files/c Then you can evaluate the resources function, for example to get the scheduler service: ```bash -dhall-to-yaml --omit-empty --explain <<< "(./conf/zuul/resources.dhall ($INPUT)).Components.Zuul.Scheduler" +dhall-to-yaml --explain <<< "(./conf/zuul/resources.dhall ($INPUT)).Components.Zuul.Scheduler" ``` Or get all the kubernetes resources: ```bash -dhall-to-yaml --omit-empty <<< "(./conf/zuul/resources.dhall ($INPUT)).List" +dhall-to-yaml <<< "(./conf/zuul/resources.dhall ($INPUT)).List" ``` ## Run the ansible roles locally diff --git a/build/Dockerfile b/build/Dockerfile index 4656d29..bb816e1 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -5,7 +5,7 @@ ARG DHALL_VERSION=1.30.0 ARG DHALL_JSON_VERSION=1.6.2 ARG DHALL_JSON_DIGEST=ea37627c4e19789af33def099d4cb145b874c03b4d5b98cb33ce06be1debf4f3 ARG DHALL_LANG_REF=v14.0.0 -ARG DHALL_KUBE_REF=v3.0.0 +ARG DHALL_KUBE_REF=v4.0.0 # kubectl versions and digests ARG KUBECTL_VERSION=v1.17.0 diff --git a/conf/Kubernetes.dhall b/conf/Kubernetes.dhall index 9ccdba2..4d5c30b 100644 --- a/conf/Kubernetes.dhall +++ b/conf/Kubernetes.dhall @@ -1,3 +1,3 @@ {- Import the kubernetes types, see the ./Prelude.dhall file for documentation -} env:DHALL_KUBERNETES -? https://raw.githubusercontent.com/dhall-lang/dhall-kubernetes/3c6d09a9409977cdde58a091d76a6d20509ca4b0/package.dhall sha256:e9c55c7ff71f901314129e7ef100c3af5ec7a918dce25e06d83fa8c5472cb680 +? https://raw.githubusercontent.com/dhall-lang/dhall-kubernetes/v4.0.0/package.dhall sha256:d9eac5668d5ed9cb3364c0a39721d4694e4247dad16d8a82827e4619ee1d6188 diff --git a/conf/zuul/resources.dhall b/conf/zuul/resources.dhall index 946efcd..5dc1c45 100644 --- a/conf/zuul/resources.dhall +++ b/conf/zuul/resources.dhall @@ -264,11 +264,11 @@ in \(input : Input) let mkObjectMeta = \(name : Text) -> \(labels : Labels) - -> Kubernetes.ObjectMeta::{ name = name, labels = labels } + -> Kubernetes.ObjectMeta::{ name = name, labels = Some labels } let mkSelector = \(labels : Labels) - -> Kubernetes.LabelSelector::{ matchLabels = labels } + -> Kubernetes.LabelSelector::{ matchLabels = Some labels } let mkService = \(name : Text) @@ -280,8 +280,8 @@ in \(input : Input) , metadata = mkObjectMeta name labels , spec = Some Kubernetes.ServiceSpec::{ , type = Some "ClusterIP" - , selector = labels - , ports = + , selector = Some labels + , ports = Some [ Kubernetes.ServicePort::{ , name = Some port-name , protocol = Some "TCP" @@ -300,9 +300,7 @@ in \(input : Input) ( \(volume : Volume.Type) -> Kubernetes.Volume::{ , name = volume.name - , emptyDir = Kubernetes.EmptyDirVolumeSource::{ - , medium = Some "" - } + , emptyDir = Some Kubernetes.EmptyDirVolumeSource::{=} } ) @@ -326,9 +324,10 @@ in \(input : Input) -> Kubernetes.PodTemplateSpec::{ , metadata = mkObjectMeta component.name labels , spec = Some Kubernetes.PodSpec::{ - , volumes = - mkVolumeSecret component.volumes - # mkVolumeEmptyDir component.data-dir + , volumes = Some + ( mkVolumeSecret component.volumes + # mkVolumeEmptyDir component.data-dir + ) , containers = [ component.container ] , automountServiceAccountToken = Some False } @@ -351,14 +350,16 @@ in \(input : Input) , metadata = mkObjectMeta component-name ([] : Labels) , spec = Some Kubernetes.PersistentVolumeClaimSpec::{ - , accessModes = [ "ReadWriteOnce" ] + , accessModes = Some [ "ReadWriteOnce" ] , resources = Some Kubernetes.ResourceRequirements::{ - , requests = toMap - { storage = - Natural/show - component.claim-size - ++ "Gi" - } + , requests = Some + ( toMap + { storage = + Natural/show + component.claim-size + ++ "Gi" + } + ) } } } @@ -371,7 +372,7 @@ in \(input : Input) , replicas = Some component.count , selector = mkSelector labels , template = mkPodTemplateSpec component labels - , volumeClaimTemplates = claim + , volumeClaimTemplates = Some claim } } @@ -408,7 +409,7 @@ in \(input : Input) ( \(env : EnvSecret) -> Kubernetes.EnvVar::{ , name = env.name - , valueFrom = Kubernetes.EnvVarSource::{ + , valueFrom = Some Kubernetes.EnvVarSource::{ , secretKeyRef = Some Kubernetes.SecretKeySelector::{ , key = env.key , name = Some env.secret @@ -433,16 +434,17 @@ in \(input : Input) -> Kubernetes.Resource.Secret Kubernetes.Secret::{ , metadata = Kubernetes.ObjectMeta::{ name = volume.name } - , stringData = - Prelude.List.map - File - { mapKey : Text, mapValue : Text } - ( \(config : File) - -> { mapKey = config.path - , mapValue = config.content - } - ) - volume.files + , stringData = Some + ( Prelude.List.map + File + { mapKey : Text, mapValue : Text } + ( \(config : File) + -> { mapKey = config.path + , mapValue = config.content + } + ) + volume.files + ) } let zk-hosts = @@ -546,23 +548,25 @@ in \(input : Input) , image = Some "docker.io/library/postgres:12.1" , imagePullPolicy = Some "IfNotPresent" - , ports = + , ports = Some [ Kubernetes.ContainerPort::{ , name = Some "pg" , containerPort = 5432 } ] - , env = - mkEnvVarValue - ( toMap - { POSTGRES_USER = "zuul" - , POSTGRES_PASSWORD = - default-db-password - , PGDATA = "/var/lib/pg/data" - } - ) - , volumeMounts = - mkVolumeMount db-volumes + , env = Some + ( mkEnvVarValue + ( toMap + { POSTGRES_USER = "zuul" + , POSTGRES_PASSWORD = + default-db-password + , PGDATA = + "/var/lib/pg/data" + } + ) + ) + , volumeMounts = Some + (mkVolumeMount db-volumes) } } ) @@ -588,14 +592,14 @@ in \(input : Input) , image = Some "docker.io/library/zookeeper" , imagePullPolicy = Some "IfNotPresent" - , ports = + , ports = Some [ Kubernetes.ContainerPort::{ , name = Some "zk" , containerPort = 2181 } ] - , volumeMounts = - mkVolumeMount zk-volumes + , volumeMounts = Some + (mkVolumeMount zk-volumes) } } ) @@ -672,21 +676,23 @@ in \(input : Input) , container = Kubernetes.Container::{ , name = "scheduler" , image = zuul-image "scheduler" - , args = [ "zuul-scheduler", "-d" ] + , args = Some [ "zuul-scheduler", "-d" ] , imagePullPolicy = Some "IfNotPresent" - , ports = + , ports = Some [ Kubernetes.ContainerPort::{ , name = Some "gearman" , containerPort = 4730 } ] - , env = - zuul-env - # db-uri-secret-env - # zk-hosts-secret-env - , volumeMounts = - mkVolumeMount - (scheduler-volumes # zuul-data-dir) + , env = Some + ( zuul-env + # db-uri-secret-env + # zk-hosts-secret-env + ) + , volumeMounts = Some + ( mkVolumeMount + (scheduler-volumes # zuul-data-dir) + ) } } ) @@ -704,18 +710,19 @@ in \(input : Input) , container = Kubernetes.Container::{ , name = "executor" , image = zuul-image "executor" - , args = [ "zuul-executor", "-d" ] + , args = Some [ "zuul-executor", "-d" ] , imagePullPolicy = Some "IfNotPresent" - , ports = + , ports = Some [ Kubernetes.ContainerPort::{ , name = Some "finger" , containerPort = 7900 } ] - , env = zuul-env - , volumeMounts = - mkVolumeMount - (executor-volumes # zuul-data-dir) + , env = Some zuul-env + , volumeMounts = Some + ( mkVolumeMount + (executor-volumes # zuul-data-dir) + ) , securityContext = Some Kubernetes.SecurityContext::{ , privileged = Some True } @@ -735,18 +742,19 @@ in \(input : Input) , container = Kubernetes.Container::{ , name = "web" , image = zuul-image "web" - , args = [ "zuul-web", "-d" ] + , args = Some [ "zuul-web", "-d" ] , imagePullPolicy = Some "IfNotPresent" - , ports = + , ports = Some [ Kubernetes.ContainerPort::{ , name = Some "api" , containerPort = 9000 } ] - , env = zuul-env - , volumeMounts = - mkVolumeMount - (web-volumes # zuul-data-dir) + , env = Some zuul-env + , volumeMounts = Some + ( mkVolumeMount + (web-volumes # zuul-data-dir) + ) } } ) @@ -762,12 +770,13 @@ in \(input : Input) , container = Kubernetes.Container::{ , name = "merger" , image = zuul-image "merger" - , args = [ "zuul-merger", "-d" ] + , args = Some [ "zuul-merger", "-d" ] , imagePullPolicy = Some "IfNotPresent" - , env = zuul-env - , volumeMounts = - mkVolumeMount - (merger-volumes # zuul-data-dir) + , env = Some zuul-env + , volumeMounts = Some + ( mkVolumeMount + (merger-volumes # zuul-data-dir) + ) } } ) @@ -852,17 +861,18 @@ in \(input : Input) , container = Kubernetes.Container::{ , name = "launcher" , image = nodepool-image "launcher" - , args = + , args = Some [ "sh" , "-c" , shard-config ++ "nodepool-launcher -d -c /var/lib/nodepool/config.yaml" ] , imagePullPolicy = Some "IfNotPresent" - , env = nodepool-env - , volumeMounts = - mkVolumeMount - (nodepool-volumes # nodepool-data-dir) + , env = Some nodepool-env + , volumeMounts = Some + ( mkVolumeMount + (nodepool-volumes # nodepool-data-dir) + ) } } ) diff --git a/playbooks/files/cr_spec.yaml b/playbooks/files/cr_spec.yaml index 65941f0..3b05643 100644 --- a/playbooks/files/cr_spec.yaml +++ b/playbooks/files/cr_spec.yaml @@ -1,8 +1,8 @@ # Render kubernetes resources using: # INPUT=$(yaml-to-dhall "(./conf/zuul/input.dhall).Input.Type" < playbooks/files/cr_spec.yaml) -# dhall-to-yaml --omit-empty --explain <<< "(./conf/zuul/resources.dhall ($INPUT)).Components.Zuul.Scheduler" +# dhall-to-yaml --explain <<< "(./conf/zuul/resources.dhall ($INPUT)).Components.Zuul.Scheduler" # Or -# dhall-to-yaml --omit-empty --explain <<< "(./conf/zuul/resources.dhall ($INPUT)).List" +# dhall-to-yaml --explain <<< "(./conf/zuul/resources.dhall ($INPUT)).List" executor: count: 1 diff --git a/roles/zuul/library/dhall_to_json.py b/roles/zuul/library/dhall_to_json.py index f4e7947..a0b16c8 100755 --- a/roles/zuul/library/dhall_to_json.py +++ b/roles/zuul/library/dhall_to_json.py @@ -23,7 +23,7 @@ from ansible.module_utils.basic import AnsibleModule # type: ignore def run(expression: str) -> Any: proc = subprocess.Popen( - ['dhall-to-json', '--omit-empty', '--explain'], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + ['dhall-to-json', '--explain'], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = proc.communicate(expression.encode('utf-8')) if stderr: return dict(failed=True, msg=stderr.decode('utf-8'))