zuul/zuul-operator
Code Issues Proposed changes

Update dhall-kubernetes version to v4.0.0

Browse Source 
This change update the kubernetes binding to use the new
Optional types. The main consequence is that all the fields that
are optional needs to be prefixed with Some. This let us remove
the `--omit-empty` parameter resulting in cleaner resources where
we don't need to set a dummy emptyDir medium value.

See this issue for the details:
https://github.com/dhall-lang/dhall-kubernetes/issues/86

Change-Id: I23a0a028909208cd58f57a6f07ee93090b3f3a1a
This commit is contained in:
Tristan Cacqueray 2020-04-03 16:13:44 +00:00
parent 27f9126874
commit 0f759638f3
6 changed files with 92 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
CONTRIBUTE.md
View File

@ -42,13 +42,13 @@ INPUT=$(yaml-to-dhall "(./conf/zuul/input.dhall).Input.Type" < playbooks/files/c
Then you can evaluate the resources function, for example to get the scheduler service: Then you can evaluate the resources function, for example to get the scheduler service:
```bash ```bash
dhall-to-yaml --omit-empty --explain <<< "(./conf/zuul/resources.dhall ($INPUT)).Components.Zuul.Scheduler" dhall-to-yaml --explain <<< "(./conf/zuul/resources.dhall ($INPUT)).Components.Zuul.Scheduler"
``` ```
Or get all the kubernetes resources: Or get all the kubernetes resources:
```bash ```bash
dhall-to-yaml --omit-empty <<< "(./conf/zuul/resources.dhall ($INPUT)).List" dhall-to-yaml <<< "(./conf/zuul/resources.dhall ($INPUT)).List"
``` ```
## Run the ansible roles locally ## Run the ansible roles locally

2
build/Dockerfile
View File

@ -5,7 +5,7 @@ ARG DHALL_VERSION=1.30.0
ARG DHALL_JSON_VERSION=1.6.2 ARG DHALL_JSON_VERSION=1.6.2
ARG DHALL_JSON_DIGEST=ea37627c4e19789af33def099d4cb145b874c03b4d5b98cb33ce06be1debf4f3 ARG DHALL_JSON_DIGEST=ea37627c4e19789af33def099d4cb145b874c03b4d5b98cb33ce06be1debf4f3
ARG DHALL_LANG_REF=v14.0.0 ARG DHALL_LANG_REF=v14.0.0
ARG DHALL_KUBE_REF=v3.0.0 ARG DHALL_KUBE_REF=v4.0.0
# kubectl versions and digests # kubectl versions and digests
ARG KUBECTL_VERSION=v1.17.0 ARG KUBECTL_VERSION=v1.17.0

2
conf/Kubernetes.dhall
View File

@ -1,3 +1,3 @@
{- Import the kubernetes types, see the ./Prelude.dhall file for documentation -} {- Import the kubernetes types, see the ./Prelude.dhall file for documentation -}
env:DHALL_KUBERNETES env:DHALL_KUBERNETES
? https://raw.githubusercontent.com/dhall-lang/dhall-kubernetes/3c6d09a9409977cdde58a091d76a6d20509ca4b0/package.dhall sha256:e9c55c7ff71f901314129e7ef100c3af5ec7a918dce25e06d83fa8c5472cb680 ? https://raw.githubusercontent.com/dhall-lang/dhall-kubernetes/v4.0.0/package.dhall sha256:d9eac5668d5ed9cb3364c0a39721d4694e4247dad16d8a82827e4619ee1d6188

106
conf/zuul/resources.dhall
View File

@ -264,11 +264,11 @@ in \(input : Input)
let mkObjectMeta = let mkObjectMeta =
\(name : Text) \(name : Text)
-> \(labels : Labels) -> \(labels : Labels)
-> Kubernetes.ObjectMeta::{ name = name, labels = labels } -> Kubernetes.ObjectMeta::{ name = name, labels = Some labels }
let mkSelector = let mkSelector =
\(labels : Labels) \(labels : Labels)
-> Kubernetes.LabelSelector::{ matchLabels = labels } -> Kubernetes.LabelSelector::{ matchLabels = Some labels }
let mkService = let mkService =
\(name : Text) \(name : Text)
@ -280,8 +280,8 @@ in \(input : Input)
, metadata = mkObjectMeta name labels , metadata = mkObjectMeta name labels
, spec = Some Kubernetes.ServiceSpec::{ , spec = Some Kubernetes.ServiceSpec::{
, type = Some "ClusterIP" , type = Some "ClusterIP"
, selector = labels , selector = Some labels
, ports = , ports = Some
[ Kubernetes.ServicePort::{ [ Kubernetes.ServicePort::{
, name = Some port-name , name = Some port-name
, protocol = Some "TCP" , protocol = Some "TCP"
@ -300,9 +300,7 @@ in \(input : Input)
( \(volume : Volume.Type) ( \(volume : Volume.Type)
-> Kubernetes.Volume::{ -> Kubernetes.Volume::{
, name = volume.name , name = volume.name
, emptyDir = Kubernetes.EmptyDirVolumeSource::{ , emptyDir = Some Kubernetes.EmptyDirVolumeSource::{=}
, medium = Some ""
}
} }
) )
@ -326,9 +324,10 @@ in \(input : Input)
-> Kubernetes.PodTemplateSpec::{ -> Kubernetes.PodTemplateSpec::{
, metadata = mkObjectMeta component.name labels , metadata = mkObjectMeta component.name labels
, spec = Some Kubernetes.PodSpec::{ , spec = Some Kubernetes.PodSpec::{
, volumes = , volumes = Some
mkVolumeSecret component.volumes ( mkVolumeSecret component.volumes
# mkVolumeEmptyDir component.data-dir # mkVolumeEmptyDir component.data-dir
)
, containers = [ component.container ] , containers = [ component.container ]
, automountServiceAccountToken = Some False , automountServiceAccountToken = Some False
} }
@ -351,14 +350,16 @@ in \(input : Input)
, metadata = , metadata =
mkObjectMeta component-name ([] : Labels) mkObjectMeta component-name ([] : Labels)
, spec = Some Kubernetes.PersistentVolumeClaimSpec::{ , spec = Some Kubernetes.PersistentVolumeClaimSpec::{
, accessModes = [ "ReadWriteOnce" ] , accessModes = Some [ "ReadWriteOnce" ]
, resources = Some Kubernetes.ResourceRequirements::{ , resources = Some Kubernetes.ResourceRequirements::{
, requests = toMap , requests = Some
( toMap
{ storage = { storage =
Natural/show Natural/show
component.claim-size component.claim-size
++ "Gi" ++ "Gi"
} }
)
} }
} }
} }
@ -371,7 +372,7 @@ in \(input : Input)
, replicas = Some component.count , replicas = Some component.count
, selector = mkSelector labels , selector = mkSelector labels
, template = mkPodTemplateSpec component labels , template = mkPodTemplateSpec component labels
, volumeClaimTemplates = claim , volumeClaimTemplates = Some claim
} }
} }
@ -408,7 +409,7 @@ in \(input : Input)
( \(env : EnvSecret) ( \(env : EnvSecret)
-> Kubernetes.EnvVar::{ -> Kubernetes.EnvVar::{
, name = env.name , name = env.name
, valueFrom = Kubernetes.EnvVarSource::{ , valueFrom = Some Kubernetes.EnvVarSource::{
, secretKeyRef = Some Kubernetes.SecretKeySelector::{ , secretKeyRef = Some Kubernetes.SecretKeySelector::{
, key = env.key , key = env.key
, name = Some env.secret , name = Some env.secret
@ -433,8 +434,8 @@ in \(input : Input)
-> Kubernetes.Resource.Secret -> Kubernetes.Resource.Secret
Kubernetes.Secret::{ Kubernetes.Secret::{
, metadata = Kubernetes.ObjectMeta::{ name = volume.name } , metadata = Kubernetes.ObjectMeta::{ name = volume.name }
, stringData = , stringData = Some
Prelude.List.map ( Prelude.List.map
File File
{ mapKey : Text, mapValue : Text } { mapKey : Text, mapValue : Text }
( \(config : File) ( \(config : File)
@ -443,6 +444,7 @@ in \(input : Input)
} }
) )
volume.files volume.files
)
} }
let zk-hosts = let zk-hosts =
@ -546,23 +548,25 @@ in \(input : Input)
, image = Some , image = Some
"docker.io/library/postgres:12.1" "docker.io/library/postgres:12.1"
, imagePullPolicy = Some "IfNotPresent" , imagePullPolicy = Some "IfNotPresent"
, ports = , ports = Some
[ Kubernetes.ContainerPort::{ [ Kubernetes.ContainerPort::{
, name = Some "pg" , name = Some "pg"
, containerPort = 5432 , containerPort = 5432
} }
] ]
, env = , env = Some
mkEnvVarValue ( mkEnvVarValue
( toMap ( toMap
{ POSTGRES_USER = "zuul" { POSTGRES_USER = "zuul"
, POSTGRES_PASSWORD = , POSTGRES_PASSWORD =
default-db-password default-db-password
, PGDATA = "/var/lib/pg/data" , PGDATA =
"/var/lib/pg/data"
} }
) )
, volumeMounts = )
mkVolumeMount db-volumes , volumeMounts = Some
(mkVolumeMount db-volumes)
} }
} }
) )
@ -588,14 +592,14 @@ in \(input : Input)
, image = Some , image = Some
"docker.io/library/zookeeper" "docker.io/library/zookeeper"
, imagePullPolicy = Some "IfNotPresent" , imagePullPolicy = Some "IfNotPresent"
, ports = , ports = Some
[ Kubernetes.ContainerPort::{ [ Kubernetes.ContainerPort::{
, name = Some "zk" , name = Some "zk"
, containerPort = 2181 , containerPort = 2181
} }
] ]
, volumeMounts = , volumeMounts = Some
mkVolumeMount zk-volumes (mkVolumeMount zk-volumes)
} }
} }
) )
@ -672,21 +676,23 @@ in \(input : Input)
, container = Kubernetes.Container::{ , container = Kubernetes.Container::{
, name = "scheduler" , name = "scheduler"
, image = zuul-image "scheduler" , image = zuul-image "scheduler"
, args = [ "zuul-scheduler", "-d" ] , args = Some [ "zuul-scheduler", "-d" ]
, imagePullPolicy = Some "IfNotPresent" , imagePullPolicy = Some "IfNotPresent"
, ports = , ports = Some
[ Kubernetes.ContainerPort::{ [ Kubernetes.ContainerPort::{
, name = Some "gearman" , name = Some "gearman"
, containerPort = 4730 , containerPort = 4730
} }
] ]
, env = , env = Some
zuul-env ( zuul-env
# db-uri-secret-env # db-uri-secret-env
# zk-hosts-secret-env # zk-hosts-secret-env
, volumeMounts = )
mkVolumeMount , volumeMounts = Some
( mkVolumeMount
(scheduler-volumes # zuul-data-dir) (scheduler-volumes # zuul-data-dir)
)
} }
} }
) )
@ -704,18 +710,19 @@ in \(input : Input)
, container = Kubernetes.Container::{ , container = Kubernetes.Container::{
, name = "executor" , name = "executor"
, image = zuul-image "executor" , image = zuul-image "executor"
, args = [ "zuul-executor", "-d" ] , args = Some [ "zuul-executor", "-d" ]
, imagePullPolicy = Some "IfNotPresent" , imagePullPolicy = Some "IfNotPresent"
, ports = , ports = Some
[ Kubernetes.ContainerPort::{ [ Kubernetes.ContainerPort::{
, name = Some "finger" , name = Some "finger"
, containerPort = 7900 , containerPort = 7900
} }
] ]
, env = zuul-env , env = Some zuul-env
, volumeMounts = , volumeMounts = Some
mkVolumeMount ( mkVolumeMount
(executor-volumes # zuul-data-dir) (executor-volumes # zuul-data-dir)
)
, securityContext = Some Kubernetes.SecurityContext::{ , securityContext = Some Kubernetes.SecurityContext::{
, privileged = Some True , privileged = Some True
} }
@ -735,18 +742,19 @@ in \(input : Input)
, container = Kubernetes.Container::{ , container = Kubernetes.Container::{
, name = "web" , name = "web"
, image = zuul-image "web" , image = zuul-image "web"
, args = [ "zuul-web", "-d" ] , args = Some [ "zuul-web", "-d" ]
, imagePullPolicy = Some "IfNotPresent" , imagePullPolicy = Some "IfNotPresent"
, ports = , ports = Some
[ Kubernetes.ContainerPort::{ [ Kubernetes.ContainerPort::{
, name = Some "api" , name = Some "api"
, containerPort = 9000 , containerPort = 9000
} }
] ]
, env = zuul-env , env = Some zuul-env
, volumeMounts = , volumeMounts = Some
mkVolumeMount ( mkVolumeMount
(web-volumes # zuul-data-dir) (web-volumes # zuul-data-dir)
)
} }
} }
) )
@ -762,12 +770,13 @@ in \(input : Input)
, container = Kubernetes.Container::{ , container = Kubernetes.Container::{
, name = "merger" , name = "merger"
, image = zuul-image "merger" , image = zuul-image "merger"
, args = [ "zuul-merger", "-d" ] , args = Some [ "zuul-merger", "-d" ]
, imagePullPolicy = Some "IfNotPresent" , imagePullPolicy = Some "IfNotPresent"
, env = zuul-env , env = Some zuul-env
, volumeMounts = , volumeMounts = Some
mkVolumeMount ( mkVolumeMount
(merger-volumes # zuul-data-dir) (merger-volumes # zuul-data-dir)
)
} }
} }
) )
@ -852,17 +861,18 @@ in \(input : Input)
, container = Kubernetes.Container::{ , container = Kubernetes.Container::{
, name = "launcher" , name = "launcher"
, image = nodepool-image "launcher" , image = nodepool-image "launcher"
, args = , args = Some
[ "sh" [ "sh"
, "-c" , "-c"
, shard-config , shard-config
++ "nodepool-launcher -d -c /var/lib/nodepool/config.yaml" ++ "nodepool-launcher -d -c /var/lib/nodepool/config.yaml"
] ]
, imagePullPolicy = Some "IfNotPresent" , imagePullPolicy = Some "IfNotPresent"
, env = nodepool-env , env = Some nodepool-env
, volumeMounts = , volumeMounts = Some
mkVolumeMount ( mkVolumeMount
(nodepool-volumes # nodepool-data-dir) (nodepool-volumes # nodepool-data-dir)
)
} }
} }
) )

4
playbooks/files/cr_spec.yaml
View File

@ -1,8 +1,8 @@
# Render kubernetes resources using: # Render kubernetes resources using:
# INPUT=$(yaml-to-dhall "(./conf/zuul/input.dhall).Input.Type" < playbooks/files/cr_spec.yaml) # INPUT=$(yaml-to-dhall "(./conf/zuul/input.dhall).Input.Type" < playbooks/files/cr_spec.yaml)
# dhall-to-yaml --omit-empty --explain <<< "(./conf/zuul/resources.dhall ($INPUT)).Components.Zuul.Scheduler" # dhall-to-yaml --explain <<< "(./conf/zuul/resources.dhall ($INPUT)).Components.Zuul.Scheduler"
# Or # Or
# dhall-to-yaml --omit-empty --explain <<< "(./conf/zuul/resources.dhall ($INPUT)).List" # dhall-to-yaml --explain <<< "(./conf/zuul/resources.dhall ($INPUT)).List"
executor: executor:
count: 1 count: 1

2
roles/zuul/library/dhall_to_json.py
View File

@ -23,7 +23,7 @@ from ansible.module_utils.basic import AnsibleModule # type: ignore
def run(expression: str) -> Any: def run(expression: str) -> Any:
proc = subprocess.Popen( proc = subprocess.Popen(
['dhall-to-json', '--omit-empty', '--explain'], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) ['dhall-to-json', '--explain'], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = proc.communicate(expression.encode('utf-8')) stdout, stderr = proc.communicate(expression.encode('utf-8'))
if stderr: if stderr:
return dict(failed=True, msg=stderr.decode('utf-8')) return dict(failed=True, msg=stderr.decode('utf-8'))