From 51c70e48dff173281a77d374d87af2c49caa6348 Mon Sep 17 00:00:00 2001 From: "Anselme, Schubert" Date: Thu, 17 Aug 2023 13:00:33 +0000 Subject: [PATCH] Deprecating the Ingress Class Annotation This PS replaces deprecated kubernetes.io/ingress.class annotation with spec.ingressClassName field that is a reference to an IngressClass resource that contains additional Ingress configuration, including the name of the Ingress controller. https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#deprecating-the-ingress-class-annotation Change-Id: I9953d966b4f9f7b1692b39f36f434f5055317025 Co-authored-by: Sergiy Markin Co-authored-by: Leointii Istomin Signed-off-by: Anselme, Schubert (sa246v) --- helm-toolkit/Chart.yaml | 2 +- helm-toolkit/templates/manifests/_ingress.tpl | 22 ++++++------- ingress/Chart.yaml | 2 +- .../templates/bin/_ingress-controller.sh.tpl | 1 + ingress/templates/deployment-ingress.yaml | 14 ++++++-- ingress/templates/ingress-class.yaml | 32 +++++++++++++++++++ ingress/templates/ingress.yaml | 7 ++-- ingress/values.yaml | 12 ++++++- releasenotes/notes/helm-toolkit.yaml | 1 + releasenotes/notes/ingress.yaml | 1 + roles/build-helm-packages/defaults/main.yml | 2 +- tools/deployment/apparmor/015-ingress.sh | 8 +++-- tools/deployment/common/020-ingress.sh | 6 ++-- tools/deployment/keystone-auth/020-ingress.sh | 5 +-- .../deployment/keystone-auth/070-keystone.sh | 1 + tools/deployment/multinode/020-ingress.sh | 7 ++-- .../openstack-support/010-ingress.sh | 5 +-- .../openstack-support/030-rabbitmq.sh | 1 + .../100-ceph-radosgateway.sh | 4 +++ .../openstack-support/130-cinder.sh | 1 + .../osh-infra-logging-tls/010-ingress.sh | 5 +-- .../osh-infra-logging/010-ingress.sh | 5 +-- .../030-radosgw-osh-infra.sh | 4 +++ .../osh-infra-logging/050-elasticsearch.sh | 6 +++- .../osh-infra-logging/070-kibana.sh | 1 + tools/deployment/tenant-ceph/020-ingress.sh | 7 ++-- 26 files changed, 122 insertions(+), 40 deletions(-) create mode 100644 ingress/templates/ingress-class.yaml diff --git a/helm-toolkit/Chart.yaml b/helm-toolkit/Chart.yaml index 2df9c7600..c4f39edf9 100644 --- a/helm-toolkit/Chart.yaml +++ b/helm-toolkit/Chart.yaml @@ -15,7 +15,7 @@ apiVersion: v1 appVersion: v1.0.0 description: OpenStack-Helm Helm-Toolkit name: helm-toolkit -version: 0.2.54 +version: 0.2.55 home: https://docs.openstack.org/openstack-helm icon: https://www.openstack.org/themes/openstack/images/project-mascots/OpenStack-Helm/OpenStack_Project_OpenStackHelm_vertical.png sources: diff --git a/helm-toolkit/templates/manifests/_ingress.tpl b/helm-toolkit/templates/manifests/_ingress.tpl index 4c476b2ce..972e42946 100644 --- a/helm-toolkit/templates/manifests/_ingress.tpl +++ b/helm-toolkit/templates/manifests/_ingress.tpl @@ -67,10 +67,10 @@ examples: metadata: name: barbican annotations: - kubernetes.io/ingress.class: "nginx" nginx.ingress.kubernetes.io/rewrite-target: / spec: + ingressClassName: "nginx" rules: - host: barbican http: @@ -108,10 +108,10 @@ examples: metadata: name: barbican-namespace-fqdn annotations: - kubernetes.io/ingress.class: "nginx" nginx.ingress.kubernetes.io/rewrite-target: / spec: + ingressClassName: "nginx" tls: - secretName: barbican-tls-public hosts: @@ -133,10 +133,10 @@ examples: metadata: name: barbican-cluster-fqdn annotations: - kubernetes.io/ingress.class: "nginx-cluster" nginx.ingress.kubernetes.io/rewrite-target: / spec: + ingressClassName: "nginx-cluster" tls: - secretName: barbican-tls-public hosts: @@ -202,10 +202,10 @@ examples: metadata: name: barbican annotations: - kubernetes.io/ingress.class: "nginx" nginx.ingress.kubernetes.io/rewrite-target: / spec: + ingressClassName: "nginx" tls: - secretName: barbican-tls-public hosts: @@ -302,12 +302,12 @@ examples: metadata: name: barbican annotations: - kubernetes.io/ingress.class: "nginx" cert-manager.io/issuer: ca-issuer certmanager.k8s.io/issuer: ca-issuer nginx.ingress.kubernetes.io/backend-protocol: https nginx.ingress.kubernetes.io/secure-backends: "true" spec: + ingressClassName: "nginx" tls: - secretName: barbican-tls-public-certmanager hosts: @@ -404,12 +404,12 @@ examples: metadata: name: barbican annotations: - kubernetes.io/ingress.class: "nginx" cert-manager.io/cluster-issuer: ca-issuer certmanager.k8s.io/cluster-issuer: ca-issuer nginx.ingress.kubernetes.io/backend-protocol: https nginx.ingress.kubernetes.io/secure-backends: "true" spec: + ingressClassName: "nginx" tls: - secretName: barbican-tls-public-certmanager hosts: @@ -488,10 +488,10 @@ examples: metadata: name: grafana annotations: - kubernetes.io/ingress.class: "nginx" nginx.ingress.kubernetes.io/rewrite-target: / spec: + ingressClassName: "nginx" rules: - host: grafana http: @@ -529,10 +529,10 @@ examples: metadata: name: grafana-namespace-fqdn annotations: - kubernetes.io/ingress.class: "nginx" nginx.ingress.kubernetes.io/rewrite-target: / spec: + ingressClassName: "nginx" tls: - secretName: grafana-tls-public hosts: @@ -565,10 +565,10 @@ examples: metadata: name: grafana-cluster-fqdn annotations: - kubernetes.io/ingress.class: "nginx-cluster" nginx.ingress.kubernetes.io/rewrite-target: / spec: + ingressClassName: "nginx-cluster" tls: - secretName: grafana-tls-public hosts: @@ -639,7 +639,6 @@ kind: Ingress metadata: name: {{ $ingressName }} annotations: - kubernetes.io/ingress.class: {{ index $envAll.Values.network $backendService "ingress" "classes" "namespace" | quote }} {{- if $certIssuer }} cert-manager.io/{{ $certIssuerType }}: {{ $certIssuer }} certmanager.k8s.io/{{ $certIssuerType }}: {{ $certIssuer }} @@ -650,6 +649,7 @@ metadata: {{- end }} {{ toYaml (index $envAll.Values.network $backendService "ingress" "annotations") | indent 4 }} spec: + ingressClassName: {{ index $envAll.Values.network $backendService "ingress" "classes" "namespace" | quote }} {{- $host := index $envAll.Values.endpoints ( $backendServiceType | replace "-" "_" ) "hosts" }} {{- if $certIssuer }} {{- $secretName := index $envAll.Values.secrets "tls" ( $backendServiceType | replace "-" "_" ) $backendService $endpoint }} @@ -695,9 +695,9 @@ kind: Ingress metadata: name: {{ printf "%s-%s-%s" $ingressName $ingressController "fqdn" }} annotations: - kubernetes.io/ingress.class: {{ index $envAll.Values.network $backendService "ingress" "classes" $ingressController | quote }} {{ toYaml (index $envAll.Values.network $backendService "ingress" "annotations") | indent 4 }} spec: + ingressClassName: {{ index $envAll.Values.network $backendService "ingress" "classes" $ingressController | quote }} {{- $host := index $envAll.Values.endpoints ( $backendServiceType | replace "-" "_" ) "host_fqdn_override" }} {{- if hasKey $host $endpoint }} {{- $endpointHost := index $host $endpoint }} diff --git a/ingress/Chart.yaml b/ingress/Chart.yaml index 584fe5007..92278e3f8 100644 --- a/ingress/Chart.yaml +++ b/ingress/Chart.yaml @@ -15,7 +15,7 @@ apiVersion: v1 appVersion: v0.42.0 description: OpenStack-Helm Ingress Controller name: ingress -version: 0.2.15 +version: 0.2.16 home: https://github.com/kubernetes/ingress sources: - https://github.com/kubernetes/ingress diff --git a/ingress/templates/bin/_ingress-controller.sh.tpl b/ingress/templates/bin/_ingress-controller.sh.tpl index 45a7023c4..19fb4fcf3 100644 --- a/ingress/templates/bin/_ingress-controller.sh.tpl +++ b/ingress/templates/bin/_ingress-controller.sh.tpl @@ -44,6 +44,7 @@ function start () { --status-port=${PORT_STATUS} \ --default-server-port=${DEFAULT_SERVER_PORT} \ --election-id=${RELEASE_NAME} \ + --controller-class=${CONTROLLER_CLASS} \ --ingress-class=${INGRESS_CLASS} \ --default-backend-service=${POD_NAMESPACE}/${ERROR_PAGE_SERVICE} \ {{- if .Values.conf.default_ssl_certificate.enabled }} diff --git a/ingress/templates/deployment-ingress.yaml b/ingress/templates/deployment-ingress.yaml index 56f169d5f..c1b2c82b7 100644 --- a/ingress/templates/deployment-ingress.yaml +++ b/ingress/templates/deployment-ingress.yaml @@ -15,13 +15,21 @@ limitations under the License. {{- if .Values.manifests.deployment_ingress }} {{- $envAll := . }} -{{- if empty .Values.conf.controller.INGRESS_CLASS -}} +# Evaluate if we are deploying in cluster mode {{- if eq .Values.deployment.mode "cluster" }} +# Check INGRESS_CLASS empty +{{- if empty .Values.conf.controller.INGRESS_CLASS -}} {{- $_ := set .Values.conf.controller "INGRESS_CLASS" .Values.deployment.cluster.class -}} +{{- end }} +# Check CONTROLLER_CLASS empty +{{- if empty .Values.conf.controller.CONTROLLER_CLASS -}} +{{- $_ := set .Values.conf.controller "CONTROLLER_CLASS" .Values.deployment.cluster.controllerClass -}} +{{- end }} +# Set default values for INGRESS_CLASS & CONTROLLER_CLASS if deploying in namespace mode {{- else if eq .Values.deployment.mode "namespace" }} {{- $_ := set .Values.conf.controller "INGRESS_CLASS" "nginx" -}} +{{- $_ := set .Values.conf.controller "CONTROLLER_CLASS" "k8s.io/nginx-ingress" -}} {{- end }} -{{- end -}} {{- $serviceAccountName := printf "%s-%s" .Release.Name "ingress" }} {{ tuple $envAll "ingress" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} @@ -68,6 +76,7 @@ rules: - "networking.k8s.io" resources: - ingresses + - ingressclasses verbs: - get - list @@ -77,6 +86,7 @@ rules: - "networking.k8s.io" resources: - ingresses/status + - ingressclasses/status verbs: - update - apiGroups: diff --git a/ingress/templates/ingress-class.yaml b/ingress/templates/ingress-class.yaml new file mode 100644 index 000000000..51461ce9a --- /dev/null +++ b/ingress/templates/ingress-class.yaml @@ -0,0 +1,32 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.ingressClass }} +{{- $envAll := . }} +{{- if empty (index .Values.network.ingress.spec "ingressClassName") }} +{{- $_ := set .Values.network.ingress.spec "ingressClassName" .Values.deployment.cluster.class -}} +{{- end }} +{{- if empty (index .Values.network.ingressClass.spec "controller") }} +{{- $_ := set .Values.network.ingressClass.spec "controller" .Values.deployment.cluster.controllerClass -}} +{{- end }} +--- +apiVersion: networking.k8s.io/v1 +kind: IngressClass +metadata: + labels: + app.kubernetes.io/component: controller + name: {{ index $envAll.Values.network.ingress.spec "ingressClassName" | quote }} +spec: + controller: {{ index $envAll.Values.network.ingressClass.spec "controller" | quote }} +{{- end }} diff --git a/ingress/templates/ingress.yaml b/ingress/templates/ingress.yaml index 1f67c7a70..b424ab55a 100644 --- a/ingress/templates/ingress.yaml +++ b/ingress/templates/ingress.yaml @@ -15,8 +15,8 @@ limitations under the License. {{- if .Values.manifests.ingress }} {{- $envAll := . }} {{- if eq .Values.deployment.mode "namespace" }} -{{- if empty (index .Values.network.ingress.annotations "kubernetes.io/ingress.class") -}} -{{- $_ := set .Values.network.ingress.annotations "kubernetes.io/ingress.class" .Values.deployment.cluster.class -}} +{{- if empty (index .Values.network.ingress.spec "ingressClassName") -}} +{{- $_ := set .Values.network.ingress.spec "ingressClassName" .Values.deployment.cluster.class -}} {{- end -}} {{- $serviceName := tuple "ingress" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" -}} {{- $servicePort := tuple "ingress" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" -}} @@ -25,9 +25,8 @@ apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: {{ .Release.Namespace }}-{{ .Release.Name }} - annotations: -{{ toYaml .Values.network.ingress.annotations | indent 4 }} spec: +{{ toYaml .Values.network.ingress.spec | indent 2 }} rules: - host: {{ printf "%s.%s.svc.%s" "*" .Release.Namespace .Values.endpoints.cluster_domain_suffix | quote }} http: diff --git a/ingress/values.yaml b/ingress/values.yaml index c42cdac4c..09d0e5f55 100644 --- a/ingress/values.yaml +++ b/ingress/values.yaml @@ -21,6 +21,7 @@ deployment: type: Deployment cluster: class: "nginx-cluster" + controllerClass: "k8s.io/nginx-ingress" images: tags: @@ -154,7 +155,12 @@ network: # Use .network.vip.addr as an external IP for the service # Useful if the CNI or provider can set up routes, etc. assign_as_external_ip: false + ingressClass: + spec: + controller: null ingress: + spec: + ingressClassName: null node_port: enabled: false http_port: 30080 @@ -162,7 +168,6 @@ network: annotations: # NOTE(portdirect): if left blank this is populated from # .deployment.cluster.class - kubernetes.io/ingress.class: null nginx.ingress.kubernetes.io/proxy-body-size: "0" nginx.ingress.kubernetes.io/configuration-snippet: | more_set_headers "X-Content-Type-Options: nosniff"; @@ -308,6 +313,10 @@ conf: # .deployment.cluster.class in cluster mode, or set to # "nginx" in namespace mode INGRESS_CLASS: null + # NOTE(portdirect): if left blank this is populated from + # .deployment.cluster.controllerClass in cluster mode, or set to + # "k8s.io/nginx-ingress" in namespace mode + CONTROLLER_CLASS: null ingress: enable-underscores-in-headers: "true" # NOTE(portdirect): if left blank this is populated from @@ -348,6 +357,7 @@ manifests: deployment_ingress: true endpoints_ingress: true ingress: true + ingressClass: true secret_ingress_tls: false secret_dhparam: false service_error: true diff --git a/releasenotes/notes/helm-toolkit.yaml b/releasenotes/notes/helm-toolkit.yaml index 18477ab33..2f002e03d 100644 --- a/releasenotes/notes/helm-toolkit.yaml +++ b/releasenotes/notes/helm-toolkit.yaml @@ -61,4 +61,5 @@ helm-toolkit: - 0.2.52 Decreased random delay to up to 30 seconds and switched remote backup verification protocol to md5 - 0.2.53 Update create db user queries - 0.2.54 Fix dependency resolver to ignore non-existing dependencyKey when dependencyMixinParam is a slice + - 0.2.55 Updated deprecated IngressClass annotation ... diff --git a/releasenotes/notes/ingress.yaml b/releasenotes/notes/ingress.yaml index f0a717080..69b01ab4e 100644 --- a/releasenotes/notes/ingress.yaml +++ b/releasenotes/notes/ingress.yaml @@ -19,4 +19,5 @@ ingress: - 0.2.13 Allow setting node_port for the svc - 0.2.14 Replace node-role.kubernetes.io/master with control-plane - 0.2.15 Update kubernetes registry to registry.k8s.io + - 0.2.16 Updated deprecated IngressClass annotation ... diff --git a/roles/build-helm-packages/defaults/main.yml b/roles/build-helm-packages/defaults/main.yml index aedd82bb1..8e76d2ca6 100644 --- a/roles/build-helm-packages/defaults/main.yml +++ b/roles/build-helm-packages/defaults/main.yml @@ -12,7 +12,7 @@ --- version: - helm: v3.12.0 + helm: v3.12.2 url: helm_repo: https://get.helm.sh ... diff --git a/tools/deployment/apparmor/015-ingress.sh b/tools/deployment/apparmor/015-ingress.sh index 48e2b4600..c63855ef8 100755 --- a/tools/deployment/apparmor/015-ingress.sh +++ b/tools/deployment/apparmor/015-ingress.sh @@ -24,8 +24,8 @@ make ingress #NOTE: Deploy command : ${OSH_INFRA_EXTRA_HELM_ARGS:=""} -#NOTE: Deploy global ingress -tee /tmp/ingress-kube-system.yaml << EOF +#NOTE: Deploy global ingress with IngressClass nginx-cluster +tee /tmp/ingress-kube-system.yaml <