From 74af40376bd6f7af6dea401fa75e964e73f00b33 Mon Sep 17 00:00:00 2001 From: Scott Hussey Date: Wed, 5 Sep 2018 15:54:37 -0500 Subject: [PATCH] MAAS support for pod mobility Upgrades to the MAAS chart to allow for the Pods running the rack and region services to work across all control plane hosts. Change-Id: I84c856599a1122a2b4a64242a7cea357887b0462 --- .../templates/bin/_maas-ingress-errors.sh.tpl | 30 +++ .../maas/templates/bin/_maas-ingress.sh.tpl | 41 ++++ .../templates/bin/_maas-vip-configure.sh.tpl | 60 +++++ charts/maas/templates/configmap-bin.yaml | 6 + charts/maas/templates/configmap-ingress.yaml | 38 +++ .../templates/deployment-ingress-errors.yaml | 63 +++++ .../maas/templates/etc/_curtin_userdata.tpl | 5 +- charts/maas/templates/etc/_regiond.conf.tpl | 6 +- charts/maas/templates/ingress-region.yaml | 35 +++ charts/maas/templates/job-import.yaml | 4 +- charts/maas/templates/secret-admin-user.yaml | 2 +- .../maas/templates/service-ingress-error.yaml | 28 +++ charts/maas/templates/service-rack.yaml | 7 - charts/maas/templates/service-region.yaml | 29 ++- charts/maas/templates/statefulset-rack.yaml | 221 +++++++++++++++++- charts/maas/templates/statefulset-region.yaml | 8 +- .../maas/templates/tests/test-maas-init.yaml | 2 +- charts/maas/values.yaml | 116 ++++++--- tools/helm_install.sh | 2 +- tools/helm_tk.sh | 2 +- 20 files changed, 633 insertions(+), 72 deletions(-) create mode 100644 charts/maas/templates/bin/_maas-ingress-errors.sh.tpl create mode 100644 charts/maas/templates/bin/_maas-ingress.sh.tpl create mode 100644 charts/maas/templates/bin/_maas-vip-configure.sh.tpl create mode 100644 charts/maas/templates/configmap-ingress.yaml create mode 100644 charts/maas/templates/deployment-ingress-errors.yaml create mode 100644 charts/maas/templates/ingress-region.yaml create mode 100644 charts/maas/templates/service-ingress-error.yaml delete mode 100644 charts/maas/templates/service-rack.yaml diff --git a/charts/maas/templates/bin/_maas-ingress-errors.sh.tpl b/charts/maas/templates/bin/_maas-ingress-errors.sh.tpl new file mode 100644 index 0000000..cca1bc4 --- /dev/null +++ b/charts/maas/templates/bin/_maas-ingress-errors.sh.tpl @@ -0,0 +1,30 @@ +#!/bin/sh + +{{/* +Copyright 2018 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License.*/}} + +set -ex +COMMAND="${@:-start}" + +if [ "x${COMMAND}" == "xstart" ]; then + if [[ -z "${BIND_PORT}" ]] + then + exec /server + else + exec /server -port ${BIND_PORT} + fi +elif [ "x${COMMAND}" == "xstop" ]; then + kill -TERM 1 +fi diff --git a/charts/maas/templates/bin/_maas-ingress.sh.tpl b/charts/maas/templates/bin/_maas-ingress.sh.tpl new file mode 100644 index 0000000..5dda64a --- /dev/null +++ b/charts/maas/templates/bin/_maas-ingress.sh.tpl @@ -0,0 +1,41 @@ +#!/bin/bash + +{{/* + Copyright 2018 The Openstack-Helm Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License.*/}} + +set -ex + +COMMAND="${1:-start}" + +function start () { + exec /usr/bin/dumb-init \ + /nginx-ingress-controller \ + --http-port="${HTTP_PORT}" \ + --watch-namespace="${POD_NAMESPACE}" \ + --https-port="${HTTPS_PORT}" \ + --status-port="${STATUS_PORT}" \ + --healthz-port="${HEALTHZ_PORT}" \ + --election-id=${RELEASE_NAME} \ + --ingress-class=maas-ingress \ + --default-backend-service=${POD_NAMESPACE}/${ERROR_PAGE_SERVICE} \ + --configmap=${POD_NAMESPACE}/maas-ingress-config \ + --tcp-services-configmap=${POD_NAMESPACE}/maas-ingress-services-tcp +} + +function stop () { + kill -TERM 1 +} + +$COMMAND diff --git a/charts/maas/templates/bin/_maas-vip-configure.sh.tpl b/charts/maas/templates/bin/_maas-vip-configure.sh.tpl new file mode 100644 index 0000000..f1f6285 --- /dev/null +++ b/charts/maas/templates/bin/_maas-vip-configure.sh.tpl @@ -0,0 +1,60 @@ +#!/bin/bash + +{{/* +Copyright 2018 The Openstack-Helm Authors. +Copyright (c) 2018 AT&T Intellectual Property. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License.*/}} + +set -ex + +COMMAND="${@:-start}" + +function kernel_modules () { + chroot /mnt/host-rootfs modprobe dummy +} + +function test_vip () { + ip addr show ${interface} | \ + awk "/inet / && /${interface}/{print \$2 }" | \ + awk -F '/' '{ print $1 }' | \ + grep -q "${addr%/*}" +} + +function start () { + kernel_modules + ip link show ${interface} > /dev/null || ip link add ${interface} type dummy + if ! test_vip; then + ip addr add ${addr} dev ${interface} + fi + ip link set ${interface} up +} + +function sleep () { + exec /usr/bin/dumb-init bash -c "while :; do sleep 2073600; done" +} + +function stop () { + ip link show ${interface} > /dev/null || exit 0 + if test_vip; then + ip addr del ${addr} dev ${interface} + fi + if [ "$(ip address show ${interface} | \ + awk "/inet / && /${interface}/{print \$2 }" | \ + wc -l)" -le "0" ]; then + ip link set ${interface} down + ip link del ${interface} + fi +} + +$COMMAND diff --git a/charts/maas/templates/configmap-bin.yaml b/charts/maas/templates/configmap-bin.yaml index 7dae307..2f875ee 100644 --- a/charts/maas/templates/configmap-bin.yaml +++ b/charts/maas/templates/configmap-bin.yaml @@ -41,3 +41,9 @@ data: {{ tuple "bin/_maas-test.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} ntpd.sh: | {{ tuple "bin/_ntpd.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + maas-ingress: | +{{ tuple "bin/_maas-ingress.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + maas-ingress-errors: | +{{ tuple "bin/_maas-ingress-errors.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + maas-vip-configure: | +{{ tuple "bin/_maas-vip-configure.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} diff --git a/charts/maas/templates/configmap-ingress.yaml b/charts/maas/templates/configmap-ingress.yaml new file mode 100644 index 0000000..e289e27 --- /dev/null +++ b/charts/maas/templates/configmap-ingress.yaml @@ -0,0 +1,38 @@ +{{/* +Copyright (c) 2018 AT&T Intellectual Property. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_ingress }} +{{ $bind_address_cidr := .Values.network.maas_ingress.addr | splitList "/" }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: maas-ingress-services-tcp +data: + {{ tuple "maas_region" "public" "region_api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}: "{{- .Release.Namespace -}}/{{- tuple "maas_region" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" -}}:region-api" + {{ tuple "maas_region" "public" "region_proxy" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}: "{{- .Release.Namespace -}}/{{- tuple "maas_region" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" -}}:region-proxy" +... +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: maas-ingress-config +data: + enable-underscores-in-headers: "true" + bind-address: {{ index $bind_address_cidr 0 | quote }} + diable-ipv6: "true" +... +{{- end }} diff --git a/charts/maas/templates/deployment-ingress-errors.yaml b/charts/maas/templates/deployment-ingress-errors.yaml new file mode 100644 index 0000000..a0381dc --- /dev/null +++ b/charts/maas/templates/deployment-ingress-errors.yaml @@ -0,0 +1,63 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. +Copyright (c) 2018 AT&T Intellectual Property. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.rack_deployment }} +{{- $envAll := . }} +{{- $serviceAccountName := "maas-ingress-errors" }} +{{- $mounts_maas_rack := .Values.pod.mounts.maas_rack }} +{{- $mounts_maas_rack_init := .Values.pod.mounts.maas_rack.init_container }} + +{{ tuple $envAll "rack_controller" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: apps/v1beta1 +kind: Deployment +metadata: + name: maas-ingress-errors +spec: + serviceName: maas-rack + replicas: {{ .Values.pod.replicas.ingress_errors }} + template: + metadata: + labels: +{{ tuple $envAll "maas" "ingress-errors" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + spec: + serviceAccountName: {{ $serviceAccountName }} + nodeSelector: + {{ .Values.labels.rack.node_selector_key }}: {{ .Values.labels.rack.node_selector_value }} + dnsPolicy: ClusterFirst + containers: + - name: maas-ingress-errors + image: {{ .Values.images.tags.error_pages }} + imagePullPolicy: {{ .Values.images.pull_policy }} +{{ tuple $envAll $envAll.Values.pod.resources.maas_ingress_errors | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + command: + - /tmp/maas-ingress-errors.sh + - start + env: + - name: BIND_PORT + value: {{ tuple "maas_ingress" "podport" "error_pages" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} + volumeMounts: + - mountPath: /tmp/maas-ingress-errors.sh + name: maas-bin + subPath: maas-ingress-errors + readOnly: true + volumes: + - name: maas-bin + configMap: + name: maas-bin + defaultMode: 0555 +{{- end }} diff --git a/charts/maas/templates/etc/_curtin_userdata.tpl b/charts/maas/templates/etc/_curtin_userdata.tpl index 9f65185..84ee704 100644 --- a/charts/maas/templates/etc/_curtin_userdata.tpl +++ b/charts/maas/templates/etc/_curtin_userdata.tpl @@ -1,3 +1,4 @@ +{{- $drydock_url := tuple "physicalprovisioner" "public" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" -}} #cloud-config debconf_selections: maas: | @@ -34,8 +35,8 @@ def find_ba_key(n): return False {{ "}}" }} {{ "{{" }}py: ba_key = find_ba_key(node){{ "}}" }} -{{ "{{" }}py: ba_units_url = ''.join([{{ .Values.conf.drydock.bootaction_url | quote }},node.hostname,'/units']){{ "}}" }} -{{ "{{" }}py: ba_files_url = ''.join([{{ .Values.conf.drydock.bootaction_url | quote }},node.hostname,'/files']){{ "}}" }} +{{ "{{" }}py: ba_units_url = ''.join([{{ quote $drydock_url }},'/bootactions/nodes/',node.hostname,'/units']){{ "}}" }} +{{ "{{" }}py: ba_files_url = ''.join([{{ quote $drydock_url }},'/bootactions/nodes/',node.hostname,'/files']){{ "}}" }} {{ "{{" }}if ba_key{{ "}}" }} drydock_00: ["sh", "-c", "echo Installing Drydock Boot Actions."] drydock_01: ["curtin", "in-target", "--", "wget", "--no-proxy", "--header=X-Bootaction-Key: {{ "{{" }}ba_key{{ "}}" }}", "{{ "{{" }}ba_units_url{{ "}}" }}", "-O", "/tmp/bootaction-units.tar.gz"] diff --git a/charts/maas/templates/etc/_regiond.conf.tpl b/charts/maas/templates/etc/_regiond.conf.tpl index c7b6cf8..83a2a5b 100644 --- a/charts/maas/templates/etc/_regiond.conf.tpl +++ b/charts/maas/templates/etc/_regiond.conf.tpl @@ -14,12 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. */}} -{{- if empty .Values.conf.maas.url.maas_url -}} -{{- tuple "maas_region_ui" "default" "region_ui" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | set .Values.conf.maas.url "maas_url" | quote | trunc 0 -}} -{{- end }} - database_host: {{ tuple "maas_db" "internal" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }} database_name: {{ .Values.endpoints.maas_db.auth.user.database }} database_pass: {{ .Values.endpoints.maas_db.auth.user.password }} database_user: {{ .Values.endpoints.maas_db.auth.user.username }} -maas_url: {{ .Values.conf.maas.url.maas_url }} +maas_url: {{ tuple "maas_region" "public" "region_api" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | quote }} diff --git a/charts/maas/templates/ingress-region.yaml b/charts/maas/templates/ingress-region.yaml new file mode 100644 index 0000000..9801fae --- /dev/null +++ b/charts/maas/templates/ingress-region.yaml @@ -0,0 +1,35 @@ +{{/* +# Copyright (c) 2018 AT&T Intellectual Property. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +*/}} + +{{- if and .Values.manifests.ingress_region .Values.network.region_api.ingress.public }} +--- +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: maas-region-api + annotations: + kubernetes.io/ingress.class: {{ .Values.network.region_api.ingress.classes.cluster | quote }} +spec: + rules: + - host: {{ tuple "maas_region" "public" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }} + http: + paths: + - path: / + backend: + serviceName: {{ tuple "maas_region" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + servicePort: region-api +... +{{ end }} diff --git a/charts/maas/templates/job-import.yaml b/charts/maas/templates/job-import.yaml index 0a05558..8077338 100644 --- a/charts/maas/templates/job-import.yaml +++ b/charts/maas/templates/job-import.yaml @@ -56,7 +56,7 @@ spec: - name: TRY_LIMIT value: {{ .Values.jobs.import_boot_resources.try_limit | quote }} - name: MAAS_ENDPOINT - value: {{ tuple "maas_region_ui" "default" "region_ui" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" }} + value: {{ tuple "maas_region" "public" "region_api" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | quote }} - name: MAAS_PROXY_ENABLED value: {{ .Values.conf.maas.proxy.proxy_enabled | quote }} - name: MAAS_PEER_PROXY_ENABLED @@ -64,7 +64,7 @@ spec: - name: MAAS_PROXY_SERVER value: {{ .Values.conf.maas.proxy.proxy_server }} - name: MAAS_INTERNAL_PROXY_PORT - value: {{ .Values.network.port.service_proxy | quote }} + value: {{ tuple "maas_region" "default" "region_proxy" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | quote }} - name: MAAS_HTTP_BOOT value: {{ .Values.conf.maas.http_boot | quote }} - name: MAAS_NTP_SERVERS diff --git a/charts/maas/templates/secret-admin-user.yaml b/charts/maas/templates/secret-admin-user.yaml index 87e8599..62d9794 100644 --- a/charts/maas/templates/secret-admin-user.yaml +++ b/charts/maas/templates/secret-admin-user.yaml @@ -17,7 +17,7 @@ limitations under the License. {{- $envAll := . }} {{- range $key1, $userClass := tuple "admin" }} {{- $secretName := index $envAll.Values.secrets.maas_users $userClass }} -{{- $auth := index $envAll.Values.endpoints.maas_region_ui.auth $userClass }} +{{- $auth := index $envAll.Values.endpoints.maas_region.auth $userClass }} --- apiVersion: v1 kind: Secret diff --git a/charts/maas/templates/service-ingress-error.yaml b/charts/maas/templates/service-ingress-error.yaml new file mode 100644 index 0000000..4ff0712 --- /dev/null +++ b/charts/maas/templates/service-ingress-error.yaml @@ -0,0 +1,28 @@ +{{/* +Copyright (c) 2018 AT&T Intellectual Property. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ tuple "maas_ingress" "error_pages" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} +spec: + ports: + - name: error-pages + port: {{ tuple "maas_ingress" "default" "error_pages" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + targetPort: {{ tuple "maas_ingress" "podport" "error_pages" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + selector: +{{ tuple . "maas" "ingress-errors" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +... diff --git a/charts/maas/templates/service-rack.yaml b/charts/maas/templates/service-rack.yaml deleted file mode 100644 index b24aef1..0000000 --- a/charts/maas/templates/service-rack.yaml +++ /dev/null @@ -1,7 +0,0 @@ ---- -apiVersion: v1 -kind: Service -metadata: - name: maas-rack -spec: - clusterIP: 'None' diff --git a/charts/maas/templates/service-region.yaml b/charts/maas/templates/service-region.yaml index 646a0c6..06241ec 100644 --- a/charts/maas/templates/service-region.yaml +++ b/charts/maas/templates/service-region.yaml @@ -8,7 +8,6 @@ You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software - app: maas-region distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and @@ -20,23 +19,23 @@ limitations under the License. apiVersion: v1 kind: Service metadata: - name: {{ tuple "maas_region_ui" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + name: {{ tuple "maas_region" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} spec: ports: - - name: r-ui - port: {{ .Values.network.port.service_gui }} - targetPort: {{ .Values.network.port.service_gui_target }} - {{ if .Values.network.gui.node_port.enabled }} - nodePort: {{ .Values.network.gui.node_port.port }} - {{ end }} - - port: {{ .Values.network.port.service_proxy }} - targetPort: {{ .Values.network.port.service_proxy }} - {{ if .Values.network.proxy.node_port.enabled }} - nodePort: {{ .Values.network.port.service_proxy }} - {{ end }} - name: proxy + - name: region-api + port: {{ tuple "maas_region" "internal" "region_api" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + targetPort: {{ tuple "maas_region" "podport" "region_api" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + {{ if .Values.network.region_api.node_port.enabled }} + nodePort: {{ tuple "maas_region" "nodeport" "region_api" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + {{ end }} + - name: region-proxy + port: {{ tuple "maas_region" "internal" "region_proxy" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + targetPort: {{ tuple "maas_region" "podport" "region_proxy" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + {{ if .Values.network.region_proxy.node_port.enabled }} + nodePort: {{ tuple "maas_region" "nodeport" "region_proxy" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + {{ end }} selector: {{ tuple $envAll "maas" "region" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} - {{ if .Values.network.proxy.node_port.enabled }} + {{ if or .Values.network.region_proxy.node_port.enabled .Values.network.region_api.node_port.enabled }} type: NodePort {{ end }} diff --git a/charts/maas/templates/statefulset-rack.yaml b/charts/maas/templates/statefulset-rack.yaml index 3086222..6b3832f 100644 --- a/charts/maas/templates/statefulset-rack.yaml +++ b/charts/maas/templates/statefulset-rack.yaml @@ -16,9 +16,6 @@ limitations under the License. */}} {{- if .Values.manifests.rack_deployment }} -{{- if empty .Values.conf.maas.url.maas_url -}} -{{- tuple "maas_region_ui" "default" "region_ui" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | set .Values.conf.maas.url "maas_url" | quote | trunc 0 -}} -{{- end -}} {{- $envAll := . }} {{- $serviceAccountName := "maas-rack" }} {{- $mounts_maas_rack := .Values.pod.mounts.maas_rack }} @@ -26,6 +23,125 @@ limitations under the License. {{ tuple $envAll "rack_controller" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} --- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: {{ $serviceAccountName }} +rules: + - apiGroups: + - "" + resources: + - configmaps + - endpoints + - nodes + - pods + - secrets + verbs: + - list + - watch + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + - apiGroups: + - "" + resources: + - services + verbs: + - get + - list + - watch + - apiGroups: + - "extensions" + resources: + - ingresses + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - apiGroups: + - "extensions" + resources: + - ingresses/status + verbs: + - update +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: {{ $serviceAccountName }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ $serviceAccountName }} +subjects: + - kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ $envAll.Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: Role +metadata: + name: {{ $serviceAccountName }} + namespace: {{ $envAll.Release.Namespace }} +rules: + - apiGroups: + - "" + resources: + - configmaps + - pods + - secrets + - namespaces + verbs: + - get + - apiGroups: + - "" + resources: + - configmaps + resourceNames: + - {{ printf "%s-maas-ingress" .Release.Name | quote }} + verbs: + - get + - update + - apiGroups: + - "" + resources: + - configmaps + verbs: + - create + - apiGroups: + - "" + resources: + - endpoints + verbs: + - get + - create + - update +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: RoleBinding +metadata: + name: {{ $serviceAccountName }} + namespace: {{ $envAll.Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ $serviceAccountName }} +subjects: + - kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ $envAll.Release.Namespace }} +--- +--- apiVersion: apps/v1beta1 kind: StatefulSet metadata: @@ -47,21 +163,107 @@ spec: spec: serviceAccountName: {{ $serviceAccountName }} affinity: -{{ tuple $envAll "maas" "rack" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }} +{{- tuple $envAll "maas" "rack" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }} nodeSelector: {{ .Values.labels.rack.node_selector_key }}: {{ .Values.labels.rack.node_selector_value }} hostNetwork: true - dnsPolicy: ClusterFirst + dnsPolicy: ClusterFirstWithHostNet initContainers: {{ tuple $envAll "rack_controller" $mounts_maas_rack_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + - name: maas-ingress-vip-init + image: {{ .Values.images.tags.ingress }} + imagePullPolicy: {{ .Values.images.pull_policy }} +{{ tuple $envAll $envAll.Values.pod.resources.maas_ingress_vip | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + securityContext: + capabilities: + add: + - 'NET_ADMIN' + - 'SYS_MODULE' + runAsUser: 0 + command: + - /tmp/maas-vip-configure.sh + - start + env: +{{ include "helm-toolkit.utils.to_k8s_env_vars" .Values.network.maas_ingress | indent 12 }} + volumeMounts: + - mountPath: /tmp/maas-vip-configure.sh + name: maas-bin + subPath: maas-vip-configure + readOnly: true + - mountPath: /mnt/host-rootfs + name: host-rootfs + readOnly: true containers: + - name: maas-ingress-vip + image: {{ .Values.images.tags.ingress }} + imagePullPolicy: {{ .Values.images.pull_policy }} +{{ tuple $envAll $envAll.Values.pod.resources.maas_ingress_vip | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + securityContext: + capabilities: + add: + - 'NET_ADMIN' + runAsUser: 0 + command: + - /tmp/maas-vip-configure.sh + - sleep + env: +{{ include "helm-toolkit.utils.to_k8s_env_vars" .Values.network.maas_ingress | indent 12 }} + volumeMounts: + - mountPath: /tmp/maas-vip-configure.sh + name: maas-bin + subPath: maas-vip-configure + readOnly: true + lifecycle: + preStop: + exec: + command: + - /tmp/maas-vip-configure.sh + - stop + - name: maas-ingress + image: {{ .Values.images.tags.ingress }} + imagePullPolicy: {{ .Values.images.pull_policy }} +{{ tuple $envAll $envAll.Values.pod.resources.maas_ingress | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + securityContext: + capabilities: + add: + - 'NET_BIND_SERVICE' + runAsUser: 0 + command: + - /tmp/maas-ingress.sh + - start + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: RELEASE_NAME + value: {{ .Release.Name | quote }} + - name: HTTP_PORT + value: "8808" + - name: HTTPS_PORT + value: "8543" + - name: HEALTHZ_PORT + value: {{ tuple "maas_ingress" "podport" "healthz" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} + - name: STATUS_PORT + value: {{ tuple "maas_ingress" "podport" "status" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} + - name: ERROR_PAGE_SERVICE + value: {{ tuple "maas_ingress" "error_pages" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" | quote }} + volumeMounts: + - mountPath: /tmp/maas-ingress.sh + name: maas-bin + subPath: maas-ingress + readOnly: true - name: maas-rack image: {{ .Values.images.tags.maas_rack }} imagePullPolicy: {{ .Values.images.pull_policy }} tty: true env: - name: MAAS_ENDPOINT - value: {{ .Values.conf.maas.url.maas_url }} + value: {{ tuple "maas_region" "public" "region_api" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | quote }} - name: MAAS_REGION_SECRET valueFrom: secretKeyRef: @@ -108,6 +310,10 @@ spec: mountPath: /var/lib/maas subPath: home readOnly: false + - name: maas-etc + mountPath: /etc/nsswitch.conf + subPath: nsswitch.conf + readOnly: true {{- if .Values.manifests.secret_ssh_key }} - name: priv-key subPath: PRIVATE_KEY @@ -118,6 +324,9 @@ spec: - name: host-sys-fs-cgroup hostPath: path: /sys/fs/cgroup + - name: host-rootfs + hostPath: + path: / - name: pod-run emptyDir: {} - name: pod-run-lock diff --git a/charts/maas/templates/statefulset-region.yaml b/charts/maas/templates/statefulset-region.yaml index 0f72dd8..7593761 100644 --- a/charts/maas/templates/statefulset-region.yaml +++ b/charts/maas/templates/statefulset-region.yaml @@ -56,11 +56,13 @@ spec: tty: true {{ tuple $envAll $envAll.Values.pod.resources.maas_region | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} ports: - - name: r-ui - containerPort: {{ .Values.network.port.region_container }} + - name: region-api + containerPort: {{ tuple "maas_region" "podport" "region_api" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + - name: region-proxy + containerPort: {{ tuple "maas_region" "podport" "region_proxy" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }} readinessProbe: tcpSocket: - port: {{ .Values.network.port.region_container }} + port: {{ tuple "maas_region" "podport" "region_api" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }} securityContext: privileged: true command: diff --git a/charts/maas/templates/tests/test-maas-init.yaml b/charts/maas/templates/tests/test-maas-init.yaml index f574c52..61240ec 100644 --- a/charts/maas/templates/tests/test-maas-init.yaml +++ b/charts/maas/templates/tests/test-maas-init.yaml @@ -34,7 +34,7 @@ spec: - name: "{{ .Release.Name }}-api-test" env: - name: 'MAAS_URL' - value: {{ tuple "maas_region_ui" "internal" "region_ui" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | quote }} + value: {{ tuple "maas_region" "internal" "region_api" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | quote }} - name: 'MAAS_API_KEY' valueFrom: secretKeyRef: diff --git a/charts/maas/values.yaml b/charts/maas/values.yaml index f2cd3b0..98f72aa 100644 --- a/charts/maas/values.yaml +++ b/charts/maas/values.yaml @@ -20,10 +20,8 @@ dependencies: static: rack_controller: - jobs: - - maas-db-sync services: - - service: maas_region_ui + - service: maas_region endpoint: internal region_controller: jobs: @@ -42,23 +40,19 @@ dependencies: jobs: - maas-db-sync services: - - service: maas_region_ui + - service: maas_region endpoint: internal - service: maas_db endpoint: internal import_resources: - jobs: - - maas-db-sync services: - - service: maas_region_ui + - service: maas_region endpoint: internal - service: maas_db endpoint: internal export_api_key: - jobs: - - maas-db-sync services: - - service: maas_region_ui + - service: maas_region endpoint: internal - service: maas_db endpoint: internal @@ -68,6 +62,8 @@ manifests: rack_deployment: true test_maas_api: true secret_ssh_key: false + ingress_region: true + configmap_ingress: true images: tags: @@ -79,6 +75,8 @@ images: export_api_key: quay.io/airshipit/maas-region-controller:latest maas_cache: quay.io/airshipit/sstream-cache:latest dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.3.1 + ingress: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.9.0 + error_pages: gcr.io/google_containers/defaultbackend:1.0 pull_policy: IfNotPresent local_registry: # TODO(portdirect): this chart does not yet support local image cacheing @@ -102,20 +100,23 @@ labels: node_selector_value: enabled network: - proxy: + maas_ingress: + mode: routed + interface: maas-vip + addr: 172.18.0.2/32 + region_proxy: node_port: - enabled: true - gui: + enabled: false + region_api: + ingress: + public: true + classes: + namespace: "maas-ingress" + cluster: "maas-ingress" + annotations: + nginx.ingress.kubernetes.io/rewrite-target: "/" node_port: - enabled: true - port: 31900 - port: - region_container: 80 - service_gui: 80 - service_gui_target: 80 - service_proxy: 31800 - db_service: 5432 - db_service_target: 5432 + enabled: false storage: rackd: @@ -145,8 +146,6 @@ conf: override: append: http_boot: true - url: - maas_url: null ntp: # These options allow you to mock out the ntpd binary within the container # by overwriting it with a script that simply sleeps - this is useful in @@ -266,6 +265,27 @@ pod: requests: memory: "128Mi" cpu: "500m" + maas_ingress_vip: + limits: + memory: "128Mi" + cpu: "500m" + requests: + memory: "128Mi" + cpu: "500m" + maas_ingress: + limits: + memory: "128Mi" + cpu: "500m" + requests: + memory: "128Mi" + cpu: "500m" + maas_ingress_errors: + limits: + memory: "128Mi" + cpu: "500m" + requests: + memory: "128Mi" + cpu: "500m" jobs: db_init: requests: @@ -305,6 +325,24 @@ pod: endpoints: cluster_domain_suffix: cluster.local + maas_ingress: + hosts: + default: maas-ingress + error_pages: maas-ingress-error + host_fqdn_override: + default: null + port: + http: + default: 80 + https: + default: 443 + error_pages: + default: 8080 + podport: 10080 + healthz: + podport: 10254 + status: + podport: 18080 maas_db: auth: admin: @@ -322,23 +360,45 @@ endpoints: default: 5432 host_fqdn_override: default: null - maas_region_ui: - name: maas-region-ui + maas_region: + name: maas-region auth: admin: username: admin password: admin email: none@none hosts: - default: maas-region-ui + default: maas-region public: maas path: default: /MAAS scheme: default: 'http' port: - region_ui: + region_api: default: 80 + nodeport: 31900 + podport: 80 public: 80 + region_proxy: + default: 8000 + # podport and public need to be the same as of MAAS 2.3.4, so + # comment them out and let the default rule + # podport: 8000 + # public: 8000 + host_fqdn_override: + default: null + physicalprovisioner: + name: drydock + hosts: + default: drydock-api + port: + api: + default: 9000 + nodeport: 31900 + path: + default: /api/v1.0 + scheme: + default: http host_fqdn_override: default: null diff --git a/tools/helm_install.sh b/tools/helm_install.sh index 0393ba1..57ac72d 100755 --- a/tools/helm_install.sh +++ b/tools/helm_install.sh @@ -17,7 +17,7 @@ set -x HELM=$1 -HELM_ARTIFACT_URL=${HELM_ARTIFACT_URL:-"https://storage.googleapis.com/kubernetes-helm/helm-v2.9.1-linux-amd64.tar.gz"} +HELM_ARTIFACT_URL=${HELM_ARTIFACT_URL:-"https://storage.googleapis.com/kubernetes-helm/helm-v2.10.0-linux-amd64.tar.gz"} function install_helm_binary { diff --git a/tools/helm_tk.sh b/tools/helm_tk.sh index 9e0043e..9e3e6a3 100755 --- a/tools/helm_tk.sh +++ b/tools/helm_tk.sh @@ -18,7 +18,7 @@ HELM=$1 HTK_REPO=${HTK_REPO:-"https://github.com/openstack/openstack-helm-infra"} HTK_PATH=${HTK_PATH:-""} -HTK_STABLE_COMMIT=${HTK_COMMIT:-"master"} +HTK_STABLE_COMMIT=${HTK_COMMIT:-"4cd00f3ac539f625e7cd9733ae46232b2082027a"} DEP_UP_LIST=${DEP_UP_LIST:-"maas"} if [[ ! -z $(echo $http_proxy) ]]