Support MAAS HA

- Create two replicas of rack and region pods
- Use required anti-affinity between rack pods
- Remove the MAAS ingress controller from the rack pod
  and into dedicated deployment
- Update rack registration script to harvest the systemid
  from the underlying host when available

Change-Id: I41e21b7bb5256d04b37a70fbd2088c617b5d239a

charts/maas/templates/bin/_register-rack-controller.sh.tpl

@@ -2,23 +2,80 @@
 
 set -x
 
-if [[ -r ~maas/maas_id && -r ~maas/secret ]]
+# Path where the host's cloud-init data is mounted
-then
+# to source the maas system_id
-  echo "Found existing maas_id and secret, assuming already registered."
+HOST_MOUNT_PATH=${HOST_MOUNT_PATH:-"/host_cloud-init/"}
-  exit 0
-fi
 
-echo "register-rack-controller URL: ${MAAS_ENDPOINT}"
+unregister_maas_rack() {
+  sys_id="$1"
+  echo "Deregister this pod as MAAS rack controller ${sys_id}."
+  maas login local "$MAAS_ENDPOINT" "$MAAS_API_KEY"
+  maas local rack-controller delete "$sys_id"
+  rm -f ~maas/maas_id
+  rm -f ~maas/secret
+}
 
-# register forever
+register_maas_rack() {
-while [ 1 ];
+  sys_id=${1:-""}
-do
+  echo "register-rack-controller URL: ${MAAS_ENDPOINT}"
+
+  if [[ ! -z "$sys_id" ]]
+  then
+    echo "Using provided system id ${sys_id}."
+    echo "$sys_id" > ~maas/maas_id
+  fi
+
+  # register forever
+  while [ 1 ];
+  do
     if maas-rack register --url=${MAAS_ENDPOINT} --secret="${MAAS_REGION_SECRET}";
     then
         echo "Successfully registered with MaaS Region Controller"
         break
     else
         echo "Unable to register with ${MAAS_ENDPOINT}... will try again"
-        sleep 10
+        sleep 30
     fi;
-done;
+  done;
+}
+
+get_host_identity() {
+  # Check if the underlying host was deployed by MAAS
+  if [[ -r "${HOST_MOUNT_PATH}/instance-data.json" ]]
+  then
+    grep -E 'instance-id' "${HOST_MOUNT_PATH}/instance-data.json" | head -1 | tr -d ' ",' | cut -d: -f 2
+  else
+    echo ""
+  fi
+}
+
+get_pod_identity() {
+  if [[ -r ~maas/maas_id ]]
+  then
+    cat ~maas/maas_id
+  else
+    echo ""
+  fi
+}
+
+HOST_SYSTEM_ID=$(get_host_identity)
+POD_SYSTEM_ID=$(get_pod_identity)
+
+# This Pod state already has a MAAS identity
+if [[ ! -z "$POD_SYSTEM_ID" ]]
+then
+  # If the pod maas identity doesn't match the
+  # host maas identity, unregister the pod identity
+  # as a rack controller
+  if [[ "$HOST_SYSTEM_ID" != "$POD_SYSTEM_ID" ]]
+  then
+    unregister_maas_rack "$POD_SYSTEM_ID"
+    register_maas_rack "$HOST_SYTEM_ID"
+  else
+    echo "Found existing maas_id, assuming already registered."
+  fi
+
+  exit 0
+else
+  register_maas_rack
+fi

charts/maas/templates/configmap-ingress.yaml

@@ -33,6 +33,6 @@ metadata:
 data:
   enable-underscores-in-headers: "true"
   bind-address: {{ index $bind_address_cidr 0 | quote }}
-  diable-ipv6: "true"
+  disable-ipv6: "true"
 ...
 {{- end }}

charts/maas/templates/deployment-maas-ingress.yaml

@@ -0,0 +1,267 @@
+
+{{/*
+Copyright (c) 2018 AT&T Intellectual Property. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.maas_ingress }}
+{{- $envAll := . }}
+{{- $serviceAccountName := "maas-ingress" }}
+{{- $mounts_maas_ingress := .Values.pod.mounts.maas_ingress }}
+{{- $mounts_maas_ingress_init := .Values.pod.mounts.maas_ingress.init_container }}
+
+{{ tuple $envAll "maas_ingress" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRole
+metadata:
+  name: {{ $serviceAccountName }}
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - configmaps
+      - endpoints
+      - nodes
+      - pods
+      - secrets
+    verbs:
+      - list
+      - watch
+  - apiGroups:
+      - ""
+    resources:
+      - nodes
+    verbs:
+      - get
+  - apiGroups:
+      - ""
+    resources:
+      - services
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - "extensions"
+    resources:
+      - ingresses
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - ""
+    resources:
+        - events
+    verbs:
+        - create
+        - patch
+  - apiGroups:
+      - "extensions"
+    resources:
+      - ingresses/status
+    verbs:
+      - update
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+  name: {{ $serviceAccountName }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: {{ $serviceAccountName }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ $serviceAccountName }}
+    namespace: {{ $envAll.Release.Namespace }}
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: Role
+metadata:
+  name: {{ $serviceAccountName }}
+  namespace: {{ $envAll.Release.Namespace }}
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - configmaps
+      - pods
+      - secrets
+      - namespaces
+    verbs:
+      - get
+  - apiGroups:
+      - ""
+    resources:
+      - configmaps
+    resourceNames:
+      - {{ printf "%s-maas-ingress" .Release.Name | quote }}
+    verbs:
+      - get
+      - update
+  - apiGroups:
+      - ""
+    resources:
+      - configmaps
+    verbs:
+      - create
+  - apiGroups:
+      - ""
+    resources:
+      - endpoints
+    verbs:
+      - get
+      - create
+      - update
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: RoleBinding
+metadata:
+  name: {{ $serviceAccountName }}
+  namespace: {{ $envAll.Release.Namespace }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ $serviceAccountName }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ $serviceAccountName }}
+    namespace: {{ $envAll.Release.Namespace }}
+---
+---
+apiVersion: apps/v1beta1
+kind: Deployment
+metadata:
+  name: maas-ingress
+  annotations:
+    {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
+spec:
+  replicas: {{ .Values.pod.replicas.ingress }}
+  template:
+    metadata:
+      labels:
+{{ tuple $envAll "maas" "ingress" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
+      annotations:
+        configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }}
+        configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }}
+    spec:
+      serviceAccountName: {{ $serviceAccountName }}
+      affinity:
+{{- tuple $envAll "maas" "ingress" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }}
+      nodeSelector:
+        {{ .Values.labels.ingress.node_selector_key }}: {{ .Values.labels.rack.node_selector_value }}
+      hostNetwork: true
+      dnsPolicy: ClusterFirstWithHostNet
+      initContainers:
+{{ tuple $envAll "maas_ingress" $mounts_maas_ingress_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
+        - name: maas-ingress-vip-init
+          image: {{ .Values.images.tags.ingress }}
+          imagePullPolicy: {{ .Values.images.pull_policy }}
+{{ tuple $envAll $envAll.Values.pod.resources.maas_ingress_vip | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+          securityContext:
+            capabilities:
+              add:
+                - 'NET_ADMIN'
+                - 'SYS_MODULE'
+            runAsUser: 0
+          command:
+            - /tmp/maas-vip-configure.sh
+            - start
+          env:
+{{ include "helm-toolkit.utils.to_k8s_env_vars" .Values.network.maas_ingress | indent 12 }}
+          volumeMounts:
+            - mountPath: /tmp/maas-vip-configure.sh
+              name: maas-bin
+              subPath: maas-vip-configure
+              readOnly: true
+            - mountPath: /mnt/host-rootfs
+              name: host-rootfs
+              readOnly: true
+      containers:
+        - name: maas-ingress-vip
+          image: {{ .Values.images.tags.ingress }}
+          imagePullPolicy: {{ .Values.images.pull_policy }}
+{{ tuple $envAll $envAll.Values.pod.resources.maas_ingress_vip | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+          securityContext:
+            capabilities:
+              add:
+                - 'NET_ADMIN'
+            runAsUser: 0
+          command:
+            - /tmp/maas-vip-configure.sh
+            - sleep
+          env:
+{{ include "helm-toolkit.utils.to_k8s_env_vars" .Values.network.maas_ingress | indent 12 }}
+          volumeMounts:
+            - mountPath: /tmp/maas-vip-configure.sh
+              name: maas-bin
+              subPath: maas-vip-configure
+              readOnly: true
+          lifecycle:
+            preStop:
+              exec:
+                command:
+                  - /tmp/maas-vip-configure.sh
+                  - stop
+        - name: maas-ingress
+          image: {{ .Values.images.tags.ingress }}
+          imagePullPolicy: {{ .Values.images.pull_policy }}
+{{ tuple $envAll $envAll.Values.pod.resources.maas_ingress | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+          securityContext:
+            capabilities:
+              add:
+                - 'NET_BIND_SERVICE'
+            runAsUser: 0
+          command:
+            - /tmp/maas-ingress.sh
+            - start
+          env:
+            - name: POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: RELEASE_NAME
+              value: {{ .Release.Name | quote }}
+            - name: HTTP_PORT
+              value: "8808"
+            - name: HTTPS_PORT
+              value: "8543"
+            - name: HEALTHZ_PORT
+              value: {{ tuple "maas_ingress" "podport" "healthz" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
+            - name: STATUS_PORT
+              value: {{ tuple "maas_ingress" "podport" "status" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
+            - name: ERROR_PAGE_SERVICE
+              value: {{ tuple "maas_ingress" "error_pages" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" | quote }}
+          volumeMounts:
+            - mountPath: /tmp/maas-ingress.sh
+              name: maas-bin
+              subPath: maas-ingress
+              readOnly: true
+{{ if $mounts_maas_ingress.volumeMounts }}{{ toYaml $mounts_maas_ingress.volumeMounts | indent 12 }}{{ end }}
+      volumes:
+        - name: maas-bin
+          configMap:
+            name: maas-bin
+            defaultMode: 0555
+        - name: host-rootfs
+          hostPath:
+            path: /
+{{ if $mounts_maas_ingress.volumes }}{{ toYaml $mounts_maas_ingress.volumes | indent 8 }}{{ end }}
+{{- end }}

charts/maas/templates/service-ingress-error.yaml

@@ -13,6 +13,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */}}
+{{- if .Values.manifests.maas_ingress }}
 ---
 apiVersion: v1
 kind: Service
@@ -26,3 +27,4 @@ spec:
   selector:
 {{ tuple . "maas" "ingress-errors" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
 ...
+{{- end }}

charts/maas/templates/statefulset-rack.yaml

@@ -23,125 +23,6 @@ limitations under the License.
 
 {{ tuple $envAll "rack_controller" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
 ---
-apiVersion: rbac.authorization.k8s.io/v1beta1
-kind: ClusterRole
-metadata:
-  name: {{ $serviceAccountName }}
-rules:
-  - apiGroups:
-      - ""
-    resources:
-      - configmaps
-      - endpoints
-      - nodes
-      - pods
-      - secrets
-    verbs:
-      - list
-      - watch
-  - apiGroups:
-      - ""
-    resources:
-      - nodes
-    verbs:
-      - get
-  - apiGroups:
-      - ""
-    resources:
-      - services
-    verbs:
-      - get
-      - list
-      - watch
-  - apiGroups:
-      - "extensions"
-    resources:
-      - ingresses
-    verbs:
-      - get
-      - list
-      - watch
-  - apiGroups:
-      - ""
-    resources:
-        - events
-    verbs:
-        - create
-        - patch
-  - apiGroups:
-      - "extensions"
-    resources:
-      - ingresses/status
-    verbs:
-      - update
----
-apiVersion: rbac.authorization.k8s.io/v1beta1
-kind: ClusterRoleBinding
-metadata:
-  name: {{ $serviceAccountName }}
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: {{ $serviceAccountName }}
-subjects:
-  - kind: ServiceAccount
-    name: {{ $serviceAccountName }}
-    namespace: {{ $envAll.Release.Namespace }}
----
-apiVersion: rbac.authorization.k8s.io/v1beta1
-kind: Role
-metadata:
-  name: {{ $serviceAccountName }}
-  namespace: {{ $envAll.Release.Namespace }}
-rules:
-  - apiGroups:
-      - ""
-    resources:
-      - configmaps
-      - pods
-      - secrets
-      - namespaces
-    verbs:
-      - get
-  - apiGroups:
-      - ""
-    resources:
-      - configmaps
-    resourceNames:
-      - {{ printf "%s-maas-ingress" .Release.Name | quote }}
-    verbs:
-      - get
-      - update
-  - apiGroups:
-      - ""
-    resources:
-      - configmaps
-    verbs:
-      - create
-  - apiGroups:
-      - ""
-    resources:
-      - endpoints
-    verbs:
-      - get
-      - create
-      - update
----
-apiVersion: rbac.authorization.k8s.io/v1beta1
-kind: RoleBinding
-metadata:
-  name: {{ $serviceAccountName }}
-  namespace: {{ $envAll.Release.Namespace }}
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: Role
-  name: {{ $serviceAccountName }}
-subjects:
-  - kind: ServiceAccount
-    name: {{ $serviceAccountName }}
-    namespace: {{ $envAll.Release.Namespace }}
----
----
 apiVersion: apps/v1beta1
 kind: StatefulSet
 metadata:
@@ -151,6 +32,7 @@ metadata:
 spec:
   serviceName: maas-rack
   replicas: {{ .Values.pod.replicas.rack }}
+  podManagementPolicy: 'Parallel'
   updateStrategy:
     type: 'RollingUpdate'
   template:
@@ -170,93 +52,7 @@ spec:
       dnsPolicy: ClusterFirstWithHostNet
       initContainers:
 {{ tuple $envAll "rack_controller" $mounts_maas_rack_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
-        - name: maas-ingress-vip-init
-          image: {{ .Values.images.tags.ingress }}
-          imagePullPolicy: {{ .Values.images.pull_policy }}
-{{ tuple $envAll $envAll.Values.pod.resources.maas_ingress_vip | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
-          securityContext:
-            capabilities:
-              add:
-                - 'NET_ADMIN'
-                - 'SYS_MODULE'
-            runAsUser: 0
-          command:
-            - /tmp/maas-vip-configure.sh
-            - start
-          env:
-{{ include "helm-toolkit.utils.to_k8s_env_vars" .Values.network.maas_ingress | indent 12 }}
-          volumeMounts:
-            - mountPath: /tmp/maas-vip-configure.sh
-              name: maas-bin
-              subPath: maas-vip-configure
-              readOnly: true
-            - mountPath: /mnt/host-rootfs
-              name: host-rootfs
-              readOnly: true
       containers:
-        - name: maas-ingress-vip
-          image: {{ .Values.images.tags.ingress }}
-          imagePullPolicy: {{ .Values.images.pull_policy }}
-{{ tuple $envAll $envAll.Values.pod.resources.maas_ingress_vip | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
-          securityContext:
-            capabilities:
-              add:
-                - 'NET_ADMIN'
-            runAsUser: 0
-          command:
-            - /tmp/maas-vip-configure.sh
-            - sleep
-          env:
-{{ include "helm-toolkit.utils.to_k8s_env_vars" .Values.network.maas_ingress | indent 12 }}
-          volumeMounts:
-            - mountPath: /tmp/maas-vip-configure.sh
-              name: maas-bin
-              subPath: maas-vip-configure
-              readOnly: true
-          lifecycle:
-            preStop:
-              exec:
-                command:
-                  - /tmp/maas-vip-configure.sh
-                  - stop
-        - name: maas-ingress
-          image: {{ .Values.images.tags.ingress }}
-          imagePullPolicy: {{ .Values.images.pull_policy }}
-{{ tuple $envAll $envAll.Values.pod.resources.maas_ingress | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
-          securityContext:
-            capabilities:
-              add:
-                - 'NET_BIND_SERVICE'
-            runAsUser: 0
-          command:
-            - /tmp/maas-ingress.sh
-            - start
-          env:
-            - name: POD_NAMESPACE
-              valueFrom:
-                fieldRef:
-                  fieldPath: metadata.namespace
-            - name: POD_NAME
-              valueFrom:
-                fieldRef:
-                  fieldPath: metadata.name
-            - name: RELEASE_NAME
-              value: {{ .Release.Name | quote }}
-            - name: HTTP_PORT
-              value: "8808"
-            - name: HTTPS_PORT
-              value: "8543"
-            - name: HEALTHZ_PORT
-              value: {{ tuple "maas_ingress" "podport" "healthz" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
-            - name: STATUS_PORT
-              value: {{ tuple "maas_ingress" "podport" "status" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
-            - name: ERROR_PAGE_SERVICE
-              value: {{ tuple "maas_ingress" "error_pages" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" | quote }}
-          volumeMounts:
-            - mountPath: /tmp/maas-ingress.sh
-              name: maas-bin
-              subPath: maas-ingress
-              readOnly: true
         - name: maas-rack
           image: {{ .Values.images.tags.maas_rack }}
           imagePullPolicy: {{ .Values.images.pull_policy }}
@@ -269,6 +65,11 @@ spec:
                 secretKeyRef:
                   name: {{ .Values.secrets.maas_region.name }}
                   key: REGION_SECRET
+            - name: MAAS_API_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Values.conf.maas.credentials.secret.name }}
+                  key: 'token'
 {{ tuple $envAll $envAll.Values.pod.resources.maas_rack | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
           command:
             - /tmp/start.sh
@@ -284,6 +85,9 @@ spec:
               name: pod-run-lock
             - mountPath: /tmp
               name: pod-tmp
+            - mountPath: /host_cloud-init
+              name: host-cloud-init
+              readOnly: true
 {{- if .Values.conf.maas.ntp.disable_ntpd_rack }}
             - name: maas-bin
               mountPath: /usr/sbin/ntpd
@@ -324,9 +128,9 @@ spec:
         - name: host-sys-fs-cgroup
           hostPath:
             path: /sys/fs/cgroup
-        - name: host-rootfs
+        - name: host-cloud-init
           hostPath:
-            path: /
+            path: /run/cloud-init
         - name: pod-run
           emptyDir: {}
         - name: pod-run-lock

charts/maas/values.yaml

@@ -19,10 +19,13 @@
 
 dependencies:
   static:
+    maas_ingress: {}
     rack_controller:
       services:
         - service: maas_region
           endpoint: internal
+      jobs:
+        - maas-export-api-key
     region_controller:
       jobs:
         - maas-db-sync
@@ -64,6 +67,7 @@ manifests:
   secret_ssh_key: false
   ingress_region: true
   configmap_ingress: true
+  maas_ingress: true
 
 images:
   tags:
@@ -98,6 +102,9 @@ labels:
   region:
     node_selector_key: ucp-control-plane
     node_selector_value: enabled
+  ingress:
+    node_selector_key: ucp-control-plane
+    node_selector_value: enabled
 
 network:
   maas_ingress:
@@ -226,6 +233,7 @@ pod:
   affinity:
     anti:
       type:
+        rack: requiredDuringSchedulingIgnoredDuringExecution
         default: preferredDuringSchedulingIgnoredDuringExecution
       topologyKey:
         default: kubernetes.io/hostname
@@ -239,9 +247,12 @@ pod:
     export_api_key:
       init_container: []
       export_api_key:
+    maas_ingress:
+      init_container: []
+      maas_region:
   replicas:
-    rack: 1
+    rack: 2
-    region: 1
+    region: 2
   resources:
     enabled: false
     test:

images/maas-rack-controller/Dockerfile

@@ -40,13 +40,6 @@ ENV MAAS_VERSION 2.3.5-6511-gf466fdb-0ubuntu1
 # install maas
 RUN rsyslogd; apt-get install -y maas-cli=$MAAS_VERSION maas-rack-controller=$MAAS_VERSION
 
-COPY scripts/register-rack-controller.sh /usr/local/bin
-RUN chmod +x /usr/local/bin/register-rack-controller.sh
-
-# register ourselves with the region controller
-COPY scripts/register-rack-controller.service /lib/systemd/system/register-rack-controller.service
-RUN systemctl enable register-rack-controller.service
-
 RUN mv /usr/sbin/tcpdump /usr/bin/tcpdump
 RUN ln -s /usr/bin/tcpdump /usr/sbin/tcpdump