Nova: Reap dead services

This PS adds a cron job that reaps dead services from Nova.

Change-Id: I59e74c7520b0341d7cb7ebddd4c21e459e9c2049
Signed-off-by: Pete Birley <pete@port.direct>

nova/templates/bin/_nova-service-cleaner.sh.tpl

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+{{/*
+Copyright 2018 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+set -xe
+
+NOVA_SERVICES_TO_CLEAN="$(openstack compute service list -f value -c Binary | sort | uniq | grep -v '^nova-compute$')"
+for NOVA_SERVICE in ${NOVA_SERVICES_TO_CLEAN}; do
+  DEAD_SERVICE_IDS=$(openstack compute service list --service ${NOVA_SERVICE} -f json | jq -r '.[] | select(.State == "down") | .ID')
+  for SERVICE_ID in ${DEAD_SERVICE_IDS}; do
+    openstack compute service delete "${SERVICE_ID}"
+  done
+done

nova/templates/configmap-bin.yaml

@@ -87,6 +87,8 @@ data:
 {{ tuple "bin/_cell-setup.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
   cell-setup-init.sh: |
 {{ tuple "bin/_cell-setup-init.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+  nova-service-cleaner.sh: |
+{{ tuple "bin/_nova-service-cleaner.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
   rabbit-init.sh: |
 {{- include "helm-toolkit.scripts.rabbit_init" . | indent 4 }}
 {{- end }}

nova/templates/cron-job-service-cleaner.yaml

@@ -0,0 +1,73 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.cron_job_service_cleaner }}
+{{- $envAll := . }}
+
+{{- $serviceAccountName := "nova-service-cleaner" }}
+{{ tuple $envAll "service_cleaner" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
+---
+apiVersion: batch/v1beta1
+kind: CronJob
+metadata:
+  name: nova-service-cleaner
+spec:
+  schedule: {{ .Values.jobs.service_cleaner.cron | quote }}
+  successfulJobsHistoryLimit: {{ .Values.jobs.service_cleaner.history.success }}
+  failedJobsHistoryLimit: {{ .Values.jobs.service_cleaner.history.failed }}
+  concurrencyPolicy: Forbid
+  jobTemplate:
+    metadata:
+      labels:
+{{ tuple $envAll "nova" "service-cleaner" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
+    spec:
+      template:
+        spec:
+          serviceAccountName: {{ $serviceAccountName }}
+          restartPolicy: OnFailure
+          nodeSelector:
+            {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }}
+          initContainers:
+{{ tuple $envAll "service_cleaner" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 12 }}
+          containers:
+            - name: nova-service-cleaner
+{{ tuple $envAll "nova_service_cleaner" | include "helm-toolkit.snippets.image" | indent 14 }}
+{{ tuple $envAll $envAll.Values.pod.resources.jobs.service_cleaner | include "helm-toolkit.snippets.kubernetes_resources" | indent 14 }}
+              env:
+{{- with $env := dict "ksUserSecret" $envAll.Values.secrets.identity.nova }}
+{{- include "helm-toolkit.snippets.keystone_openrc_env_vars" $env | indent 14 }}
+{{- end }}
+              command:
+                - /tmp/nova-service-cleaner.sh
+              volumeMounts:
+              - name: nova-bin
+                mountPath: /tmp/nova-service-cleaner.sh
+                subPath: nova-service-cleaner.sh
+                readOnly: true
+              - name: etcnova
+                mountPath: /etc/nova
+          volumes:
+          - name: etcnova
+            emptyDir: {}
+          - name: nova-etc
+            configMap:
+              name: nova-etc
+              defaultMode: 0444
+          - name: nova-bin
+            configMap:
+              name: nova-bin
+              defaultMode: 0555
+{{- end }}

nova/values.yaml

@@ -82,6 +82,9 @@ images:
     nova_novncproxy_assets: 'docker.io/kolla/ubuntu-source-nova-novncproxy:3.0.3'
     nova_placement: docker.io/openstackhelm/nova:newton
     nova_scheduler: docker.io/openstackhelm/nova:newton
+    # NOTE(portdirect): we simply use the ceph config helper here,
+    # as it has both oscli and jq.
+    nova_service_cleaner: 'docker.io/port/ceph-config-helper:v1.10.3'
     nova_spiceproxy: docker.io/openstackhelm/nova:newton
     nova_spiceproxy_assets: 'docker.io/kolla/ubuntu-source-nova-spicehtml5proxy:3.0.3'
     test: 'docker.io/kolla/ubuntu-source-rally:4.0.0'
@@ -100,6 +103,11 @@ jobs:
     history:
       success: 3
       failed: 1
+  service_cleaner:
+    cron: "0 */1 * * *"
+    history:
+      success: 3
+      failed: 1
 
 bootstrap:
   enabled: true
@@ -273,6 +281,19 @@ dependencies:
           labels:
             application: nova
             component: compute
+    service_cleaner:
+      jobs:
+        - nova-db-sync
+        - nova-rabbit-init
+      services:
+        - endpoint: internal
+          service: oslo_messaging
+        - endpoint: internal
+          service: oslo_db
+        - endpoint: internal
+          service: identity
+        - endpoint: internal
+          service: compute
     compute:
       pod:
         - requireSameNode: true
@@ -1765,6 +1786,13 @@ pod:
         limits:
           memory: "1024Mi"
           cpu: "2000m"
+      service_cleaner:
+        requests:
+          memory: "128Mi"
+          cpu: "100m"
+        limits:
+          memory: "1024Mi"
+          cpu: "2000m"
       image_repo_sync:
         requests:
           memory: "128Mi"
@@ -1777,6 +1805,7 @@ manifests:
   configmap_bin: true
   configmap_etc: true
   cron_job_cell_setup: true
+  cron_job_service_cleaner: true
   daemonset_compute: true
   deployment_api_metadata: true
   deployment_api_osapi: true