Heat: clean up dead engines with cronjob

This PS adds a cronjob to clean up dead engines.

Change-Id: I5482ee8c21203d9e889437947f37f1355446c43a

heat/templates/bin/_heat-engine-cleaner.sh.tpl

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+set -ex
+
+heat-manage service clean

heat/templates/configmap-bin.yaml

@@ -50,4 +50,6 @@ data:
 {{ tuple "bin/_heat-cloudwatch.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
   heat-engine.sh: |
 {{ tuple "bin/_heat-engine.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+  heat-engine-cleaner.sh: |
+{{ tuple "bin/_heat-engine-cleaner.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
 {{- end }}

heat/templates/cron-job-engine-cleaner.yaml

@@ -0,0 +1,82 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.cron_job_engine_cleaner }}
+{{- if .Capabilities.APIVersions.Has "batch/v2alpha1" }}
+{{- $envAll := . }}
+{{- $dependencies := .Values.dependencies.engine_cleaner }}
+
+{{- $mounts_heat_engine_cleaner := .Values.pod.mounts.heat_engine_cleaner.heat_engine_cleaner }}
+{{- $mounts_heat_engine_cleaner_init := .Values.pod.mounts.heat_engine_cleaner.init_container }}
+
+{{- $serviceAccountName := "heat-engine-cleaner" }}
+{{ tuple $envAll $dependencies $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
+---
+apiVersion: batch/v1beta1
+kind: CronJob
+metadata:
+  name: heat-engine-cleaner
+spec:
+  schedule: {{ .Values.jobs.engine_cleaner.cron | quote }}
+  successfulJobsHistoryLimit: {{ .Values.jobs.engine_cleaner.history.success }}
+  failedJobsHistoryLimit: {{ .Values.jobs.engine_cleaner.history.failed }}
+  concurrencyPolicy: Forbid
+  jobTemplate:
+    metadata:
+      labels:
+{{ tuple $envAll "heat" "engine-cleaner" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
+    spec:
+      template:
+        spec:
+          serviceAccountName: {{ $serviceAccountName }}
+          restartPolicy: OnFailure
+          nodeSelector:
+            {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }}
+          initContainers:
+{{ tuple $envAll $dependencies $mounts_heat_engine_cleaner_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 12 }}
+          containers:
+            - name: heat-engine-cleaner
+              image: {{ .Values.images.tags.heat_engine_cleaner }}
+              imagePullPolicy: {{ .Values.images.pull_policy }}
+{{ tuple $envAll $envAll.Values.pod.resources.jobs.engine_cleaner | include "helm-toolkit.snippets.kubernetes_resources" | indent 14 }}
+              command:
+                - /tmp/heat-engine-cleaner.sh
+              volumeMounts:
+              - name: heat-bin
+                mountPath: /tmp/heat-engine-cleaner.sh
+                subPath: heat-engine-cleaner.sh
+                readOnly: true
+              - name: etcheat
+                mountPath: /etc/heat
+              - name: heat-etc
+                mountPath: /etc/heat/heat.conf
+                subPath: heat.conf
+                readOnly: true
+{{ if $mounts_heat_engine_cleaner.volumeMounts }}{{ toYaml $mounts_heat_engine_cleaner.volumeMounts | indent 14 }}{{ end }}
+          volumes:
+          - name: etcheat
+            emptyDir: {}
+          - name: heat-etc
+            configMap:
+              name: heat-etc
+              defaultMode: 0444
+          - name: heat-bin
+            configMap:
+              name: heat-bin
+              defaultMode: 0555
+{{ if $mounts_heat_engine_cleaner.volumes }}{{ toYaml $mounts_heat_engine_cleaner.volumes | indent 10 }}{{ end }}
+{{- end }}
+{{- end }}

heat/values.yaml

@@ -39,9 +39,17 @@ images:
     heat_cfn: docker.io/openstackhelm/heat:newton
     heat_cloudwatch: docker.io/openstackhelm/heat:newton
     heat_engine: docker.io/openstackhelm/heat:newton
+    heat_engine_cleaner: docker.io/openstackhelm/heat:newton
     dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.2.1
   pull_policy: "IfNotPresent"
 
+jobs:
+  engine_cleaner:
+    cron: "*/5 * * * *"
+    history:
+      success: 3
+      failed: 1
+
 conf:
   paste:
     pipeline:heat-api:
@@ -341,6 +349,18 @@ dependencies:
       endpoint: internal
     - service: identity
       endpoint: internal
+  engine_cleaner:
+    jobs:
+    - heat-db-sync
+    - heat-ks-user
+    - heat-trustee-ks-user
+    - heat-domain-ks-user
+    - heat-ks-endpoints
+    services:
+    - service: oslo_db
+      endpoint: internal
+    - service: identity
+      endpoint: internal
 
 # Names of secrets used by bootstrap and environmental checks
 secrets:
@@ -520,6 +540,9 @@ pod:
     heat_trusts:
       init_container: null
       heat_trusts:
+    heat_engine_cleaner:
+      init_container: null
+      heat_engine_cleaner:
   replicas:
     api: 1
     cfn: 1
@@ -643,10 +666,18 @@ pod:
         limits:
           memory: "1024Mi"
           cpu: "2000m"
+      engine_cleaner:
+        requests:
+          memory: "124Mi"
+          cpu: "100m"
+        limits:
+          memory: "1024Mi"
+          cpu: "2000m"
 
 manifests:
   configmap_bin: true
   configmap_etc: true
+  cron_job_engine_cleaner: true
   deployment_api: true
   deployment_cfn: true
   deployment_cloudwatch: true

tools/overrides/releases/newton/kolla.yaml

@@ -33,6 +33,7 @@ images:
     heat_cloudwatch: 'docker.io/kolla/ubuntu-source-heat-api:3.0.3'
     heat_db_sync: 'docker.io/kolla/ubuntu-source-heat-api:3.0.3'
     heat_engine: 'docker.io/kolla/ubuntu-source-heat-engine:3.0.3'
+    heat_engine_cleaner: 'docker.io/kolla/ubuntu-source-heat-engine:3.0.3'
     horizon: 'docker.io/kolla/ubuntu-source-horizon:ocata'
     horizon_db_sync: 'docker.io/kolla/ubuntu-source-horizon:ocata'
     ironic_api: 'docker.io/kolla/ubuntu-source-ironic-api:3.0.3'