From 5be8e34606a52074c4689612c8b2b67558c08feb Mon Sep 17 00:00:00 2001 From: portdirect Date: Mon, 19 Feb 2018 14:35:49 -0500 Subject: [PATCH] Heat: clean up dead engines with cronjob This PS adds a cronjob to clean up dead engines. Change-Id: I5482ee8c21203d9e889437947f37f1355446c43a --- .../templates/bin/_heat-engine-cleaner.sh.tpl | 21 +++++ heat/templates/configmap-bin.yaml | 2 + heat/templates/cron-job-engine-cleaner.yaml | 82 +++++++++++++++++++ heat/values.yaml | 31 +++++++ tools/overrides/releases/newton/kolla.yaml | 1 + 5 files changed, 137 insertions(+) create mode 100644 heat/templates/bin/_heat-engine-cleaner.sh.tpl create mode 100644 heat/templates/cron-job-engine-cleaner.yaml diff --git a/heat/templates/bin/_heat-engine-cleaner.sh.tpl b/heat/templates/bin/_heat-engine-cleaner.sh.tpl new file mode 100644 index 0000000000..5c659c6d4d --- /dev/null +++ b/heat/templates/bin/_heat-engine-cleaner.sh.tpl @@ -0,0 +1,21 @@ +#!/bin/bash + +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex + +heat-manage service clean diff --git a/heat/templates/configmap-bin.yaml b/heat/templates/configmap-bin.yaml index 7b34f80be3..9661f30196 100644 --- a/heat/templates/configmap-bin.yaml +++ b/heat/templates/configmap-bin.yaml @@ -50,4 +50,6 @@ data: {{ tuple "bin/_heat-cloudwatch.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} heat-engine.sh: | {{ tuple "bin/_heat-engine.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + heat-engine-cleaner.sh: | +{{ tuple "bin/_heat-engine-cleaner.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} {{- end }} diff --git a/heat/templates/cron-job-engine-cleaner.yaml b/heat/templates/cron-job-engine-cleaner.yaml new file mode 100644 index 0000000000..1bf36b92d1 --- /dev/null +++ b/heat/templates/cron-job-engine-cleaner.yaml @@ -0,0 +1,82 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.cron_job_engine_cleaner }} +{{- if .Capabilities.APIVersions.Has "batch/v2alpha1" }} +{{- $envAll := . }} +{{- $dependencies := .Values.dependencies.engine_cleaner }} + +{{- $mounts_heat_engine_cleaner := .Values.pod.mounts.heat_engine_cleaner.heat_engine_cleaner }} +{{- $mounts_heat_engine_cleaner_init := .Values.pod.mounts.heat_engine_cleaner.init_container }} + +{{- $serviceAccountName := "heat-engine-cleaner" }} +{{ tuple $envAll $dependencies $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: heat-engine-cleaner +spec: + schedule: {{ .Values.jobs.engine_cleaner.cron | quote }} + successfulJobsHistoryLimit: {{ .Values.jobs.engine_cleaner.history.success }} + failedJobsHistoryLimit: {{ .Values.jobs.engine_cleaner.history.failed }} + concurrencyPolicy: Forbid + jobTemplate: + metadata: + labels: +{{ tuple $envAll "heat" "engine-cleaner" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + spec: + template: + spec: + serviceAccountName: {{ $serviceAccountName }} + restartPolicy: OnFailure + nodeSelector: + {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }} + initContainers: +{{ tuple $envAll $dependencies $mounts_heat_engine_cleaner_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 12 }} + containers: + - name: heat-engine-cleaner + image: {{ .Values.images.tags.heat_engine_cleaner }} + imagePullPolicy: {{ .Values.images.pull_policy }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.engine_cleaner | include "helm-toolkit.snippets.kubernetes_resources" | indent 14 }} + command: + - /tmp/heat-engine-cleaner.sh + volumeMounts: + - name: heat-bin + mountPath: /tmp/heat-engine-cleaner.sh + subPath: heat-engine-cleaner.sh + readOnly: true + - name: etcheat + mountPath: /etc/heat + - name: heat-etc + mountPath: /etc/heat/heat.conf + subPath: heat.conf + readOnly: true +{{ if $mounts_heat_engine_cleaner.volumeMounts }}{{ toYaml $mounts_heat_engine_cleaner.volumeMounts | indent 14 }}{{ end }} + volumes: + - name: etcheat + emptyDir: {} + - name: heat-etc + configMap: + name: heat-etc + defaultMode: 0444 + - name: heat-bin + configMap: + name: heat-bin + defaultMode: 0555 +{{ if $mounts_heat_engine_cleaner.volumes }}{{ toYaml $mounts_heat_engine_cleaner.volumes | indent 10 }}{{ end }} +{{- end }} +{{- end }} diff --git a/heat/values.yaml b/heat/values.yaml index 6c33b3098a..b653c9a917 100644 --- a/heat/values.yaml +++ b/heat/values.yaml @@ -39,9 +39,17 @@ images: heat_cfn: docker.io/openstackhelm/heat:newton heat_cloudwatch: docker.io/openstackhelm/heat:newton heat_engine: docker.io/openstackhelm/heat:newton + heat_engine_cleaner: docker.io/openstackhelm/heat:newton dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.2.1 pull_policy: "IfNotPresent" +jobs: + engine_cleaner: + cron: "*/5 * * * *" + history: + success: 3 + failed: 1 + conf: paste: pipeline:heat-api: @@ -341,6 +349,18 @@ dependencies: endpoint: internal - service: identity endpoint: internal + engine_cleaner: + jobs: + - heat-db-sync + - heat-ks-user + - heat-trustee-ks-user + - heat-domain-ks-user + - heat-ks-endpoints + services: + - service: oslo_db + endpoint: internal + - service: identity + endpoint: internal # Names of secrets used by bootstrap and environmental checks secrets: @@ -520,6 +540,9 @@ pod: heat_trusts: init_container: null heat_trusts: + heat_engine_cleaner: + init_container: null + heat_engine_cleaner: replicas: api: 1 cfn: 1 @@ -643,10 +666,18 @@ pod: limits: memory: "1024Mi" cpu: "2000m" + engine_cleaner: + requests: + memory: "124Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" manifests: configmap_bin: true configmap_etc: true + cron_job_engine_cleaner: true deployment_api: true deployment_cfn: true deployment_cloudwatch: true diff --git a/tools/overrides/releases/newton/kolla.yaml b/tools/overrides/releases/newton/kolla.yaml index c77f3ca2a4..e5a235c8d6 100644 --- a/tools/overrides/releases/newton/kolla.yaml +++ b/tools/overrides/releases/newton/kolla.yaml @@ -33,6 +33,7 @@ images: heat_cloudwatch: 'docker.io/kolla/ubuntu-source-heat-api:3.0.3' heat_db_sync: 'docker.io/kolla/ubuntu-source-heat-api:3.0.3' heat_engine: 'docker.io/kolla/ubuntu-source-heat-engine:3.0.3' + heat_engine_cleaner: 'docker.io/kolla/ubuntu-source-heat-engine:3.0.3' horizon: 'docker.io/kolla/ubuntu-source-horizon:ocata' horizon_db_sync: 'docker.io/kolla/ubuntu-source-horizon:ocata' ironic_api: 'docker.io/kolla/ubuntu-source-ironic-api:3.0.3'