From f830186c45ec0e20d96586ebd71abd8a1da8efe6 Mon Sep 17 00:00:00 2001 From: ad620p Date: Mon, 16 Sep 2019 07:53:02 +0000 Subject: [PATCH] Support for Neutron Helm test cleanup and reset for abnormal failures Rally usually cleans up all its resources in normal executions - normal test success cases and normal test failure cases. But the generic cleanup does not work well for out of the system failures like process interruptions, pod failures, disaster cleanup etc. This is a known issue in rally-openstack. - "Current generic mechanism is nice but it doesn't work enough well in real life. And in cases of existing users, persistence context and disaster cleanups it doesn't work well." Hence, if we shall face above such issues, it is becoming impossible to run "helm test neutron" again because of the stale data and different quota limits mentioned in the values.yaml. Hence we need to purge the stale data from the "test" project as well as reset the quota limit for such scenarios. For the normal executions, this patch has to do nothing, but for unexpected failures, this patch will purge the stale data from test project and reset the quota as defined in values.yaml for the next run. Change-Id: I3f6851582e2ac1aa1d375fcd13c07f4f57f45dc8 --- neutron/templates/bin/_neutron-cleanup.sh.tpl | 27 +++++++++++++++ neutron/templates/configmap-bin.yaml | 2 ++ neutron/templates/pod-rally-test.yaml | 33 +++++++++++++++++-- 3 files changed, 60 insertions(+), 2 deletions(-) create mode 100644 neutron/templates/bin/_neutron-cleanup.sh.tpl diff --git a/neutron/templates/bin/_neutron-cleanup.sh.tpl b/neutron/templates/bin/_neutron-cleanup.sh.tpl new file mode 100644 index 0000000000..b36d2b4276 --- /dev/null +++ b/neutron/templates/bin/_neutron-cleanup.sh.tpl @@ -0,0 +1,27 @@ +#!/bin/bash + +# Copyright 2019 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +while [ ! -f /tmp/pod-tmp/test_done.log ] +do + sleep 2s +done + +ospurge --purge-project $OS_TEST_PROJECT_NAME + +openstack quota set $OS_TEST_PROJECT_NAME --networks $NETWORK_QUOTA --ports $PORT_QUOTA --routers $ROUTER_QUOTA --subnets $SUBNET_QUOTA --secgroups $SEC_GROUP_QUOTA + diff --git a/neutron/templates/configmap-bin.yaml b/neutron/templates/configmap-bin.yaml index d36cb1b857..a8ed6edc6e 100644 --- a/neutron/templates/configmap-bin.yaml +++ b/neutron/templates/configmap-bin.yaml @@ -81,4 +81,6 @@ data: {{ tuple "bin/_neutron-server.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} rabbit-init.sh: | {{- include "helm-toolkit.scripts.rabbit_init" . | indent 4 }} + neutron-cleanup.sh: | +{{ tuple "bin/_neutron-cleanup.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} {{- end }} diff --git a/neutron/templates/pod-rally-test.yaml b/neutron/templates/pod-rally-test.yaml index 374039af40..d4fad87f5b 100644 --- a/neutron/templates/pod-rally-test.yaml +++ b/neutron/templates/pod-rally-test.yaml @@ -64,6 +64,35 @@ spec: - name: SERVICE_OS_ROLE value: {{ .Values.endpoints.identity.auth.test.role | quote }} containers: + - name: {{ .Release.Name }}-reset + image: openstackhelm/ospurge + env: +{{- with $env := dict "ksUserSecret" .Values.secrets.identity.admin }} +{{- include "helm-toolkit.snippets.keystone_openrc_env_vars" $env | indent 8 }} +{{- end }} +{{- with $env := dict "ksUserSecret" .Values.secrets.identity.test }} +{{- include "helm-toolkit.snippets.keystone_user_create_env_vars" $env | indent 8 }} +{{- end }} + - name: OS_TEST_PROJECT_NAME + value: {{ .Values.endpoints.identity.auth.test.project_name }} + - name: NETWORK_QUOTA + value: {{ (first (index .Values.conf.rally_tests.tests "NeutronNetworks.create_and_delete_networks")).context.quotas.neutron.network | quote }} + - name: PORT_QUOTA + value: {{ (first (index .Values.conf.rally_tests.tests "NeutronNetworks.create_and_delete_ports")).context.quotas.neutron.port | quote }} + - name: ROUTER_QUOTA + value: {{ (first (index .Values.conf.rally_tests.tests "NeutronNetworks.create_and_delete_routers")).context.quotas.neutron.router | quote }} + - name: SUBNET_QUOTA + value: {{ (first (index .Values.conf.rally_tests.tests "NeutronNetworks.create_and_delete_subnets")).context.quotas.neutron.subnet | quote }} + - name: SEC_GROUP_QUOTA + value: {{ (first (index .Values.conf.rally_tests.tests "NeutronSecurityGroup.create_and_list_security_groups")).context.quotas.neutron.security_group | quote }} + command: ["/bin/bash", "-c", "/tmp/neutron-cleanup.sh"] + volumeMounts: + - name: neutron-bin + mountPath: /tmp/neutron-cleanup.sh + subPath: neutron-cleanup.sh + readOnly: true + - name: pod-tmp + mountPath: /tmp/pod-tmp - name: {{ .Release.Name }}-test {{ tuple $envAll "test" | include "helm-toolkit.snippets.image" | indent 6 }} {{ tuple $envAll $envAll.Values.pod.resources.jobs.tests | include "helm-toolkit.snippets.kubernetes_resources" | indent 6 }} @@ -76,8 +105,8 @@ spec: {{- end }} - name: RALLY_ENV_NAME value: {{.Release.Name}} - command: - - /tmp/rally-test.sh + command: ["/bin/bash", "-c"] + args: ["/tmp/rally-test.sh; touch /tmp/test_done.log"] volumeMounts: - name: pod-tmp mountPath: /tmp