Merge "[neutron] Unhardcode probes timings"

neutron/templates/bin/_health-probe.py.tpl

@@ -49,6 +49,8 @@ from oslo_context import context
 from oslo_log import log
 import oslo_messaging
 
+rpc_timeout = int(os.getenv('RPC_PROBE_TIMEOUT', '60'))
+rpc_retries = int(os.getenv('RPC_PROBE_RETRIES', '2'))
 rabbit_port = 5672
 tcp_established = "ESTABLISHED"
 log.logging.basicConfig(level=log.ERROR)
@@ -67,8 +69,8 @@ def check_agent_status(transport):
             topic=cfg.CONF.agent_queue_name,
             server=_get_hostname(use_fqdn))
         client = oslo_messaging.RPCClient(transport, target,
-                                          timeout=60,
+                                          timeout=rpc_timeout,
-                                          retry=2)
+                                          retry=rpc_retries)
         client.call(context.RequestContext(),
                     'pod_health_probe_method_ignore_errors')
     except oslo_messaging.exceptions.MessageDeliveryFailure:

neutron/templates/daemonset-bagpipe-bgp.yaml

@@ -12,6 +12,16 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */}}
 
+{{- define "bagpipeBgpLivenessProbeTemplate" }}
+tcpSocket:
+  port: {{ tuple "network" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+{{- end }}
+
+{{- define "bagpipeBgpReadinessProbeTemplate" }}
+tcpSocket:
+  port: {{ tuple "network" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+{{- end }}
+
 {{- define "neutron.bagpipe_bgp.daemonset" }}
 {{- $daemonset := index . 0 }}
 {{- $configMapName := index . 1 }}
@@ -79,13 +89,8 @@ spec:
 {{ tuple $envAll "neutron_bagpipe_bgp" | include "helm-toolkit.snippets.image" | indent 10 }}
 {{ tuple $envAll $envAll.Values.pod.resources.agent.bagpipe_bgp | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
 {{ dict "envAll" $envAll "application" "neutron_bagpipe_bgp" "container" "neutron_bagpipe_bgp" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
-          readinessProbe:
+{{ dict "envAll" $envAll "component" "bagpipe_bgp" "container" "bagpipe_bgp" "type" "liveness" "probeTemplate" (include "bagpipeBgpLivenessProbeTemplate" $envAll | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }}
-            tcpSocket:
+{{ dict "envAll" $envAll "component" "bagpipe_bgp" "container" "bagpipe_bgp" "type" "readiness" "probeTemplate" (include "bagpipeBgpReadinessProbeTemplate" $envAll | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }}
-              port: {{ tuple "network" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
-          livenessProbe:
-            tcpSocket:
-              port: {{ tuple "network" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
-            initialDelaySeconds: 60
           command:
             - /tmp/neutron-bagpipe-bgp.sh
           volumeMounts:

neutron/templates/daemonset-dhcp-agent.yaml

@@ -155,6 +155,11 @@ spec:
 {{ tuple $envAll "neutron_dhcp" | include "helm-toolkit.snippets.image" | indent 10 }}
 {{ tuple $envAll $envAll.Values.pod.resources.agent.dhcp | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
 {{ dict "envAll" $envAll "application" "neutron_dhcp_agent" "container" "neutron_dhcp_agent" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
+          env:
+            - name: RPC_PROBE_TIMEOUT
+              value: "{{ .Values.pod.probes.rpc_timeout }}"
+            - name: RPC_PROBE_RETRIES
+              value: "{{ .Values.pod.probes.rpc_retries }}"
 {{ dict "envAll" $envAll "component" "dhcp_agent" "container" "dhcp_agent" "type" "readiness" "probeTemplate" (include "dhcpAgentReadinessProbeTemplate" $envAll | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }}
 {{ dict "envAll" $envAll "component" "dhcp_agent" "container" "dhcp_agent" "type" "liveness" "probeTemplate" (include "dhcpAgentLivenessProbeTemplate" $envAll | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }}
           command:

neutron/templates/daemonset-l2gw-agent.yaml

@@ -12,6 +12,39 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */}}
 
+{{- define "l2gwAgentLivenessProbeTemplate" }}
+exec:
+  command:
+    - python
+    - /tmp/health-probe.py
+    - --config-file
+    - /etc/neutron/neutron.conf
+    - --config-file
+    - /etc/neutron/l2gw_agent.ini
+    - --agent-queue-name
+    - l2gateway_agent
+    - --liveness-probe
+{{- if .Values.pod.use_fqdn.neutron_agent }}
+    - --use-fqdn
+{{- end }}
+{{- end }}
+
+{{- define "l2gwAgentReadinessProbeTemplate" }}
+exec:
+  command:
+    - python
+    - /tmp/health-probe.py
+    - --config-file
+    - /etc/neutron/neutron.conf
+    - --config-file
+    - /etc/neutron/l2gw_agent.ini
+    - --agent-queue-name
+    - l2gateway_agent
+{{- if .Values.pod.use_fqdn.neutron_agent }}
+    - --use-fqdn
+{{- end }}
+{{- end }}
+
 {{- define "neutron.l2gw_agent.daemonset" }}
 {{- $daemonset := index . 0 }}
 {{- $configMapName := index . 1 }}
@@ -64,41 +97,13 @@ spec:
 {{ tuple $envAll $envAll.Values.pod.resources.agent.l2gw | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
           securityContext:
             privileged: true
-          readinessProbe:
+          env:
-            exec:
+            - name: RPC_PROBE_TIMEOUT
-              command:
+              value: "{{ .Values.pod.probes.rpc_timeout }}"
-                - python
+            - name: RPC_PROBE_RETRIES
-                - /tmp/health-probe.py
+              value: "{{ .Values.pod.probes.rpc_retries }}"
-                - --config-file
+{{ dict "envAll" $envAll "component" "l2gw_agent" "container" "l2gw_agent" "type" "liveness" "probeTemplate" (include "l2gwAgentLivenessProbeTemplate" $envAll | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }}
-                - /etc/neutron/neutron.conf
+{{ dict "envAll" $envAll "component" "l2gw_agent" "container" "l2gw_agent" "type" "readiness" "probeTemplate" (include "l2gwAgentReadinessProbeTemplate" $envAll | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }}
-                - --config-file
-                - /etc/neutron/l2gw_agent.ini
-                - --agent-queue-name
-                - l2gateway_agent
-{{- if .Values.pod.use_fqdn.neutron_agent }}
-                - --use-fqdn
-{{- end }}
-            initialDelaySeconds: 30
-            periodSeconds: 190
-            timeoutSeconds: 185
-          livenessProbe:
-            exec:
-              command:
-                - python
-                - /tmp/health-probe.py
-                - --config-file
-                - /etc/neutron/neutron.conf
-                - --config-file
-                - /etc/neutron/l2gw_agent.ini
-                - --agent-queue-name
-                - l2gateway_agent
-                - --liveness-probe
-{{- if .Values.pod.use_fqdn.neutron_agent }}
-                - --use-fqdn
-{{- end }}
-            initialDelaySeconds: 120
-            periodSeconds: 600
-            timeoutSeconds: 580
           command:
             - /tmp/neutron-l2gw-agent.sh
           volumeMounts:

neutron/templates/daemonset-l3-agent.yaml

@@ -156,6 +156,11 @@ spec:
 {{ tuple $envAll "neutron_l3" | include "helm-toolkit.snippets.image" | indent 10 }}
 {{ tuple $envAll $envAll.Values.pod.resources.agent.l3 | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
 {{ dict "envAll" $envAll "application" "neutron_l3_agent" "container" "neutron_l3_agent" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
+          env:
+            - name: RPC_PROBE_TIMEOUT
+              value: "{{ .Values.pod.probes.rpc_timeout }}"
+            - name: RPC_PROBE_RETRIES
+              value: "{{ .Values.pod.probes.rpc_retries }}"
 {{ dict "envAll" $envAll "component" "l3_agent" "container" "l3_agent" "type" "readiness" "probeTemplate" (include "l3AgentReadinessProbeTemplate" $envAll | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }}
 {{ dict "envAll" $envAll "component" "l3_agent" "container" "l3_agent" "type" "liveness" "probeTemplate" (include "l3AgentLivenessProbeTemplate" $envAll | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }}
           command:

neutron/templates/daemonset-metadata-agent.yaml

@@ -113,6 +113,11 @@ spec:
         - name: neutron-metadata-agent
 {{ tuple $envAll "neutron_metadata" | include "helm-toolkit.snippets.image" | indent 10 }}
 {{ tuple $envAll $envAll.Values.pod.resources.agent.metadata | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+          env:
+            - name: RPC_PROBE_TIMEOUT
+              value: "{{ .Values.pod.probes.rpc_timeout }}"
+            - name: RPC_PROBE_RETRIES
+              value: "{{ .Values.pod.probes.rpc_retries }}"
 {{ dict "envAll" $envAll "component" "metadata_agent" "container" "metadata_agent" "type" "readiness" "probeTemplate" (include "metadataAgentReadinessProbeTemplate" $envAll | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }}
 {{ dict "envAll" $envAll "component" "metadata_agent" "container" "metadata_agent" "type" "liveness" "probeTemplate" (include "metadataAgentLivenessProbeTemplate" $envAll | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }}
           securityContext:

neutron/templates/daemonset-ovs-agent.yaml

@@ -177,6 +177,11 @@ spec:
         - name: neutron-ovs-agent
 {{ tuple $envAll "neutron_openvswitch_agent" | include "helm-toolkit.snippets.image" | indent 10 }}
 {{ tuple $envAll $envAll.Values.pod.resources.agent.ovs | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+          env:
+            - name: RPC_PROBE_TIMEOUT
+              value: "{{ .Values.pod.probes.rpc_timeout }}"
+            - name: RPC_PROBE_RETRIES
+              value: "{{ .Values.pod.probes.rpc_retries }}"
 {{ dict "envAll" $envAll "component" "ovs_agent" "container" "ovs_agent" "type" "readiness" "probeTemplate" (include "ovsAgentReadinessProbeTemplate" $envAll | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }}
 {{ dict "envAll" $envAll "component" "ovs_agent" "container" "ovs_agent" "type" "liveness" "probeTemplate" (include "ovsAgentLivenessProbeTemplate" $envAll | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }}
 {{ dict "envAll" $envAll "application" "neutron_ovs_agent" "container" "neutron_ovs_agent" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}

neutron/templates/daemonset-sriov-agent.yaml

@@ -135,6 +135,11 @@ spec:
 {{ tuple $envAll "neutron_sriov_agent" | include "helm-toolkit.snippets.image" | indent 10 }}
 {{ tuple $envAll $envAll.Values.pod.resources.agent.sriov | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
 {{ dict "envAll" $envAll "application" "neutron_sriov_agent" "container" "neutron_sriov_agent" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
+          env:
+            - name: RPC_PROBE_TIMEOUT
+              value: "{{ .Values.pod.probes.rpc_timeout }}"
+            - name: RPC_PROBE_RETRIES
+              value: "{{ .Values.pod.probes.rpc_retries }}"
 {{ dict "envAll" $envAll "component" "sriov_agent" "container" "sriov_agent" "type" "readiness" "probeTemplate" (include "sriovAgentReadinessProbeTemplate" $envAll | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }}
           command:
             - /tmp/neutron-sriov-agent.sh

neutron/values.yaml

@@ -341,6 +341,8 @@ pod:
   use_fqdn:
     neutron_agent: true
   probes:
+    rpc_timeout: 60
+    rpc_retries: 2
     dhcp_agent:
       dhcp_agent:
         readiness:
@@ -406,6 +408,29 @@ pod:
             initialDelaySeconds: 30
             periodSeconds: 190
             timeoutSeconds: 185
+    bagpipe_bgp:
+      bagpipe_bgp:
+        readiness:
+          enabled: true
+          params:
+        liveness:
+          enabled: true
+          params:
+            initialDelaySeconds: 60
+    l2gw_agent:
+      l2gw_agent:
+        readiness:
+          enabled: true
+          params:
+            initialDelaySeconds: 30
+            periodSeconds: 15
+            timeoutSeconds: 65
+        liveness:
+          enabled: true
+          params:
+            initialDelaySeconds: 120
+            periodSeconds: 90
+            timeoutSeconds: 70
     server:
       server:
         readiness: