From 49b58b7e7d51ac9fc793512a2a26403ac6af65e3 Mon Sep 17 00:00:00 2001 From: Hemachandra Reddy Date: Fri, 1 Mar 2019 16:49:26 +0000 Subject: [PATCH] Health probe for Nova components Health probe for Nova pods is used for both liveness and readiness probe. nova-compute, nova-conductor, nova-consoleauth and nova-scheduler: Check if the rpc socket status on the nova pods to rabbitmq and database are in established state. sends an RPC call with a non-existence method to component's queue. Probe is success if agent returns with NoSuchMethod error. If agent is not reachable or fails to respond in time, returns failure to probe. novnc/spice proxy: uses Kubernetes tcp probe on corresponding ports they expose. Added code to catch nova config file not present exception. Change-Id: Ib8e4b93486588320fd2d562c3bc90b65844e52e5 --- nova/templates/bin/_health-probe.py.tpl | 208 +++++++++++++++++++++ nova/templates/configmap-bin.yaml | 2 + nova/templates/daemonset-compute.yaml | 29 +++ nova/templates/deployment-conductor.yaml | 29 +++ nova/templates/deployment-consoleauth.yaml | 29 +++ nova/templates/deployment-novncproxy.yaml | 8 + nova/templates/deployment-scheduler.yaml | 29 +++ nova/templates/deployment-spiceproxy.yaml | 8 + 8 files changed, 342 insertions(+) create mode 100644 nova/templates/bin/_health-probe.py.tpl diff --git a/nova/templates/bin/_health-probe.py.tpl b/nova/templates/bin/_health-probe.py.tpl new file mode 100644 index 0000000000..683387476a --- /dev/null +++ b/nova/templates/bin/_health-probe.py.tpl @@ -0,0 +1,208 @@ +#!/usr/bin/env python2 + +# Copyright 2019 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Health probe script for OpenStack service that uses RPC/unix domain socket for +communication. Check's the RPC tcp socket status on the process and send +message to service through rpc call method and expects a reply. It is expected +to receive failure from the service's RPC server as the method does not exist. + +Script returns failure to Kubernetes only when + a. TCP socket for the RPC communication are not established. + b. service is not reachable or + c. service times out sending a reply. + +sys.stderr.write() writes to pod's events on failures. + +Usage example for Nova Compute: +# python health-probe-rpc.py --config-file /etc/nova/nova.conf \ +# --service-queue-name compute + +""" + +import psutil +import socket +import sys + +from oslo_config import cfg +from oslo_context import context +from oslo_log import log +import oslo_messaging + + +tcp_established = "ESTABLISHED" + + +def check_service_status(transport): + """Verify service status. Return success if service consumes message""" + try: + target = oslo_messaging.Target(topic=cfg.CONF.service_queue_name, + server=socket.gethostname()) + client = oslo_messaging.RPCClient(transport, target, + timeout=60, + retry=2) + client.call(context.RequestContext(), + 'pod_health_probe_method_ignore_errors') + except oslo_messaging.exceptions.MessageDeliveryFailure: + # Log to pod events + sys.stderr.write("Health probe unable to reach message bus") + sys.exit(0) # return success + except oslo_messaging.rpc.client.RemoteError as re: + if ("Endpoint does not support RPC method" in re.message) or \ + ("Endpoint does not support RPC version" in re.message): + sys.exit(0) # Call reached the service + else: + sys.stderr.write("Health probe unable to reach service") + sys.exit(1) # return failure + except oslo_messaging.exceptions.MessagingTimeout: + sys.stderr.write("Health probe timed out. Agent is down or response " + "timed out") + sys.exit(1) # return failure + except Exception as ex: + sys.stderr.write("Health probe caught exception sending message to " + "service: %s" % ex.message) + sys.exit(0) + except: + sys.stderr.write("Health probe caught exception sending message to" + " service") + sys.exit(0) + + +def tcp_socket_status(process, port): + """Check the tcp socket status on a process""" + sock_count = 0 + parentId = 0 + for pr in psutil.pids(): + try: + p = psutil.Process(pr) + if p.name() == process: + if parentId == 0: + parentId = p.pid + else: + if p.ppid() == parentId: + continue + pcon = p.connections() + for con in pcon: + try: + rport = con.raddr[1] + status = con.status + except IndexError: + continue + if rport == port and status == tcp_established: + sock_count = sock_count + 1 + except psutil.NoSuchProcess: + continue + + if sock_count == 0: + return 0 + else: + return 1 + + +def configured_port_in_conf(): + """Get the rabbitmq/Database port configured in config file""" + rabbitmq_port = 0 + database_port = 0 + try: + with open(sys.argv[2]) as conf_file: + for line in conf_file: + if "transport_url" in line: + rabbitmq_port = int(line.split(':', 3)[3].split('/')[0]) + elif "connection =" in line: + service = line.split(':', 3)[3].split('/')[1].rstrip('\n') + if service == "nova": + database_port = int( + line.split(':', 3)[3].split('/')[0]) + return rabbitmq_port, database_port + except IOError: + sys.stderr.write("Nova Config file not present") + sys.exit(1) + + +def test_tcp_socket(service): + """Check tcp socket to rabbitmq/db is in Established state""" + dict_services = { + "compute": "nova-compute", + "conductor": "nova-conductor", + "consoleauth": "nova-consoleaut", + "scheduler": "nova-scheduler" + } + r_port, d_port = configured_port_in_conf() + + if service in dict_services: + proc = dict_services[service] + if r_port != 0 and tcp_socket_status(proc, r_port) == 0: + sys.stderr.write("RabbitMQ socket not established") + # Do not kill the pod if RabbitMQ is not reachable/down + if not cfg.CONF.liveness_probe: + sys.exit(1) + + # let's do the db check + if service != "compute": + if d_port != 0 and tcp_socket_status(proc, d_port) == 0: + sys.stderr.write("Database socket not established") + # Do not kill the pod if database is not reachable/down + # there could be no socket as well as typically connections + # get closed after an idle timeout + # Just log it to pod events + if not cfg.CONF.liveness_probe: + sys.exit(1) + + +def test_rpc_liveness(): + """Test if service can consume message from queue""" + oslo_messaging.set_transport_defaults(control_exchange='nova') + + rabbit_group = cfg.OptGroup(name='oslo_messaging_rabbit', + title='RabbitMQ options') + cfg.CONF.register_group(rabbit_group) + cfg.CONF.register_cli_opt(cfg.StrOpt('service-queue-name')) + cfg.CONF.register_cli_opt(cfg.BoolOpt('liveness-probe', default=False, + required=False)) + + cfg.CONF(sys.argv[1:]) + + log.logging.basicConfig(level=log.ERROR) + + try: + transport = oslo_messaging.get_transport(cfg.CONF) + except Exception as ex: + sys.stderr.write("Message bus driver load error: %s" % ex.message) + sys.exit(0) # return success + + if not cfg.CONF.transport_url or \ + not cfg.CONF.service_queue_name: + sys.stderr.write("Both message bus URL and service's queue name are " + "required for health probe to work") + sys.exit(0) # return success + + try: + cfg.CONF.set_override('rabbit_max_retries', 2, + group=rabbit_group) # 3 attempts + except cfg.NoSuchOptError as ex: + cfg.CONF.register_opt(cfg.IntOpt('rabbit_max_retries', default=2), + group=rabbit_group) + + service = cfg.CONF.service_queue_name + test_tcp_socket(service) + + check_service_status(transport) + + +if __name__ == "__main__": + test_rpc_liveness() + + sys.exit(0) # return success diff --git a/nova/templates/configmap-bin.yaml b/nova/templates/configmap-bin.yaml index e422b62196..c58b90bd7e 100644 --- a/nova/templates/configmap-bin.yaml +++ b/nova/templates/configmap-bin.yaml @@ -51,6 +51,8 @@ data: ceph-admin-keyring.sh: | {{ tuple "bin/_ceph-admin-keyring.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} {{- end }} + health-probe.py: | +{{ tuple "bin/_health-probe.py.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} nova-api.sh: | {{ tuple "bin/_nova-api.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} nova-api-metadata.sh: | diff --git a/nova/templates/daemonset-compute.yaml b/nova/templates/daemonset-compute.yaml index 463ea72bb7..35236bdadc 100644 --- a/nova/templates/daemonset-compute.yaml +++ b/nova/templates/daemonset-compute.yaml @@ -180,6 +180,31 @@ spec: - name: LIBVIRT_CEPH_SECRET_UUID value: "{{ .Values.conf.ceph.secret_uuid }}" {{ end }} + readinessProbe: + exec: + command: + - python + - /tmp/health-probe.py + - --config-file + - /etc/nova/nova.conf + - --service-queue-name + - compute + initialDelaySeconds: 80 + periodSeconds: 90 + timeoutSeconds: 70 + livenessProbe: + exec: + command: + - python + - /tmp/health-probe.py + - --config-file + - /etc/nova/nova.conf + - --service-queue-name + - compute + - --liveness-probe + initialDelaySeconds: 120 + periodSeconds: 90 + timeoutSeconds: 70 command: - /tmp/nova-compute.sh volumeMounts: @@ -187,6 +212,10 @@ spec: mountPath: /tmp/nova-compute.sh subPath: nova-compute.sh readOnly: true + - name: nova-bin + mountPath: /tmp/health-probe.py + subPath: health-probe.py + readOnly: true - name: nova-etc mountPath: /etc/nova/nova.conf subPath: nova.conf diff --git a/nova/templates/deployment-conductor.yaml b/nova/templates/deployment-conductor.yaml index 33de6413c5..1e66e41932 100644 --- a/nova/templates/deployment-conductor.yaml +++ b/nova/templates/deployment-conductor.yaml @@ -60,6 +60,31 @@ spec: {{ tuple $envAll $envAll.Values.pod.resources.conductor | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} securityContext: allowPrivilegeEscalation: false + readinessProbe: + exec: + command: + - python + - /tmp/health-probe.py + - --config-file + - /etc/nova/nova.conf + - --service-queue-name + - conductor + initialDelaySeconds: 80 + periodSeconds: 90 + timeoutSeconds: 70 + livenessProbe: + exec: + command: + - python + - /tmp/health-probe.py + - --config-file + - /etc/nova/nova.conf + - --service-queue-name + - conductor + - --liveness-probe + initialDelaySeconds: 120 + periodSeconds: 90 + timeoutSeconds: 70 command: - /tmp/nova-conductor.sh volumeMounts: @@ -67,6 +92,10 @@ spec: mountPath: /tmp/nova-conductor.sh subPath: nova-conductor.sh readOnly: true + - name: nova-bin + mountPath: /tmp/health-probe.py + subPath: health-probe.py + readOnly: true - name: nova-etc mountPath: /etc/nova/nova.conf subPath: nova.conf diff --git a/nova/templates/deployment-consoleauth.yaml b/nova/templates/deployment-consoleauth.yaml index 29832d56a6..75b66e7939 100644 --- a/nova/templates/deployment-consoleauth.yaml +++ b/nova/templates/deployment-consoleauth.yaml @@ -60,6 +60,31 @@ spec: {{ tuple $envAll $envAll.Values.pod.resources.consoleauth | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} securityContext: allowPrivilegeEscalation: false + readinessProbe: + exec: + command: + - python + - /tmp/health-probe.py + - --config-file + - /etc/nova/nova.conf + - --service-queue-name + - consoleauth + initialDelaySeconds: 80 + periodSeconds: 90 + timeoutSeconds: 70 + livenessProbe: + exec: + command: + - python + - /tmp/health-probe.py + - --config-file + - /etc/nova/nova.conf + - --service-queue-name + - consoleauth + - --liveness-probe + initialDelaySeconds: 120 + periodSeconds: 90 + timeoutSeconds: 70 command: - /tmp/nova-consoleauth.sh volumeMounts: @@ -67,6 +92,10 @@ spec: mountPath: /tmp/nova-consoleauth.sh subPath: nova-consoleauth.sh readOnly: true + - name: nova-bin + mountPath: /tmp/health-probe.py + subPath: health-probe.py + readOnly: true - name: nova-etc mountPath: /etc/nova/nova.conf subPath: nova.conf diff --git a/nova/templates/deployment-novncproxy.yaml b/nova/templates/deployment-novncproxy.yaml index 8d187c8b88..cf9fda0243 100644 --- a/nova/templates/deployment-novncproxy.yaml +++ b/nova/templates/deployment-novncproxy.yaml @@ -94,6 +94,14 @@ spec: - name: nova-novncproxy {{ tuple $envAll "nova_novncproxy" | include "helm-toolkit.snippets.image" | indent 10 }} {{ tuple $envAll $envAll.Values.pod.resources.novncproxy | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + readinessProbe: + tcpSocket: + port: {{ tuple "compute_novnc_proxy" "internal" "novnc_proxy" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + initialDelaySeconds: 30 + livenessProbe: + tcpSocket: + port: {{ tuple "compute_novnc_proxy" "internal" "novnc_proxy" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + initialDelaySeconds: 30 command: - /tmp/nova-console-proxy.sh ports: diff --git a/nova/templates/deployment-scheduler.yaml b/nova/templates/deployment-scheduler.yaml index a3d46e5db0..9611d9509f 100644 --- a/nova/templates/deployment-scheduler.yaml +++ b/nova/templates/deployment-scheduler.yaml @@ -60,6 +60,31 @@ spec: {{ tuple $envAll $envAll.Values.pod.resources.scheduler | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} securityContext: allowPrivilegeEscalation: false + readinessProbe: + exec: + command: + - python + - /tmp/health-probe.py + - --config-file + - /etc/nova/nova.conf + - --service-queue-name + - scheduler + initialDelaySeconds: 80 + periodSeconds: 90 + timeoutSeconds: 70 + livenessProbe: + exec: + command: + - python + - /tmp/health-probe.py + - --config-file + - /etc/nova/nova.conf + - --service-queue-name + - scheduler + - --liveness-probe + initialDelaySeconds: 120 + periodSeconds: 90 + timeoutSeconds: 70 command: - /tmp/nova-scheduler.sh volumeMounts: @@ -67,6 +92,10 @@ spec: mountPath: /tmp/nova-scheduler.sh subPath: nova-scheduler.sh readOnly: true + - name: nova-bin + mountPath: /tmp/health-probe.py + subPath: health-probe.py + readOnly: true - name: nova-etc mountPath: /etc/nova/nova.conf subPath: nova.conf diff --git a/nova/templates/deployment-spiceproxy.yaml b/nova/templates/deployment-spiceproxy.yaml index b026d753ea..4507bde4ce 100644 --- a/nova/templates/deployment-spiceproxy.yaml +++ b/nova/templates/deployment-spiceproxy.yaml @@ -94,6 +94,14 @@ spec: - name: nova-spiceproxy {{ tuple $envAll "nova_spiceproxy" | include "helm-toolkit.snippets.image" | indent 10 }} {{ tuple $envAll $envAll.Values.pod.resources.spiceproxy | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + readinessProbe: + tcpSocket: + port: {{ tuple "compute_spice_proxy" "internal" "spice_proxy" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + initialDelaySeconds: 30 + livenessProbe: + tcpSocket: + port: {{ tuple "compute_spice_proxy" "internal" "spice_proxy" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + initialDelaySeconds: 30 command: - /tmp/nova-console-proxy.sh ports: