Split keepalived liveness checks for internal/external networks

If external connectivity fails, it is important that internal
services can still access an HAProxy instance. The current
defaults can cause a situation where all keepalived instances
enter the fault state despite internal connectivity being
available.

This patch splits the ping checks to allow deployments to define
a separate ping check for internal and external connectivity to
ensure that when one instance fails the other VIP remains in
operation.

Change-Id: Ideb34c43d1b1a30499cc88f28406cfa0368713ea

etc/openstack_deploy/openstack_user_config.yml.example

@@ -826,6 +826,7 @@
 # deployment. Refer to the ``user_variables.yml`` file for
 # more information.
 #
-# Keepalived cam ping a public IP address to check its status.  To enable this
-# feature, set the ``keepalived_ping_address`` variable in the
-# ``user_variables.yml`` file.
+# Keepalived can ping a public and private IP address to check its status.  To
+# enable this feature, set the ``keepalived_external_ping_address`` and
+# ``keepalived_internal_ping_address`` variables in the ``user_variables.yml``
+# file.

etc/openstack_deploy/user_variables.yml

@@ -190,4 +190,5 @@ install_method: source
 # haproxy_keepalived_priority_backup:
 
 # Keepalived default IP address used to check its alive status (IPv4 only)
-# keepalived_ping_address: "193.0.14.129"
+# keepalived_external_ping_address: "193.0.14.129"
+# keepalived_internal_ping_address: "193.0.14.129"

inventory/group_vars/haproxy/keepalived.yml

@@ -17,6 +17,8 @@ keepalived_ping_count: 1
 keepalived_ping_interval: 10
 keepalived_ubuntu_src: "native"
 keepalived_ping_address: "{{ ansible_facts['default_ipv4']['gateway'] | default('127.0.0.1') }}"
+keepalived_external_ping_address: "{{ keepalived_ping_address }}"
+keepalived_internal_ping_address: "{{ keepalived_ping_address }}"
 
 keepalived_global_defs:
   - "enable_script_security"
@@ -29,11 +31,20 @@ keepalived_scripts:
     ##on the deploy host to the check_script location. If the check_script needs
     ##parameters, you can define the location under dest_check_script.
     src_check_script: "{{ playbook_dir }}/../scripts/keepalived_haproxy_check.sh"
-  pingable_check_script:
-    check_script: "/etc/keepalived/pingable_check.sh {{ keepalived_ping_count }} {{ keepalived_ping_address }}"
+  pingable_check_script_external:
+    check_script: "/etc/keepalived/pingable_check.sh {{ keepalived_ping_count }} {{ keepalived_external_ping_address }}"
     dest_check_script: "/etc/keepalived/pingable_check.sh"
     src_check_script: "{{ playbook_dir }}/../scripts/keepalived_pingable_check.sh"
     interval: "{{ keepalived_ping_interval }}"
+    instance: external
+    fall: 2
+    rise: 4
+  pingable_check_script_internal:
+    check_script: "/etc/keepalived/pingable_check.sh {{ keepalived_ping_count }} {{ keepalived_internal_ping_address }}"
+    dest_check_script: "/etc/keepalived/pingable_check.sh"
+    src_check_script: "{{ playbook_dir }}/../scripts/keepalived_pingable_check.sh"
+    interval: "{{ keepalived_ping_interval }}"
+    instance: internal
     fall: 2
     rise: 4
 
@@ -51,13 +62,13 @@ keepalived_instances:
     authentication_password: "{{ haproxy_keepalived_authentication_password }}"
     vips:
       - "{{ haproxy_keepalived_external_vip_cidr | default('169.254.1.1/24')  }} dev {{ haproxy_keepalived_external_interface | default(management_bridge) }}"
-    track_scripts: "{{ keepalived_scripts.keys() | list }}"
+    track_scripts: "{{ keepalived_scripts | dict2items | json_query('[*].{name: key, instance: value.instance}') | rejectattr('instance', 'equalto', 'internal') | map(attribute='name') | list }}"
   internal:
     interface: "{{ haproxy_keepalived_internal_interface | default(management_bridge) }}"
     state: "{{ (groups['haproxy'].index(inventory_hostname) == 0) | ternary('MASTER', 'BACKUP') }}"
     virtual_router_id: "{{ haproxy_keepalived_internal_virtual_router_id | default ('11') }}"
     priority: "{{ (groups['haproxy']|length-groups['haproxy'].index(inventory_hostname))*50 }}"
     authentication_password: "{{ haproxy_keepalived_authentication_password }}"
-    track_scripts: "{{ keepalived_scripts.keys() | list }}"
     vips:
       - "{{ haproxy_keepalived_internal_vip_cidr | default('169.254.2.1/24') }} dev {{ haproxy_keepalived_internal_interface | default(management_bridge) }}"
+    track_scripts: "{{ keepalived_scripts | dict2items | json_query('[*].{name: key, instance: value.instance}') | rejectattr('instance', 'equalto', 'external') | map(attribute='name') | list }}"

playbooks/healthcheck-infrastructure.yml

@@ -40,8 +40,12 @@
   hosts: haproxy
   gather_facts: yes
   tasks:
-    - name: Check if host can connect to keepalived ping IP
-      command: "ping -c 2 {{ keepalived_ping_address }}"
+    - name: Check if host can connect to external keepalived ping IP
+      command: "ping -c 2 {{ keepalived_external_ping_address }}"
+      changed_when: false
+
+    - name: Check if host can connect to internal keepalived ping IP
+      command: "ping -c 2 {{ keepalived_internal_ping_address }}"
       changed_when: false
 
     - name: Checking if keepalived is running

releasenotes/notes/keepalived-checks-b184a87635c3b188.yaml

@@ -0,0 +1,8 @@
+---
+features:
+  - |
+    New variables 'keepalived_internal_ping_address' and
+    'keepalived_external_ping_address' allow deployments to decouple liveness
+    checks for HAProxy accessibility via internal and external networks. The
+    previous 'keepalived_ping_address' variable is maintained for backwards
+    compatibility.