From df335525ab0d7848566a43d497d1b926cbca4970 Mon Sep 17 00:00:00 2001
From: Clark Boylan <clark.boylan@gmail.com>
Date: Mon, 14 Feb 2022 15:38:45 -0800
Subject: [PATCH] Haproxy http checks for Gitea

Previously we were only checking that Apache can open TCP connections to
determine if Gitea is up or down on a backend. This is insufficient
because Gitea itself may be down while Apache is up. In this situation
TCP connection to Apache will function, but if we make an HTTP request
we should get back an error.

To check if both Apache and Gitea are working properly we switch to
using http checks instead. Then if Gitea is down Apache can return a 500
and the Gitea backend will be removed from the pool. Similarly if Apache
is non functional the check will fail to connect via TCP.

Note we don't verify ssl certs for simplicity as checking these in
testing is not straightforward. We didn't have verification with the old
tcp checks so this isn't a regression, but does represent something we
could try and improve in the future.

Change-Id: Id47a1f9028c7575e8fbbd10fabfc9730095cb541
---
 inventory/service/group_vars/gitea-lb.yaml       | 16 ++++++++++++++++
 .../gitea-lb/templates/gitea-haproxy.cfg.j2      |  4 ++--
 .../zuul/templates/group_vars/gitea-lb.yaml.j2   |  2 ++
 3 files changed, 20 insertions(+), 2 deletions(-)

diff --git a/inventory/service/group_vars/gitea-lb.yaml b/inventory/service/group_vars/gitea-lb.yaml
index 41780cc8e8..6b9f3d8d91 100644
--- a/inventory/service/group_vars/gitea-lb.yaml
+++ b/inventory/service/group_vars/gitea-lb.yaml
@@ -5,40 +5,56 @@ gitea_lb_listeners:
     servers:
       - name: 'gitea01.opendev.org'
         address: '38.108.68.172:3080'
+        check_method: 'check'
       - name: 'gitea02.opendev.org'
         address: '38.108.68.23:3080'
+        check_method: 'check'
       - name: 'gitea03.opendev.org'
         address: '38.108.68.110:3080'
+        check_method: 'check'
       - name: 'gitea04.opendev.org'
         address: '38.108.68.147:3080'
+        check_method: 'check'
       - name: 'gitea05.opendev.org'
         address: '38.108.68.131:3080'
+        check_method: 'check'
       - name: 'gitea06.opendev.org'
         address: '38.108.68.250:3080'
+        check_method: 'check'
       - name: 'gitea07.opendev.org'
         address: '38.108.68.134:3080'
+        check_method: 'check'
       - name: 'gitea08.opendev.org'
         address: '38.108.68.205:3080'
+        check_method: 'check'
   - name: balance_git_https
     bind:
       - ':::443'
     servers:
       - name: 'gitea01.opendev.org'
         address: '38.108.68.172:3081'
+        check_method: 'check-ssl verify none'
       - name: 'gitea02.opendev.org'
         address: '38.108.68.23:3081'
+        check_method: 'check-ssl verify none'
       - name: 'gitea03.opendev.org'
         address: '38.108.68.110:3081'
+        check_method: 'check-ssl verify none'
       - name: 'gitea04.opendev.org'
         address: '38.108.68.147:3081'
+        check_method: 'check-ssl verify none'
       - name: 'gitea05.opendev.org'
         address: '38.108.68.131:3081'
+        check_method: 'check-ssl verify none'
       - name: 'gitea06.opendev.org'
         address: '38.108.68.250:3081'
+        check_method: 'check-ssl verify none'
       - name: 'gitea07.opendev.org'
         address: '38.108.68.134:3081'
+        check_method: 'check-ssl verify none'
       - name: 'gitea08.opendev.org'
         address: '38.108.68.205:3081'
+        check_method: 'check-ssl verify none'
 iptables_extra_public_tcp_ports:
   - 443
   - 80
diff --git a/playbooks/roles/gitea-lb/templates/gitea-haproxy.cfg.j2 b/playbooks/roles/gitea-lb/templates/gitea-haproxy.cfg.j2
index 200de378ba..916f39738a 100644
--- a/playbooks/roles/gitea-lb/templates/gitea-haproxy.cfg.j2
+++ b/playbooks/roles/gitea-lb/templates/gitea-haproxy.cfg.j2
@@ -27,10 +27,10 @@ listen {{ listener.name }}
   {% endfor %}
   mode tcp
   balance source
-  option tcp-check
+  option httpchk
 
   {% for server in listener.servers %}
-  server {{ server.name }} {{ server.address }} check
+  server {{ server.name }} {{ server.address }} {{ server.check_method }}
   {% endfor %}
 
 {% endfor %}
diff --git a/playbooks/zuul/templates/group_vars/gitea-lb.yaml.j2 b/playbooks/zuul/templates/group_vars/gitea-lb.yaml.j2
index 9eb0f3f456..5b70eb3c8b 100644
--- a/playbooks/zuul/templates/group_vars/gitea-lb.yaml.j2
+++ b/playbooks/zuul/templates/group_vars/gitea-lb.yaml.j2
@@ -5,9 +5,11 @@ gitea_lb_listeners:
     servers:
       - name: "gitea99.opendev.org"
         address: "{{ (hostvars['gitea99.opendev.org'] | default({})).get('nodepool', {}).get('public_ipv4', '') }}:3080"
+        check_method: "check"
   - name: balance_git_https
     bind:
       - ":::443"
     servers:
       - name: "gitea99.opendev.org"
         address: "{{ (hostvars['gitea99.opendev.org'] | default({})).get('nodepool', {}).get('public_ipv4', '') }}:3000"
+        check_method: "check-ssl verify none"