From f97b9b8b8bb4dfad24ce4f2949f023ae900646e9 Mon Sep 17 00:00:00 2001
From: Ian Wienand <iwienand@redhat.com>
Date: Fri, 10 Jun 2022 14:37:14 +1000
Subject: [PATCH] haproxy: redirect logs to a separate file

haproxy only logs to /dev/log; this means all our access logs get
mixed into syslog.  This makes it impossible to pick out anything in
syslog that might be interesting (and vice-versa, means you have to
filter out things if analysing just the haproxy logs).

It seems like the standard way to deal with this is to have rsyslogd
listen on a separate socket, and then point haproxy to that.  So this
configures rsyslogd to create /var/run/dev/log and maps that into the
container as /dev/log (i.e. don't have to reconfigure the container at
all).

We then capture this sockets logs to /var/log/haproxy.log, and install
rotation for it.

Additionally we collect this log from our tests.

Change-Id: I32948793df7fd9b990c948730349b24361a8f307
---
 .../haproxy/files/rsyslog.d/49-haproxy.conf   |  8 +++++++
 playbooks/roles/haproxy/tasks/main.yaml       | 22 +++++++++++++++++++
 .../haproxy/templates/docker-compose.yaml.j2  |  2 +-
 zuul.d/system-config-run.yaml                 |  2 ++
 4 files changed, 33 insertions(+), 1 deletion(-)
 create mode 100644 playbooks/roles/haproxy/files/rsyslog.d/49-haproxy.conf

diff --git a/playbooks/roles/haproxy/files/rsyslog.d/49-haproxy.conf b/playbooks/roles/haproxy/files/rsyslog.d/49-haproxy.conf
new file mode 100644
index 0000000000..402c122984
--- /dev/null
+++ b/playbooks/roles/haproxy/files/rsyslog.d/49-haproxy.conf
@@ -0,0 +1,8 @@
+# Create additional haproxy socket to be mapped into container
+
+$AddUnixListenSocket /var/haproxy/dev/log
+
+:programname, startswith, "haproxy" {
+   /var/log/haproxy.log
+   stop
+}
diff --git a/playbooks/roles/haproxy/tasks/main.yaml b/playbooks/roles/haproxy/tasks/main.yaml
index 5cfce31519..4c7913a023 100644
--- a/playbooks/roles/haproxy/tasks/main.yaml
+++ b/playbooks/roles/haproxy/tasks/main.yaml
@@ -12,12 +12,34 @@
   loop:
     - etc
     - run
+    - dev
 
 - name: Ensure haproxy config template available
   assert:
     that:
       - haproxy_config_template is defined
 
+- name: Write rsyslog file
+  copy:
+    src: rsyslog.d/49-haproxy.conf
+    dest: /etc/rsyslog.d/
+    owner: root
+    group: root
+    mode: 0644
+  register: _rsyslog_added
+
+- name: Restart rsyslog if config updates
+  service:
+    name: rsyslog
+    state: restarted
+  when: _rsyslog_added.changed
+
+- name: Add haproxy log rotation
+  include_role:
+    name: logrotate
+  vars:
+    logrotate_file_name: '/var/log/haproxy.log'
+
 - name: Write haproxy config file
   template:
     src: '{{ haproxy_config_template }}'
diff --git a/playbooks/roles/haproxy/templates/docker-compose.yaml.j2 b/playbooks/roles/haproxy/templates/docker-compose.yaml.j2
index d28b52c4e1..04488bde3f 100644
--- a/playbooks/roles/haproxy/templates/docker-compose.yaml.j2
+++ b/playbooks/roles/haproxy/templates/docker-compose.yaml.j2
@@ -35,7 +35,7 @@ services:
     user: "root:root"
     network_mode: host
     volumes:
-      - /dev/log:/dev/log
+      - /var/haproxy/dev/log:/dev/log
       - /var/haproxy/etc:/usr/local/etc/haproxy:ro
       - /var/haproxy/run:/var/haproxy/run
     logging:
diff --git a/zuul.d/system-config-run.yaml b/zuul.d/system-config-run.yaml
index 80863fbc6a..20b6583c96 100644
--- a/zuul.d/system-config-run.yaml
+++ b/zuul.d/system-config-run.yaml
@@ -611,6 +611,7 @@
       gitea-lb01.opendev.org:
         host_copy_output:
           '/var/haproxy/etc': logs
+          '/var/log/haproxy.log': logs
     files:
       - playbooks/bootstrap-bridge.yaml
       - playbooks/letsencrypt.yaml
@@ -894,6 +895,7 @@
       zuul-lb01.opendev.org:
         host_copy_output:
           '/var/haproxy/etc': logs
+          '/var/log/haproxy.log': logs
     files:
       - playbooks/bootstrap-bridge.yaml
       - playbooks/service-zookeeper.yaml