haproxy: redirect logs to a separate file

haproxy only logs to /dev/log; this means all our access logs get
mixed into syslog.  This makes it impossible to pick out anything in
syslog that might be interesting (and vice-versa, means you have to
filter out things if analysing just the haproxy logs).

It seems like the standard way to deal with this is to have rsyslogd
listen on a separate socket, and then point haproxy to that.  So this
configures rsyslogd to create /var/run/dev/log and maps that into the
container as /dev/log (i.e. don't have to reconfigure the container at
all).

We then capture this sockets logs to /var/log/haproxy.log, and install
rotation for it.

Additionally we collect this log from our tests.

Change-Id: I32948793df7fd9b990c948730349b24361a8f307

playbooks/roles/haproxy/files/rsyslog.d/49-haproxy.conf

@@ -0,0 +1,8 @@
+# Create additional haproxy socket to be mapped into container
+
+$AddUnixListenSocket /var/haproxy/dev/log
+
+:programname, startswith, "haproxy" {
+   /var/log/haproxy.log
+   stop
+}

playbooks/roles/haproxy/tasks/main.yaml

@@ -12,12 +12,34 @@
   loop:
     - etc
     - run
+    - dev
 
 - name: Ensure haproxy config template available
   assert:
     that:
       - haproxy_config_template is defined
 
+- name: Write rsyslog file
+  copy:
+    src: rsyslog.d/49-haproxy.conf
+    dest: /etc/rsyslog.d/
+    owner: root
+    group: root
+    mode: 0644
+  register: _rsyslog_added
+
+- name: Restart rsyslog if config updates
+  service:
+    name: rsyslog
+    state: restarted
+  when: _rsyslog_added.changed
+
+- name: Add haproxy log rotation
+  include_role:
+    name: logrotate
+  vars:
+    logrotate_file_name: '/var/log/haproxy.log'
+
 - name: Write haproxy config file
   template:
     src: '{{ haproxy_config_template }}'

playbooks/roles/haproxy/templates/docker-compose.yaml.j2

@@ -35,7 +35,7 @@ services:
     user: "root:root"
     network_mode: host
     volumes:
-      - /dev/log:/dev/log
+      - /var/haproxy/dev/log:/dev/log
       - /var/haproxy/etc:/usr/local/etc/haproxy:ro
       - /var/haproxy/run:/var/haproxy/run
     logging:

zuul.d/system-config-run.yaml

@@ -611,6 +611,7 @@
       gitea-lb01.opendev.org:
         host_copy_output:
           '/var/haproxy/etc': logs
+          '/var/log/haproxy.log': logs
     files:
       - playbooks/bootstrap-bridge.yaml
       - playbooks/letsencrypt.yaml
@@ -894,6 +895,7 @@
       zuul-lb01.opendev.org:
         host_copy_output:
           '/var/haproxy/etc': logs
+          '/var/log/haproxy.log': logs
     files:
       - playbooks/bootstrap-bridge.yaml
       - playbooks/service-zookeeper.yaml