From 1560b01f7ee803ddd611fb5e30786c9b3ddd4a4b Mon Sep 17 00:00:00 2001
From: Clark Boylan <clark.boylan@gmail.com>
Date: Wed, 10 Feb 2021 16:32:51 -0800
Subject: [PATCH] Use dstat to record performance of system-config-run hosts

We have seen some poor performance from gitea which may be related to
manage project updates. Start a dstat service which logs to a csv file
on our system-config-run job hosts in order to collect performance info
from our services in pre merge testing. This will include gitea and
should help us evaluate service upgrades and other changes from a
performance perspective before they hit production.

Change-Id: I7bdaab0a0aeb9e1c00fcfcca3d114ae13a76ccc9
---
 playbooks/roles/dstat-logger/README.rst      |  6 +++++
 playbooks/roles/dstat-logger/tasks/main.yaml | 28 ++++++++++++++++++++
 playbooks/service-dstatlogger.yaml           |  3 +++
 playbooks/zuul/run-base.yaml                 |  2 ++
 zuul.d/system-config-run.yaml                |  1 +
 5 files changed, 40 insertions(+)
 create mode 100644 playbooks/roles/dstat-logger/README.rst
 create mode 100644 playbooks/roles/dstat-logger/tasks/main.yaml
 create mode 100644 playbooks/service-dstatlogger.yaml

diff --git a/playbooks/roles/dstat-logger/README.rst b/playbooks/roles/dstat-logger/README.rst
new file mode 100644
index 0000000000..7fcf4ea3e8
--- /dev/null
+++ b/playbooks/roles/dstat-logger/README.rst
@@ -0,0 +1,6 @@
+Install, configure, and run a dstat logger
+
+This is primarily useful for testing where we don't have instances hooked
+up to cacti. You can use this to get a csv log file at
+/var/log/dstat-csv.log in test jobs that records similar system performance
+information.
diff --git a/playbooks/roles/dstat-logger/tasks/main.yaml b/playbooks/roles/dstat-logger/tasks/main.yaml
new file mode 100644
index 0000000000..92e8a84cd7
--- /dev/null
+++ b/playbooks/roles/dstat-logger/tasks/main.yaml
@@ -0,0 +1,28 @@
+- name: Install dstat
+  become: yes
+  package:
+    name: dstat
+    state: present
+- name: Install dstat unit
+  become: yes
+  copy:
+    dest: /etc/systemd/system/dstatlogger.service
+    owner: root
+    group: root
+    mode: 0644
+    content: |
+      [Unit]
+      Description=Dstat logging service
+      [Service]
+      User=root
+      ExecStart=/usr/bin/dstat -tcmndrylpg --tcp --swap --output /var/log/dstat-csv.log
+      KillMode=process
+      TimeoutStopSec=300
+      [Install]
+      WantedBy=multi-user.target
+- name: Reload systemd
+  become: yes
+  command: systemctl daemon-reload
+- name: Start dstat unit
+  become: yes
+  command: systemctl start dstatlogger
diff --git a/playbooks/service-dstatlogger.yaml b/playbooks/service-dstatlogger.yaml
new file mode 100644
index 0000000000..0d5ab09143
--- /dev/null
+++ b/playbooks/service-dstatlogger.yaml
@@ -0,0 +1,3 @@
+- hosts: "*:!disabled"
+  roles:
+    - dstat-logger
diff --git a/playbooks/zuul/run-base.yaml b/playbooks/zuul/run-base.yaml
index 87a322edde..2c7fa8c615 100644
--- a/playbooks/zuul/run-base.yaml
+++ b/playbooks/zuul/run-base.yaml
@@ -92,6 +92,8 @@
       command: ansible-playbook -f 50 -v /home/zuul/src/opendev.org/opendev/system-config/playbooks/base.yaml
     - name: Run bridge service playbook
       command: ansible-playbook -v /home/zuul/src/opendev.org/opendev/system-config/playbooks/service-bridge.yaml
+    - name: Run dstat logger playbook
+      command: ansible-playbook -v /home/zuul/src/opendev.org/opendev/system-config/playbooks/service-dstatlogger.yaml
     - name: Run playbook
       when: run_playbooks is defined
       loop: "{{ run_playbooks }}"
diff --git a/zuul.d/system-config-run.yaml b/zuul.d/system-config-run.yaml
index b27a85bec3..7539186f4b 100644
--- a/zuul.d/system-config-run.yaml
+++ b/zuul.d/system-config-run.yaml
@@ -17,6 +17,7 @@
         '/var/log/messages': logs_txt
         '/var/log/docker': logs
         '/var/log/containers': logs
+        '/var/log/dstat-csv.log': logs
         '/etc/iptables/rules.v4': logs_txt
         '/etc/iptables/rules.v6': logs_txt
     host-vars: