From bd54b99132d019b747dcd98af9371628625c8a1b Mon Sep 17 00:00:00 2001
From: Doug Szumski <doug@stackhpc.com>
Date: Thu, 11 Oct 2018 10:12:41 +0100
Subject: [PATCH] Constrain the size of Docker logs

Even though Kolla services are configured to log output to file rather than
stdout, some stdout still occurs when for example the container re(starts).
Since the Docker logs are not constrained in size, they can fill up the
docker volumes drive and bring down the host. One example of when this is
particularly problematic is when Fluentd cannot parse a log message. The
warning output is written to the Docker log and in production we have seen
it eat 100GB of disk space in less than a day. We could configure Fluentd
not to do this, but the problem may still occur via another mechanism.

Change-Id: Ia6d3935263a5909c71750b34eb69e72e6e558b7a
Closes-Bug: #1794249
---
 ansible/group_vars/all.yml                                | 4 ++++
 .../roles/baremetal/templates/docker_systemd_service.j2   | 2 +-
 .../notes/limit-docker-log-size-33133da03b232ece.yaml     | 8 ++++++++
 3 files changed, 13 insertions(+), 1 deletion(-)
 create mode 100644 releasenotes/notes/limit-docker-log-size-33133da03b232ece.yaml

diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml
index 48a6bd82aa..d6e9110a58 100644
--- a/ansible/group_vars/all.yml
+++ b/ansible/group_vars/all.yml
@@ -92,6 +92,10 @@ docker_namespace: "kolla"
 docker_registry_username:
 docker_registry_insecure: "{{ 'yes' if docker_registry else 'no' }}"
 
+# Retention settings for Docker logs
+docker_log_max_file: 5
+docker_log_max_size: 50m
+
 # Valid options are [ never, on-failure, always, unless-stopped ]
 docker_restart_policy: "unless-stopped"
 
diff --git a/ansible/roles/baremetal/templates/docker_systemd_service.j2 b/ansible/roles/baremetal/templates/docker_systemd_service.j2
index 31b9ef5032..1a5bc8d2aa 100644
--- a/ansible/roles/baremetal/templates/docker_systemd_service.j2
+++ b/ansible/roles/baremetal/templates/docker_systemd_service.j2
@@ -1,4 +1,4 @@
 [Service]
 MountFlags=shared
 ExecStart=
-ExecStart=/usr/bin/{{ docker_binary_name|default("docker daemon", true) }}{% if docker_registry_insecure | bool %} --insecure-registry {{ docker_registry }}{% endif %}{% if docker_storage_driver %} --storage-driver {{ docker_storage_driver }}{% endif %}{% if docker_runtime_directory %} --graph {{ docker_runtime_directory }}{% endif %}{% if docker_custom_option %} {{ docker_custom_option }}{% endif %}
+ExecStart=/usr/bin/{{ docker_binary_name|default("docker daemon", true) }}{% if docker_registry_insecure | bool %} --insecure-registry {{ docker_registry }}{% endif %}{% if docker_storage_driver %} --storage-driver {{ docker_storage_driver }}{% endif %}{% if docker_runtime_directory %} --graph {{ docker_runtime_directory }}{% endif %}{% if docker_custom_option %} {{ docker_custom_option }}{% endif %} --log-opt max-file={{ docker_log_max_file }} --log-opt max-size={{ docker_log_max_size }}
diff --git a/releasenotes/notes/limit-docker-log-size-33133da03b232ece.yaml b/releasenotes/notes/limit-docker-log-size-33133da03b232ece.yaml
new file mode 100644
index 0000000000..c98e2c308f
--- /dev/null
+++ b/releasenotes/notes/limit-docker-log-size-33133da03b232ece.yaml
@@ -0,0 +1,8 @@
+---
+features:
+  - Docker logs are no longer allowed to grow unbounded and have
+    been limited to a fixed size per container. Two new variables
+    have been added, `docker_log_max_file` and `docker_log_max_size`
+    which default to 5 and 50MB respectively. This means that for
+    each container, there should be no more than 250MB of Docker
+    logs.