From ba54f8cddaf24a9da5c51c575a4679726136d68b Mon Sep 17 00:00:00 2001
From: Michal Nasiadka <mnasiadka@gmail.com>
Date: Wed, 22 Jul 2020 13:50:17 +0200
Subject: [PATCH] CI: Fail on fluentd log parsing errors

Change-Id: Ie3963f5ed20f7fb61ef2e03f0cf12a4ea1c87c9c
---
 tests/check-logs.sh | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/tests/check-logs.sh b/tests/check-logs.sh
index 33806845d0..fa5447c821 100755
--- a/tests/check-logs.sh
+++ b/tests/check-logs.sh
@@ -8,6 +8,7 @@ set -o pipefail
 # Enable unbuffered output for Ansible in Jenkins.
 export PYTHONUNBUFFERED=1
 
+declare -a fluentchecks=("got incomplete line before first line" "pattern not matched")
 
 function check_openstack_log_file_for_level {
     # $1: file
@@ -22,6 +23,12 @@ function check_fluentd_log_file_for_level {
     sudo egrep "\[$2\]:" $1
 }
 
+function check_fluentd_log_file_for_content {
+    # $1: file
+    # $2: content
+    sudo egrep " $2 " $1
+}
+
 function check_docker_log_file_for_sigkill {
     sudo journalctl --no-pager -u ${CONTAINER_ENGINE}.service | grep "signal 9"
 }
@@ -73,7 +80,6 @@ for level in CRITICAL ERROR WARNING; do
     fi
 done
 
-
 # check fluentd errors (we consider them critical)
 fluentd_log_file=/var/log/kolla/fluentd/fluentd.log
 fluentd_error_summary_file=/tmp/logs/kolla/fluentd-error.log
@@ -84,6 +90,16 @@ if check_fluentd_log_file_for_level $fluentd_log_file error >/dev/null; then
     echo >> $fluentd_error_summary_file
 fi
 
+for string in "${fluentchecks[@]}"; do
+    fluentd_file=/tmp/logs/kolla/fluentd-errors.log
+    if check_fluentd_log_file_for_content $fluentd_log_file "$string" >/dev/null; then
+        any_critical=1
+        echo "$string" >> $fluentd_file
+        check_fluentd_log_file_for_content $fluentd_log_file "$string" >> $fluentd_file
+        echo >> $fluentd_file
+    fi
+done
+
 if check_docker_log_file_for_sigkill >/dev/null; then
     any_critical=1
     echo "(critical) Found containers killed using signal 9 (SIGKILL) in docker logs."