From 581544a03715c6e123b0a541fb4aacd6b4c1ea80 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rados=C5=82aw=20Piliszek?= <radoslaw.piliszek@gmail.com>
Date: Wed, 29 Dec 2021 10:15:55 +0000
Subject: [PATCH] [CI] Check fluentd errors

We consider them critical and prefer to fail the jobs.

Change-Id: I3c2475522f8f37e3b608d3c321aee27277fc9dee
---
 tests/check-logs.sh | 28 +++++++++++++++++++++++-----
 1 file changed, 23 insertions(+), 5 deletions(-)

diff --git a/tests/check-logs.sh b/tests/check-logs.sh
index d1795a9948..a47e89e835 100755
--- a/tests/check-logs.sh
+++ b/tests/check-logs.sh
@@ -9,13 +9,19 @@ set -o pipefail
 export PYTHONUNBUFFERED=1
 
 
-function check_file_for_level {
+function check_openstack_log_file_for_level {
     # $1: file
     # $2: log level
     # Filter out false positives from logged config options.
     sudo egrep " $2 " $1 | egrep -v "(logging_exception_prefix|rate_limit_except_level)"
 }
 
+function check_fluentd_log_file_for_level {
+    # $1: file
+    # $2: log level
+    sudo egrep "\[$2\]" $1
+}
+
 function filter_out_expected_critical {
     # $1: file
     # Filter out expected critical log messages that we do not want to fail the
@@ -34,7 +40,7 @@ function filter_out_expected_critical {
     *)
         # We have to provide some pass-through consumer to avoid:
         #   grep: write error: Broken pipe
-        # from check_file_for_level
+        # from check_openstack_log_file_for_level
         cat
         ;;
     esac
@@ -48,15 +54,15 @@ for level in CRITICAL ERROR WARNING; do
     any_matched=0
     echo "Checking for $level log messages"
     for f in $(sudo find /var/log/kolla/ -type f); do
-        if check_file_for_level $f $level >/dev/null; then
+        if check_openstack_log_file_for_level $f $level >/dev/null; then
             any_matched=1
             if [[ $level = CRITICAL ]]; then
-                if check_file_for_level $f $level | filter_out_expected_critical $f >/dev/null; then
+                if check_openstack_log_file_for_level $f $level | filter_out_expected_critical $f >/dev/null; then
                     any_critical=1
                 fi
             fi
             echo $f >> $all_file
-            check_file_for_level $f $level >> $all_file
+            check_openstack_log_file_for_level $f $level >> $all_file
             echo >> $all_file
         fi
     done
@@ -65,6 +71,18 @@ for level in CRITICAL ERROR WARNING; do
     fi
 done
 
+
+# check fluentd errors (we consider them critical)
+fluentd_log_file=/var/log/kolla/fluentd/fluentd.log
+fluentd_error_summary_file=/tmp/logs/kolla/fluentd-error.log
+if check_fluentd_log_file_for_level $fluentd_log_file error >/dev/null; then
+    any_critical=1
+    echo "(critical) Found some error log messages in fluentd logs. Matches in $fluentd_error_summary_file"
+    check_fluentd_log_file_for_level $fluentd_log_file error > $fluentd_error_summary_file
+    echo >> $fluentd_error_summary_file
+fi
+
+
 if [[ $any_critical -eq 1 ]]; then
     echo "Found critical log messages - failing job."
     exit 1