openstack/kolla-ansible
Code Issues Proposed changes

Merge "[CI] Check fluentd errors"

Browse Source
This commit is contained in:
Zuul 2022-01-27 11:13:55 +00:00 committed by Gerrit Code Review
commit e4bfedd731
1 changed files with 23 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
tests/check-logs.sh
View File

@ -9,13 +9,19 @@ set -o pipefail
export PYTHONUNBUFFERED=1 export PYTHONUNBUFFERED=1
function check_file_for_level { function check_openstack_log_file_for_level {
# $1: file # $1: file
# $2: log level # $2: log level
# Filter out false positives from logged config options. # Filter out false positives from logged config options.
sudo egrep " $2 " $1 | egrep -v "(logging_exception_prefix|rate_limit_except_level)" sudo egrep " $2 " $1 | egrep -v "(logging_exception_prefix|rate_limit_except_level)"
} }
function check_fluentd_log_file_for_level {
# $1: file
# $2: log level
sudo egrep "\[$2\]" $1
}
function filter_out_expected_critical { function filter_out_expected_critical {
# $1: file # $1: file
# Filter out expected critical log messages that we do not want to fail the # Filter out expected critical log messages that we do not want to fail the
@ -35,7 +41,7 @@ function filter_out_expected_critical {
*) *)
# We have to provide some pass-through consumer to avoid: # We have to provide some pass-through consumer to avoid:
# grep: write error: Broken pipe # grep: write error: Broken pipe
# from check_file_for_level # from check_openstack_log_file_for_level
cat cat
;; ;;
esac esac
@ -49,15 +55,15 @@ for level in CRITICAL ERROR WARNING; do
any_matched=0 any_matched=0
echo "Checking for $level log messages" echo "Checking for $level log messages"
for f in $(sudo find /var/log/kolla/ -type f); do for f in $(sudo find /var/log/kolla/ -type f); do
if check_file_for_level $f $level >/dev/null; then if check_openstack_log_file_for_level $f $level >/dev/null; then
any_matched=1 any_matched=1
if [[ $level = CRITICAL ]]; then if [[ $level = CRITICAL ]]; then
if check_file_for_level $f $level | filter_out_expected_critical $f >/dev/null; then if check_openstack_log_file_for_level $f $level | filter_out_expected_critical $f >/dev/null; then
any_critical=1 any_critical=1
fi fi
fi fi
echo $f >> $all_file echo $f >> $all_file
check_file_for_level $f $level >> $all_file check_openstack_log_file_for_level $f $level >> $all_file
echo >> $all_file echo >> $all_file
fi fi
done done
@ -66,6 +72,18 @@ for level in CRITICAL ERROR WARNING; do
fi fi
done done
# check fluentd errors (we consider them critical)
fluentd_log_file=/var/log/kolla/fluentd/fluentd.log
fluentd_error_summary_file=/tmp/logs/kolla/fluentd-error.log
if check_fluentd_log_file_for_level $fluentd_log_file error >/dev/null; then
any_critical=1
echo "(critical) Found some error log messages in fluentd logs. Matches in $fluentd_error_summary_file"
check_fluentd_log_file_for_level $fluentd_log_file error > $fluentd_error_summary_file
echo >> $fluentd_error_summary_file
fi
if [[ $any_critical -eq 1 ]]; then if [[ $any_critical -eq 1 ]]; then
echo "Found critical log messages - failing job." echo "Found critical log messages - failing job."
exit 1 exit 1