Check for CRITICAL, WARNING and ERROR log messages in CI

At the end of a CI run, check all log files.

Change-Id: I99afc1c5207757e35beabf7daebd86c56151c96d

tests/check-logs.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+# Check for CRITICAL, ERROR or WARNING messages in log files.
+
+set -o errexit
+
+# Enable unbuffered output for Ansible in Jenkins.
+export PYTHONUNBUFFERED=1
+
+
+for level in CRITICAL ERROR WARNING; do
+    all_file=/tmp/logs/kolla/all-${level}.log
+    any_matched=0
+    any_critical=0
+    echo "Checking for $level log messages"
+    for f in $(sudo find /var/log/kolla/ -type f); do
+        if sudo egrep "^.* .* .* $level" $f >/dev/null; then
+            any_matched=1
+            if [[ $level = CRITICAL ]]; then
+                any_critical=1
+            fi
+            echo $f >> $all_file
+            sudo egrep "^.* .* .* $level" $f >> $all_file
+            echo >> $all_file
+        fi
+    done
+    if [[ $any_matched -eq 1 ]]; then
+        echo "Found some $level log messages. Matches in $all_file"
+    fi
+done
+
+if [[ $any_critical -eq 1 ]]; then
+    echo "Found critical log messages - failing job."
+    exit 1
+fi

tests/run.yml

@@ -248,6 +248,14 @@
             cmd: tests/check-config.sh
             executable: /bin/bash
             chdir: "{{ kolla_ansible_src_dir }}"
+
+        # Using script rather than shell here because check-logs.sh does not
+        # exist in Stein branch.
+        - name: Run check-logs.sh script
+          script:
+            cmd: check-logs.sh
+            executable: /bin/bash
+            chdir: "{{ kolla_ansible_src_dir }}"
       when: is_upgrade
 
 - hosts: primary
@@ -368,3 +376,9 @@
         cmd: tests/check-config.sh
         executable: /bin/bash
         chdir: "{{ kolla_ansible_src_dir }}"
+
+    - name: Run check-logs.sh script
+      shell:
+        cmd: tests/check-logs.sh
+        executable: /bin/bash
+        chdir: "{{ kolla_ansible_src_dir }}"