From 538793a0517345bbe6c106bd889cf1d2a3be6e52 Mon Sep 17 00:00:00 2001
From: Clark Boylan <clark.boylan@gmail.com>
Date: Tue, 21 May 2013 12:59:57 -0700
Subject: [PATCH] Add cinder and quantum screen logs to logstash.

* modules/openstack_project/files/logstash/jenkins-log-pusher.yaml:
Add the cinder and quantum screen log files to the log pusher. Tag them
with screen and oslofmt (screen because they are screen logs and oslofmt
because they use the oslo log message format).

* modules/openstack_project/templates/logstash/indexer.conf.erb:
Add a grep filter to remove the screen log header lines. Add a multiline
filter to handle oslo log format multi line events. Add a grok filter to
parse the oslo format logs. Handle timestamps without millisecond
precision. Remove event_message field if that message was properly
parsed.

Change-Id: Icd18e252a512416e0cce5ee0e27942b072a25e09
Reviewed-on: https://review.openstack.org/29985
Reviewed-by: James E. Blair <corvus@inaugust.com>
Approved: Clark Boylan <clark.boylan@gmail.com>
Reviewed-by: Clark Boylan <clark.boylan@gmail.com>
Tested-by: Jenkins
---
 .../files/logstash/jenkins-log-pusher.yaml    | 32 +++++++++++++++++++
 .../templates/logstash/indexer.conf.erb       | 29 +++++++++++++++--
 2 files changed, 59 insertions(+), 2 deletions(-)

diff --git a/modules/openstack_project/files/logstash/jenkins-log-pusher.yaml b/modules/openstack_project/files/logstash/jenkins-log-pusher.yaml
index fbb1a57a24..50b3b33ad3 100644
--- a/modules/openstack_project/files/logstash/jenkins-log-pusher.yaml
+++ b/modules/openstack_project/files/logstash/jenkins-log-pusher.yaml
@@ -14,3 +14,35 @@ zmq-publishers:
 source-files:
   - name: console.html
     retry-get: True
+  - name: logs/screen-c-api.txt
+    tags:
+      - screen
+      - oslofmt
+  - name: logs/screen-c-sch.txt
+    tags:
+      - screen
+      - oslofmt
+  - name: logs/screen-c-vol.txt
+    tags:
+      - screen
+      - oslofmt
+  - name: logs/screen-q-agt.txt
+    tags:
+      - screen
+      - oslofmt
+  - name: logs/screen-q-dhcp.txt
+    tags:
+      - screen
+      - oslofmt
+  - name: logs/screen-q-l3.txt
+    tags:
+      - screen
+      - oslofmt
+  - name: logs/screen-q-meta.txt
+    tags:
+      - screen
+      - oslofmt
+  - name: logs/screen-q-svc.txt
+    tags:
+      - screen
+      - oslofmt
diff --git a/modules/openstack_project/templates/logstash/indexer.conf.erb b/modules/openstack_project/templates/logstash/indexer.conf.erb
index c03ffcc80a..940f77a8d1 100644
--- a/modules/openstack_project/templates/logstash/indexer.conf.erb
+++ b/modules/openstack_project/templates/logstash/indexer.conf.erb
@@ -11,12 +11,21 @@ input {
 # You can check grok patterns at http://grokdebug.herokuapp.com/
 filter {
   grep {
+    # Remove unneeded html tags.
     type => "jenkins"
     tags => ["console.html"]
     # Drop matches.
     negate => true
     match => ["@message", "^</?pre>$"]
   }
+  grep {
+    # Remove screen log headers.
+    type => "jenkins"
+    tags => ["screen"]
+    # Drop matches.
+    negate => true
+    match => ["@message", "^\+ "]
+  }
   multiline {
     type => "jenkins"
     tags => ["console.html"]
@@ -24,6 +33,13 @@ filter {
     pattern => "^%{DATESTAMP} \|"
     what => "previous"
   }
+  multiline {
+    type => "jenkins"
+    tags => ["oslofmt"]
+    negate => true
+    pattern => "^%{DATESTAMP} "
+    what => "previous"
+  }
   grok {
     type => "jenkins"
     tags => ["console.html"]
@@ -32,10 +48,19 @@ filter {
     pattern => [ "(?m)^%{DATESTAMP:logdate} \| %{GREEDYDATA:logmessage}" ]
     add_field => [ "received_at", "%{@timestamp}" ]
   }
+  grok {
+    type => "jenkins"
+    tags => ["oslofmt"]
+    # Do multiline matching as the above mutliline filter may add newlines
+    # to the log messages.
+    # TODO move the LOGLEVELs into a proper grok pattern.
+    pattern => [ "(?m)^%{DATESTAMP:logdate}%{SPACE}%{NUMBER:pid}?%{SPACE}?(?<loglevel>AUDIT|CRITICAL|DEBUG|INFO|WARNING|ERROR) \[?\b%{NOTSPACE:module}\b\]?%{SPACE}?%{GREEDYDATA:logmessage}?" ]
+    add_field => [ "received_at", "%{@timestamp}" ]
+  }
   date {
     type => "jenkins"
     exclude_tags => "_grokparsefailure"
-    match => [ "logdate", "yyyy-MM-dd HH:mm:ss.SSS" ]
+    match => [ "logdate", "yyyy-MM-dd HH:mm:ss.SSS", "yyyy-MM-dd HH:mm:ss" ]
   }
   mutate {
     type => "jenkins"
@@ -45,7 +70,7 @@ filter {
   mutate {
     type => "jenkins"
     exclude_tags => "_grokparsefailure"
-    remove => [ "logdate", "logmessage" ]
+    remove => [ "logdate", "logmessage", "event_message" ]
   }
 }