Add cinder and quantum screen logs to logstash.

* modules/openstack_project/files/logstash/jenkins-log-pusher.yaml:
Add the cinder and quantum screen log files to the log pusher. Tag them
with screen and oslofmt (screen because they are screen logs and oslofmt
because they use the oslo log message format).

* modules/openstack_project/templates/logstash/indexer.conf.erb:
Add a grep filter to remove the screen log header lines. Add a multiline
filter to handle oslo log format multi line events. Add a grok filter to
parse the oslo format logs. Handle timestamps without millisecond
precision. Remove event_message field if that message was properly
parsed.

Change-Id: Icd18e252a512416e0cce5ee0e27942b072a25e09
Reviewed-on: https://review.openstack.org/29985
Reviewed-by: James E. Blair <corvus@inaugust.com>
Approved: Clark Boylan <clark.boylan@gmail.com>
Reviewed-by: Clark Boylan <clark.boylan@gmail.com>
Tested-by: Jenkins

modules/openstack_project/files/logstash/jenkins-log-pusher.yaml

@@ -14,3 +14,35 @@ zmq-publishers:
 source-files:
   - name: console.html
     retry-get: True
+  - name: logs/screen-c-api.txt
+    tags:
+      - screen
+      - oslofmt
+  - name: logs/screen-c-sch.txt
+    tags:
+      - screen
+      - oslofmt
+  - name: logs/screen-c-vol.txt
+    tags:
+      - screen
+      - oslofmt
+  - name: logs/screen-q-agt.txt
+    tags:
+      - screen
+      - oslofmt
+  - name: logs/screen-q-dhcp.txt
+    tags:
+      - screen
+      - oslofmt
+  - name: logs/screen-q-l3.txt
+    tags:
+      - screen
+      - oslofmt
+  - name: logs/screen-q-meta.txt
+    tags:
+      - screen
+      - oslofmt
+  - name: logs/screen-q-svc.txt
+    tags:
+      - screen
+      - oslofmt

modules/openstack_project/templates/logstash/indexer.conf.erb

@@ -11,12 +11,21 @@ input {
 # You can check grok patterns at http://grokdebug.herokuapp.com/
 filter {
   grep {
+    # Remove unneeded html tags.
     type => "jenkins"
     tags => ["console.html"]
     # Drop matches.
     negate => true
     match => ["@message", "^</?pre>$"]
   }
+  grep {
+    # Remove screen log headers.
+    type => "jenkins"
+    tags => ["screen"]
+    # Drop matches.
+    negate => true
+    match => ["@message", "^\+ "]
+  }
   multiline {
     type => "jenkins"
     tags => ["console.html"]
@@ -24,6 +33,13 @@ filter {
     pattern => "^%{DATESTAMP} \|"
     what => "previous"
   }
+  multiline {
+    type => "jenkins"
+    tags => ["oslofmt"]
+    negate => true
+    pattern => "^%{DATESTAMP} "
+    what => "previous"
+  }
   grok {
     type => "jenkins"
     tags => ["console.html"]
@@ -32,10 +48,19 @@ filter {
     pattern => [ "(?m)^%{DATESTAMP:logdate} \| %{GREEDYDATA:logmessage}" ]
     add_field => [ "received_at", "%{@timestamp}" ]
   }
+  grok {
+    type => "jenkins"
+    tags => ["oslofmt"]
+    # Do multiline matching as the above mutliline filter may add newlines
+    # to the log messages.
+    # TODO move the LOGLEVELs into a proper grok pattern.
+    pattern => [ "(?m)^%{DATESTAMP:logdate}%{SPACE}%{NUMBER:pid}?%{SPACE}?(?<loglevel>AUDIT|CRITICAL|DEBUG|INFO|WARNING|ERROR) \[?\b%{NOTSPACE:module}\b\]?%{SPACE}?%{GREEDYDATA:logmessage}?" ]
+    add_field => [ "received_at", "%{@timestamp}" ]
+  }
   date {
     type => "jenkins"
     exclude_tags => "_grokparsefailure"
-    match => [ "logdate", "yyyy-MM-dd HH:mm:ss.SSS" ]
+    match => [ "logdate", "yyyy-MM-dd HH:mm:ss.SSS", "yyyy-MM-dd HH:mm:ss" ]
   }
   mutate {
     type => "jenkins"
@@ -45,7 +70,7 @@ filter {
   mutate {
     type => "jenkins"
     exclude_tags => "_grokparsefailure"
-    remove => [ "logdate", "logmessage" ]
+    remove => [ "logdate", "logmessage", "event_message" ]
   }
 }