Add grenade logs to logstash.

Add grenade new/ and old/ logs to logstash. To do this without tripling
HTTP GETs for every finished job add a job filter to the log gearman
client that when present only attempts to grab files if the job name
matches the job filter.

Change-Id: Ia33722bf71d482f2fd6b655b28090a10bf46af54

modules/log_processor/files/log-gearman-client.py

@@ -20,6 +20,7 @@ import gear
 import json
 import logging
 import os
+import re
 import signal
 import threading
 import yaml
@@ -66,6 +67,10 @@ class EventProcessor(threading.Thread):
         for fileopts in self.files:
             output = {}
             source_url, out_event = self._parse_event(event, fileopts)
+            job_filter = fileopts.get('job-filter')
+            if (job_filter and
+                not re.match(job_filter, out_event['fields']['build_name'])):
+                continue
             output['source_url'] = source_url
             output['retry'] = fileopts.get('retry-get', False)
             output['event'] = out_event

modules/openstack_project/files/logstash/jenkins-log-client.yaml

@@ -127,3 +127,155 @@ source-files:
   - name: logs/syslog.txt
     tags:
       - syslog
+# Grenade new logs below
+  - name: logs/new/screen-c-api.txt
+    tags:
+      - screen
+      - oslofmt
+    job-filter: '.*grenade.*'
+  - name: logs/new/screen-c-sch.txt
+    tags:
+      - screen
+      - oslofmt
+    job-filter: '.*grenade.*'
+  - name: logs/new/screen-c-vol.txt
+    tags:
+      - screen
+      - oslofmt
+    job-filter: '.*grenade.*'
+  - name: logs/new/screen-g-api.txt
+    tags:
+      - screen
+      - oslofmt
+    job-filter: '.*grenade.*'
+  - name: logs/new/screen-g-reg.txt
+    tags:
+      - screen
+      - oslofmt
+    job-filter: '.*grenade.*'
+  - name: logs/new/screen-key.txt
+    tags:
+      - screen
+      - keystonefmt
+    job-filter: '.*grenade.*'
+  - name: logs/new/screen-n-api.txt
+    tags:
+      - screen
+      - oslofmt
+    job-filter: '.*grenade.*'
+  - name: logs/new/screen-n-cond.txt
+    tags:
+      - screen
+      - oslofmt
+    job-filter: '.*grenade.*'
+  - name: logs/new/screen-n-cpu.txt
+    tags:
+      - screen
+      - oslofmt
+    job-filter: '.*grenade.*'
+  - name: logs/new/screen-n-crt.txt
+    tags:
+      - screen
+      - oslofmt
+    job-filter: '.*grenade.*'
+  - name: logs/new/screen-n-net.txt
+    tags:
+      - screen
+      - oslofmt
+    job-filter: '.*grenade.*'
+  - name: logs/new/screen-n-obj.txt
+    tags:
+      - screen
+      - oslofmt
+    job-filter: '.*grenade.*'
+  - name: logs/new/screen-s-account.txt
+    tags:
+      - screen
+      - apachecombined
+    job-filter: '.*grenade.*'
+  - name: logs/new/screen-s-container.txt
+    tags:
+      - screen
+      - apachecombined
+    job-filter: '.*grenade.*'
+  - name: logs/new/screen-s-object.txt
+    tags:
+      - screen
+      - apachecombined
+    job-filter: '.*grenade.*'
+# Grenade old logs below
+  - name: logs/old/screen-c-api.txt
+    tags:
+      - screen
+      - oslofmt
+    job-filter: '.*grenade.*'
+  - name: logs/old/screen-c-sch.txt
+    tags:
+      - screen
+      - oslofmt
+    job-filter: '.*grenade.*'
+  - name: logs/old/screen-c-vol.txt
+    tags:
+      - screen
+      - oslofmt
+    job-filter: '.*grenade.*'
+  - name: logs/old/screen-g-api.txt
+    tags:
+      - screen
+      - oslofmt
+    job-filter: '.*grenade.*'
+  - name: logs/old/screen-g-reg.txt
+    tags:
+      - screen
+      - oslofmt
+    job-filter: '.*grenade.*'
+  - name: logs/old/screen-key.txt
+    tags:
+      - screen
+      - keystonefmt
+    job-filter: '.*grenade.*'
+  - name: logs/old/screen-n-api.txt
+    tags:
+      - screen
+      - oslofmt
+    job-filter: '.*grenade.*'
+  - name: logs/old/screen-n-cond.txt
+    tags:
+      - screen
+      - oslofmt
+    job-filter: '.*grenade.*'
+  - name: logs/old/screen-n-cpu.txt
+    tags:
+      - screen
+      - oslofmt
+    job-filter: '.*grenade.*'
+  - name: logs/old/screen-n-crt.txt
+    tags:
+      - screen
+      - oslofmt
+    job-filter: '.*grenade.*'
+  - name: logs/old/screen-n-net.txt
+    tags:
+      - screen
+      - oslofmt
+    job-filter: '.*grenade.*'
+  - name: logs/old/screen-n-obj.txt
+    tags:
+      - screen
+      - oslofmt
+    job-filter: '.*grenade.*'
+  - name: logs/old/screen-s-account.txt
+    tags:
+      - screen
+      - apachecombined
+    job-filter: '.*grenade.*'
+  - name: logs/old/screen-s-container.txt
+    tags:
+      - screen
+      - apachecombined
+    job-filter: '.*grenade.*'
+  - name: logs/old/screen-s-object.txt
+    tags:
+      - screen
+      - apachecombined
+    job-filter: '.*grenade.*'