From 5f78a77de0b70858277ac1206ce803f0f7882394 Mon Sep 17 00:00:00 2001
From: Clark Boylan <clark.boylan@gmail.com>
Date: Wed, 29 Jan 2014 14:01:58 -0800
Subject: [PATCH] Add grenade logs to logstash.

Add grenade new/ and old/ logs to logstash. To do this without tripling
HTTP GETs for every finished job add a job filter to the log gearman
client that when present only attempts to grab files if the job name
matches the job filter.

Change-Id: Ia33722bf71d482f2fd6b655b28090a10bf46af54
---
 .../log_processor/files/log-gearman-client.py |   5 +
 .../files/logstash/jenkins-log-client.yaml    | 152 ++++++++++++++++++
 2 files changed, 157 insertions(+)

diff --git a/modules/log_processor/files/log-gearman-client.py b/modules/log_processor/files/log-gearman-client.py
index 09e02e7e46..0f36649aec 100644
--- a/modules/log_processor/files/log-gearman-client.py
+++ b/modules/log_processor/files/log-gearman-client.py
@@ -20,6 +20,7 @@ import gear
 import json
 import logging
 import os
+import re
 import signal
 import threading
 import yaml
@@ -66,6 +67,10 @@ class EventProcessor(threading.Thread):
         for fileopts in self.files:
             output = {}
             source_url, out_event = self._parse_event(event, fileopts)
+            job_filter = fileopts.get('job-filter')
+            if (job_filter and
+                not re.match(job_filter, out_event['fields']['build_name'])):
+                continue
             output['source_url'] = source_url
             output['retry'] = fileopts.get('retry-get', False)
             output['event'] = out_event
diff --git a/modules/openstack_project/files/logstash/jenkins-log-client.yaml b/modules/openstack_project/files/logstash/jenkins-log-client.yaml
index 93fd07917c..9f4c8d18b2 100644
--- a/modules/openstack_project/files/logstash/jenkins-log-client.yaml
+++ b/modules/openstack_project/files/logstash/jenkins-log-client.yaml
@@ -127,3 +127,155 @@ source-files:
   - name: logs/syslog.txt
     tags:
       - syslog
+# Grenade new logs below
+  - name: logs/new/screen-c-api.txt
+    tags:
+      - screen
+      - oslofmt
+    job-filter: '.*grenade.*'
+  - name: logs/new/screen-c-sch.txt
+    tags:
+      - screen
+      - oslofmt
+    job-filter: '.*grenade.*'
+  - name: logs/new/screen-c-vol.txt
+    tags:
+      - screen
+      - oslofmt
+    job-filter: '.*grenade.*'
+  - name: logs/new/screen-g-api.txt
+    tags:
+      - screen
+      - oslofmt
+    job-filter: '.*grenade.*'
+  - name: logs/new/screen-g-reg.txt
+    tags:
+      - screen
+      - oslofmt
+    job-filter: '.*grenade.*'
+  - name: logs/new/screen-key.txt
+    tags:
+      - screen
+      - keystonefmt
+    job-filter: '.*grenade.*'
+  - name: logs/new/screen-n-api.txt
+    tags:
+      - screen
+      - oslofmt
+    job-filter: '.*grenade.*'
+  - name: logs/new/screen-n-cond.txt
+    tags:
+      - screen
+      - oslofmt
+    job-filter: '.*grenade.*'
+  - name: logs/new/screen-n-cpu.txt
+    tags:
+      - screen
+      - oslofmt
+    job-filter: '.*grenade.*'
+  - name: logs/new/screen-n-crt.txt
+    tags:
+      - screen
+      - oslofmt
+    job-filter: '.*grenade.*'
+  - name: logs/new/screen-n-net.txt
+    tags:
+      - screen
+      - oslofmt
+    job-filter: '.*grenade.*'
+  - name: logs/new/screen-n-obj.txt
+    tags:
+      - screen
+      - oslofmt
+    job-filter: '.*grenade.*'
+  - name: logs/new/screen-s-account.txt
+    tags:
+      - screen
+      - apachecombined
+    job-filter: '.*grenade.*'
+  - name: logs/new/screen-s-container.txt
+    tags:
+      - screen
+      - apachecombined
+    job-filter: '.*grenade.*'
+  - name: logs/new/screen-s-object.txt
+    tags:
+      - screen
+      - apachecombined
+    job-filter: '.*grenade.*'
+# Grenade old logs below
+  - name: logs/old/screen-c-api.txt
+    tags:
+      - screen
+      - oslofmt
+    job-filter: '.*grenade.*'
+  - name: logs/old/screen-c-sch.txt
+    tags:
+      - screen
+      - oslofmt
+    job-filter: '.*grenade.*'
+  - name: logs/old/screen-c-vol.txt
+    tags:
+      - screen
+      - oslofmt
+    job-filter: '.*grenade.*'
+  - name: logs/old/screen-g-api.txt
+    tags:
+      - screen
+      - oslofmt
+    job-filter: '.*grenade.*'
+  - name: logs/old/screen-g-reg.txt
+    tags:
+      - screen
+      - oslofmt
+    job-filter: '.*grenade.*'
+  - name: logs/old/screen-key.txt
+    tags:
+      - screen
+      - keystonefmt
+    job-filter: '.*grenade.*'
+  - name: logs/old/screen-n-api.txt
+    tags:
+      - screen
+      - oslofmt
+    job-filter: '.*grenade.*'
+  - name: logs/old/screen-n-cond.txt
+    tags:
+      - screen
+      - oslofmt
+    job-filter: '.*grenade.*'
+  - name: logs/old/screen-n-cpu.txt
+    tags:
+      - screen
+      - oslofmt
+    job-filter: '.*grenade.*'
+  - name: logs/old/screen-n-crt.txt
+    tags:
+      - screen
+      - oslofmt
+    job-filter: '.*grenade.*'
+  - name: logs/old/screen-n-net.txt
+    tags:
+      - screen
+      - oslofmt
+    job-filter: '.*grenade.*'
+  - name: logs/old/screen-n-obj.txt
+    tags:
+      - screen
+      - oslofmt
+    job-filter: '.*grenade.*'
+  - name: logs/old/screen-s-account.txt
+    tags:
+      - screen
+      - apachecombined
+    job-filter: '.*grenade.*'
+  - name: logs/old/screen-s-container.txt
+    tags:
+      - screen
+      - apachecombined
+    job-filter: '.*grenade.*'
+  - name: logs/old/screen-s-object.txt
+    tags:
+      - screen
+      - apachecombined
+    job-filter: '.*grenade.*'