From 4cb647a7f3cdf6e030babbf190d4b34e91be6691 Mon Sep 17 00:00:00 2001
From: Sandy Walsh <sandy.walsh@rackspace.com>
Date: Tue, 17 Feb 2015 14:50:28 -0800
Subject: [PATCH] Updates yagi.conf to write json files of every event.

For uploading raw notifications to archive (like an HDFS store).

Also, removes msgpack-python (installed by oslo), which currently
conflict with carrot. This is a temp fix.

Change-Id: I973949c83c9018235169646ce0370d71a2260241
---
 .../roles/stv3-workers/templates/yagi.conf.j2 | 34 +++----------------
 build.sh                                      |  3 ++
 yagi.conf.common                              | 29 ++--------------
 yagi.conf.winchester                          |  5 +--
 4 files changed, 12 insertions(+), 59 deletions(-)

diff --git a/ansible/roles/stv3-workers/templates/yagi.conf.j2 b/ansible/roles/stv3-workers/templates/yagi.conf.j2
index 41faf11..03f9228 100644
--- a/ansible/roles/stv3-workers/templates/yagi.conf.j2
+++ b/ansible/roles/stv3-workers/templates/yagi.conf.j2
@@ -26,8 +26,7 @@ default_level = {{ yagi_log_level }}
 queues = monitor.info3,monitor.error3
 
 [consumer:monitor.info3]
-#apps = yagi.handler.shoebox_handler.ShoeboxHandler
-apps = winchester.yagi_handler.WinchesterHandler
+apps = winchester.yagi_handler.WinchesterHandler, yagi.handler.shoebox_handler.ShoeboxHandler
 exchange = nova
 exchange_type = topic
 routing_key = monitor.info
@@ -35,7 +34,7 @@ durable = True
 max_messages = 100
 
 [consumer:monitor.error3]
-apps = winchester.yagi_handler.WinchesterHandler
+apps = winchester.yagi_handler.WinchesterHandler, yagi.handler.shoebox_handler.ShoeboxHandler
 exchange = nova
 exchange_type = topic
 routing_key = monitor.error
@@ -55,29 +54,6 @@ config_class = .|oahu_config:Config
 config_file = /etc/stv3/winchester.yaml
 
 [shoebox]
-# Store in-process files in ./working
-# Move them to ./archive when full via the MoveFileCallback
-# Roll files every 1mb
-
-working_directory=data/working
-destination_folder=data/archive
-
-filename_template=events_%Y_%m_%d_%X_%f.dat
-roll_checker=shoebox.roll_checker:SizeRollChecker
-roll_size_mb=1
-
-distiller_conf=distiller.conf
-
-# Swift upload support
-# create a credentials file (see shoebox/bin/sample_credentials.conf)
-callback=shoebox.handlers:CallbackList
-callback_list=shoebox.handlers:MoveFileCallback, shoebox.handlers:SwiftUploadCallback, shoebox.handlers:DeleteFileCallback
-container=sandbox
-credentials_file=swift_credentials.conf
-auth_method=rackspace
-region={{ item.swift_region }}
-
-# If you don't want Swift support, comment the above callback=
-# entry and uncomment this one:
-#callback=shoebox.handlers:MoveFileCallback
-# which will just move the file into the archive directory.
+working_directory=/etc/stv3/events
+filename_template=%Y%m%d-[[TIMESTAMP]]-stv3-[[CRC]].json.gz
+roll_manager=shoebox.roll_manager:WritingJSONRollManager
diff --git a/build.sh b/build.sh
index 2d74a19..5dbccb6 100755
--- a/build.sh
+++ b/build.sh
@@ -99,6 +99,9 @@ do
     cd ../..
 done
 
+# Hack(sandy): remove msgpack that conflicts with carrot
+pip uninstall -y msgpack-python
+
 (cat yagi.conf.$PIPELINE_ENGINE ; cat yagi.conf.common ) > yagi.conf
 
 if [ $PIPELINE_ENGINE == "winchester" ]
diff --git a/yagi.conf.common b/yagi.conf.common
index f0c4500..6a32510 100644
--- a/yagi.conf.common
+++ b/yagi.conf.common
@@ -36,29 +36,6 @@ config_class = .|oahu_config:Config
 config_file = winchester.yaml
 
 [shoebox]
-# Store in-process files in ./working
-# Move them to ./archive when full via the MoveFileCallback
-# Roll files every 1mb
-
-working_directory=data/working
-destination_folder=data/archive
-
-filename_template=events_%Y_%m_%d_%X_%f.dat
-roll_checker=shoebox.roll_checker:SizeRollChecker
-roll_size_mb=1
-
-distiller_conf=distiller.conf
-
-# Swift upload support
-# create a credentials file (see shoebox/bin/sample_credentials.conf)
-callback=shoebox.handlers:CallbackList
-callback_list=shoebox.handlers:MoveFileCallback, shoebox.handlers:SwiftUploadCallback, shoebox.handlers:DeleteFileCallback
-container=sandbox
-credentials_file=swift_credentials.conf
-auth_method=rackspace
-region=DFW
-
-# If you don't want Swift support, comment the above callback=
-# entry and uncomment this one:
-#callback=shoebox.handlers:MoveFileCallback
-# which will just move the file into the archive directory.
+working_directory=data/events
+filename_template=%Y%m%d-[[TIMESTAMP]]-stv3-[[CRC]].json.gz
+roll_manager=shoebox.roll_manager:WritingJSONRollManager
diff --git a/yagi.conf.winchester b/yagi.conf.winchester
index 732160b..4586e6f 100644
--- a/yagi.conf.winchester
+++ b/yagi.conf.winchester
@@ -1,10 +1,7 @@
 [consumer:monitor.info]
-#apps = yagi.handler.shoebox_handler.ShoeboxHandler
-apps = winchester.yagi_handler.WinchesterHandler
+apps = winchester.yagi_handler.WinchesterHandler, yagi.handler.shoebox_handler.ShoeboxHandler
 exchange = monitor
 exchange_type = topic
 routing_key = monitor.info
 durable = True
 max_messages = 100
-
-