Updates yagi.conf to write json files of every event.

For uploading raw notifications to archive (like an HDFS store).

Also, removes msgpack-python (installed by oslo), which currently
conflict with carrot. This is a temp fix.

Change-Id: I973949c83c9018235169646ce0370d71a2260241

ansible/roles/stv3-workers/templates/yagi.conf.j2

@@ -26,8 +26,7 @@ default_level = {{ yagi_log_level }}
 queues = monitor.info3,monitor.error3
 
 [consumer:monitor.info3]
-#apps = yagi.handler.shoebox_handler.ShoeboxHandler
+apps = winchester.yagi_handler.WinchesterHandler, yagi.handler.shoebox_handler.ShoeboxHandler
-apps = winchester.yagi_handler.WinchesterHandler
 exchange = nova
 exchange_type = topic
 routing_key = monitor.info
@@ -35,7 +34,7 @@ durable = True
 max_messages = 100
 
 [consumer:monitor.error3]
-apps = winchester.yagi_handler.WinchesterHandler
+apps = winchester.yagi_handler.WinchesterHandler, yagi.handler.shoebox_handler.ShoeboxHandler
 exchange = nova
 exchange_type = topic
 routing_key = monitor.error
@@ -55,29 +54,6 @@ config_class = .|oahu_config:Config
 config_file = /etc/stv3/winchester.yaml
 
 [shoebox]
-# Store in-process files in ./working
+working_directory=/etc/stv3/events
-# Move them to ./archive when full via the MoveFileCallback
+filename_template=%Y%m%d-[[TIMESTAMP]]-stv3-[[CRC]].json.gz
-# Roll files every 1mb
+roll_manager=shoebox.roll_manager:WritingJSONRollManager
-
-working_directory=data/working
-destination_folder=data/archive
-
-filename_template=events_%Y_%m_%d_%X_%f.dat
-roll_checker=shoebox.roll_checker:SizeRollChecker
-roll_size_mb=1
-
-distiller_conf=distiller.conf
-
-# Swift upload support
-# create a credentials file (see shoebox/bin/sample_credentials.conf)
-callback=shoebox.handlers:CallbackList
-callback_list=shoebox.handlers:MoveFileCallback, shoebox.handlers:SwiftUploadCallback, shoebox.handlers:DeleteFileCallback
-container=sandbox
-credentials_file=swift_credentials.conf
-auth_method=rackspace
-region={{ item.swift_region }}
-
-# If you don't want Swift support, comment the above callback=
-# entry and uncomment this one:
-#callback=shoebox.handlers:MoveFileCallback
-# which will just move the file into the archive directory.

build.sh

@@ -99,6 +99,9 @@ do
     cd ../..
 done
 
+# Hack(sandy): remove msgpack that conflicts with carrot
+pip uninstall -y msgpack-python
+
 (cat yagi.conf.$PIPELINE_ENGINE ; cat yagi.conf.common ) > yagi.conf
 
 if [ $PIPELINE_ENGINE == "winchester" ]

yagi.conf.common

@@ -36,29 +36,6 @@ config_class = .|oahu_config:Config
 config_file = winchester.yaml
 
 [shoebox]
-# Store in-process files in ./working
+working_directory=data/events
-# Move them to ./archive when full via the MoveFileCallback
+filename_template=%Y%m%d-[[TIMESTAMP]]-stv3-[[CRC]].json.gz
-# Roll files every 1mb
+roll_manager=shoebox.roll_manager:WritingJSONRollManager
-
-working_directory=data/working
-destination_folder=data/archive
-
-filename_template=events_%Y_%m_%d_%X_%f.dat
-roll_checker=shoebox.roll_checker:SizeRollChecker
-roll_size_mb=1
-
-distiller_conf=distiller.conf
-
-# Swift upload support
-# create a credentials file (see shoebox/bin/sample_credentials.conf)
-callback=shoebox.handlers:CallbackList
-callback_list=shoebox.handlers:MoveFileCallback, shoebox.handlers:SwiftUploadCallback, shoebox.handlers:DeleteFileCallback
-container=sandbox
-credentials_file=swift_credentials.conf
-auth_method=rackspace
-region=DFW
-
-# If you don't want Swift support, comment the above callback=
-# entry and uncomment this one:
-#callback=shoebox.handlers:MoveFileCallback
-# which will just move the file into the archive directory.

yagi.conf.winchester

@@ -1,10 +1,7 @@
 [consumer:monitor.info]
-#apps = yagi.handler.shoebox_handler.ShoeboxHandler
+apps = winchester.yagi_handler.WinchesterHandler, yagi.handler.shoebox_handler.ShoeboxHandler
-apps = winchester.yagi_handler.WinchesterHandler
 exchange = monitor
 exchange_type = topic
 routing_key = monitor.info
 durable = True
 max_messages = 100
-
-