x/stacktach-sandbox
Code Issues Proposed changes

Updates yagi.conf to write json files of every event.

Browse Source 
For uploading raw notifications to archive (like an HDFS store).

Also, removes msgpack-python (installed by oslo), which currently
conflict with carrot. This is a temp fix.

Change-Id: I973949c83c9018235169646ce0370d71a2260241
This commit is contained in:
Sandy Walsh 2015-02-17 14:50:28 -08:00
parent 0833226b93
commit 4cb647a7f3
4 changed files with 12 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
ansible/roles/stv3-workers/templates/yagi.conf.j2
View File

@ -26,8 +26,7 @@ default_level = {{ yagi_log_level }}
queues = monitor.info3,monitor.error3
[consumer:monitor.info3]
#apps = yagi.handler.shoebox_handler.ShoeboxHandler
apps = winchester.yagi_handler.WinchesterHandler
apps = winchester.yagi_handler.WinchesterHandler, yagi.handler.shoebox_handler.ShoeboxHandler
exchange = nova
exchange_type = topic
routing_key = monitor.info
@ -35,7 +34,7 @@ durable = True
max_messages = 100
[consumer:monitor.error3]
apps = winchester.yagi_handler.WinchesterHandler
apps = winchester.yagi_handler.WinchesterHandler, yagi.handler.shoebox_handler.ShoeboxHandler
exchange = nova
exchange_type = topic
routing_key = monitor.error
@ -55,29 +54,6 @@ config_class = .|oahu_config:Config
config_file = /etc/stv3/winchester.yaml
[shoebox]
# Store in-process files in ./working
# Move them to ./archive when full via the MoveFileCallback
# Roll files every 1mb
working_directory=data/working
destination_folder=data/archive
filename_template=events_%Y_%m_%d_%X_%f.dat
roll_checker=shoebox.roll_checker:SizeRollChecker
roll_size_mb=1
distiller_conf=distiller.conf
# Swift upload support
# create a credentials file (see shoebox/bin/sample_credentials.conf)
callback=shoebox.handlers:CallbackList
callback_list=shoebox.handlers:MoveFileCallback, shoebox.handlers:SwiftUploadCallback, shoebox.handlers:DeleteFileCallback
container=sandbox
credentials_file=swift_credentials.conf
auth_method=rackspace
region={{ item.swift_region }}
# If you don't want Swift support, comment the above callback=
# entry and uncomment this one:
#callback=shoebox.handlers:MoveFileCallback
# which will just move the file into the archive directory.
working_directory=/etc/stv3/events
filename_template=%Y%m%d-[[TIMESTAMP]]-stv3-[[CRC]].json.gz
roll_manager=shoebox.roll_manager:WritingJSONRollManager

3
build.sh
View File

@ -99,6 +99,9 @@ do
cd ../..
done
# Hack(sandy): remove msgpack that conflicts with carrot
pip uninstall -y msgpack-python
(cat yagi.conf.$PIPELINE_ENGINE ; cat yagi.conf.common ) > yagi.conf
if [ $PIPELINE_ENGINE == "winchester" ]

29
yagi.conf.common
View File

@ -36,29 +36,6 @@ config_class = .|oahu_config:Config
config_file = winchester.yaml
[shoebox]
# Store in-process files in ./working
# Move them to ./archive when full via the MoveFileCallback
# Roll files every 1mb
working_directory=data/working
destination_folder=data/archive
filename_template=events_%Y_%m_%d_%X_%f.dat
roll_checker=shoebox.roll_checker:SizeRollChecker
roll_size_mb=1
distiller_conf=distiller.conf
# Swift upload support
# create a credentials file (see shoebox/bin/sample_credentials.conf)
callback=shoebox.handlers:CallbackList
callback_list=shoebox.handlers:MoveFileCallback, shoebox.handlers:SwiftUploadCallback, shoebox.handlers:DeleteFileCallback
container=sandbox
credentials_file=swift_credentials.conf
auth_method=rackspace
region=DFW
# If you don't want Swift support, comment the above callback=
# entry and uncomment this one:
#callback=shoebox.handlers:MoveFileCallback
# which will just move the file into the archive directory.
working_directory=data/events
filename_template=%Y%m%d-[[TIMESTAMP]]-stv3-[[CRC]].json.gz
roll_manager=shoebox.roll_manager:WritingJSONRollManager

5
yagi.conf.winchester
View File

@ -1,10 +1,7 @@
[consumer:monitor.info]
#apps = yagi.handler.shoebox_handler.ShoeboxHandler
apps = winchester.yagi_handler.WinchesterHandler
apps = winchester.yagi_handler.WinchesterHandler, yagi.handler.shoebox_handler.ShoeboxHandler
exchange = monitor
exchange_type = topic
routing_key = monitor.info
durable = True
max_messages = 100