diff --git a/ansible/roles/stv3-workers/templates/yagi.conf.j2 b/ansible/roles/stv3-workers/templates/yagi.conf.j2 index 41faf11..03f9228 100644 --- a/ansible/roles/stv3-workers/templates/yagi.conf.j2 +++ b/ansible/roles/stv3-workers/templates/yagi.conf.j2 @@ -26,8 +26,7 @@ default_level = {{ yagi_log_level }} queues = monitor.info3,monitor.error3 [consumer:monitor.info3] -#apps = yagi.handler.shoebox_handler.ShoeboxHandler -apps = winchester.yagi_handler.WinchesterHandler +apps = winchester.yagi_handler.WinchesterHandler, yagi.handler.shoebox_handler.ShoeboxHandler exchange = nova exchange_type = topic routing_key = monitor.info @@ -35,7 +34,7 @@ durable = True max_messages = 100 [consumer:monitor.error3] -apps = winchester.yagi_handler.WinchesterHandler +apps = winchester.yagi_handler.WinchesterHandler, yagi.handler.shoebox_handler.ShoeboxHandler exchange = nova exchange_type = topic routing_key = monitor.error @@ -55,29 +54,6 @@ config_class = .|oahu_config:Config config_file = /etc/stv3/winchester.yaml [shoebox] -# Store in-process files in ./working -# Move them to ./archive when full via the MoveFileCallback -# Roll files every 1mb - -working_directory=data/working -destination_folder=data/archive - -filename_template=events_%Y_%m_%d_%X_%f.dat -roll_checker=shoebox.roll_checker:SizeRollChecker -roll_size_mb=1 - -distiller_conf=distiller.conf - -# Swift upload support -# create a credentials file (see shoebox/bin/sample_credentials.conf) -callback=shoebox.handlers:CallbackList -callback_list=shoebox.handlers:MoveFileCallback, shoebox.handlers:SwiftUploadCallback, shoebox.handlers:DeleteFileCallback -container=sandbox -credentials_file=swift_credentials.conf -auth_method=rackspace -region={{ item.swift_region }} - -# If you don't want Swift support, comment the above callback= -# entry and uncomment this one: -#callback=shoebox.handlers:MoveFileCallback -# which will just move the file into the archive directory. +working_directory=/etc/stv3/events +filename_template=%Y%m%d-[[TIMESTAMP]]-stv3-[[CRC]].json.gz +roll_manager=shoebox.roll_manager:WritingJSONRollManager diff --git a/build.sh b/build.sh index 2d74a19..5dbccb6 100755 --- a/build.sh +++ b/build.sh @@ -99,6 +99,9 @@ do cd ../.. done +# Hack(sandy): remove msgpack that conflicts with carrot +pip uninstall -y msgpack-python + (cat yagi.conf.$PIPELINE_ENGINE ; cat yagi.conf.common ) > yagi.conf if [ $PIPELINE_ENGINE == "winchester" ] diff --git a/yagi.conf.common b/yagi.conf.common index f0c4500..6a32510 100644 --- a/yagi.conf.common +++ b/yagi.conf.common @@ -36,29 +36,6 @@ config_class = .|oahu_config:Config config_file = winchester.yaml [shoebox] -# Store in-process files in ./working -# Move them to ./archive when full via the MoveFileCallback -# Roll files every 1mb - -working_directory=data/working -destination_folder=data/archive - -filename_template=events_%Y_%m_%d_%X_%f.dat -roll_checker=shoebox.roll_checker:SizeRollChecker -roll_size_mb=1 - -distiller_conf=distiller.conf - -# Swift upload support -# create a credentials file (see shoebox/bin/sample_credentials.conf) -callback=shoebox.handlers:CallbackList -callback_list=shoebox.handlers:MoveFileCallback, shoebox.handlers:SwiftUploadCallback, shoebox.handlers:DeleteFileCallback -container=sandbox -credentials_file=swift_credentials.conf -auth_method=rackspace -region=DFW - -# If you don't want Swift support, comment the above callback= -# entry and uncomment this one: -#callback=shoebox.handlers:MoveFileCallback -# which will just move the file into the archive directory. +working_directory=data/events +filename_template=%Y%m%d-[[TIMESTAMP]]-stv3-[[CRC]].json.gz +roll_manager=shoebox.roll_manager:WritingJSONRollManager diff --git a/yagi.conf.winchester b/yagi.conf.winchester index 732160b..4586e6f 100644 --- a/yagi.conf.winchester +++ b/yagi.conf.winchester @@ -1,10 +1,7 @@ [consumer:monitor.info] -#apps = yagi.handler.shoebox_handler.ShoeboxHandler -apps = winchester.yagi_handler.WinchesterHandler +apps = winchester.yagi_handler.WinchesterHandler, yagi.handler.shoebox_handler.ShoeboxHandler exchange = monitor exchange_type = topic routing_key = monitor.info durable = True max_messages = 100 - -