Add winchester support

README.md

@@ -6,39 +6,47 @@ Dev env for StackTach development. Uses Notigen event generator and yagi.
 Prerequisites:
 
     make sure you have a working python dev environment (2.7+ ideally)
-    including virtualenv. Install rabbitmq and mongodb
+        including virtualenv.
+    Install rabbitmq.
+    If using the oahu pipeline engine, install mongodb.
+    If using the winchester pipeline engine, install MySQL.
 
 TL;DR:
 
     handle the prerequisites above.
     git clone https://github.com/StackTach/sandbox.git
     cd sandbox
+    If running winchester:
+        create a mysql database to use
+        set the database url appropriately in winchester.yaml
     ./build.sh
 
-Tweaks:    
-    
+Tweaks:
+
 You can create a `local.sh` to override the defaults:
 
     SOURCE_DIR=git  # where the StackTach repos are cloned
     VENV_DIR=.venv  # name of the .venv
+    PIPELINE_ENGINE=oahu # Name of pipeline processing library to run.
 
 The `build.sh` script will create clone each of the StackTach projects
-into the `$SOURCE_DIR` directory (so you can work on them in a running env). 
+into the `$SOURCE_DIR` directory (so you can work on them in a running env).
 
-The virtualenv will be created and each of the projects 
-(and their dependencies) installed into it. 
+The virtualenv will be created and each of the projects
+(and their dependencies) installed into it.
 
-A `screen` session is started, based on `screenrc` which will start the 
+A `screen` session is started, based on `screenrc.oahu` or
+`screenrc.winchester`  which will start the
 `notigen` event generator. The event generator simulated OpenStack
 notifications and pumps them into rabbitmq. `yagi-event` is also started
-with the `shoebox.conf` configuration file. This will read events from
+with the `yagi.conf` configuration file. This will read events from
 the rabbit queue and save them to local files. The working directory
-and archive directory for `shoebox` is specified in `shoebox.conf`.
+and archive directory for `shoebox` is specified in `yagi.conf`.
 
 The sandbox environment configures `shoebox` to upload archive files
 to Swift automatically. This requires you create a credentials file
-in the `.../sandbox/` directory (like in 
+in the `.../sandbox/` directory (like in
 `.../git/sandbox/etc/sample_rax_credentials.conf`) Call it
 `swift_credentials.conf` or alter the `shoebox.conf` file accordingly. If
 you don't have access to a Swift server, like CloudFiles, read
-the config file for details on disabling this feature. 
+the config file for details on disabling this feature.

build.sh

@@ -4,6 +4,7 @@ echo "StackTach dev env build script"
 
 SOURCE_DIR=git
 VENV_DIR=.venv
+PIPELINE_ENGINE=oahu
 
 if [[ -f local.sh ]]; then
     source local.sh
@@ -21,7 +22,7 @@ cd $SOURCE_DIR
 for file in StackTach/shoebox StackTach/simport StackTach/notigen \
             StackTach/notabene StackTach/notification_utils rackerlabs/yagi \
             StackTach/stackdistiller StackTach/quincy StackTach/quince \
-            StackTach/klugman StackTach/oahu
+            StackTach/klugman StackTach/oahu StackTach/winchester
 do
     git clone https://github.com/$file
 done
@@ -46,4 +47,12 @@ do
     cd ../..
 done
 
-screen -c screenrc
+(cat yagi.conf.$PIPELINE_ENGINE ; cat yagi.conf.common ) > yagi.conf
+
+if [ $PIPELINE_ENGINE == "winchester" ]
+then
+    winchester_db -c winchester.yaml upgrade head
+fi
+
+screen -c screenrc.$PIPELINE_ENGINE
+

logging.conf

@@ -0,0 +1,74 @@
+[loggers]
+keys = root, yagi, winchester, oahu
+
+[handlers]
+keys = stderr, stdout, watchedfile, syslog, null
+
+[formatters]
+keys = yagi, default
+
+[logger_root]
+level = WARNING
+handlers = null
+
+[logger_yagi]
+level = INFO
+handlers = stderr
+qualname = yagi
+
+[logger_winchester]
+level = DEBUG
+handlers = stderr
+qualname = winchester
+
+[logger_oahu]
+level = DEBUG
+handlers = stderr
+qualname = oahu
+
+[logger_amqplib]
+level = WARNING
+handlers = stderr
+qualname = amqplib
+
+[logger_sqlalchemy]
+# yagi does not use sqlalchemy... yet. -mdragon
+level = WARNING
+handlers = stderr
+qualname = sqlalchemy
+# "level = INFO" logs SQL queries.
+# "level = DEBUG" logs SQL queries and results.
+# "level = WARNING" logs neither.  (Recommended for production systems.)
+
+[handler_stderr]
+class = StreamHandler
+args = (sys.stderr,)
+formatter = yagi
+
+[handler_stdout]
+class = StreamHandler
+args = (sys.stdout,)
+formatter = yagi
+
+[handler_watchedfile]
+class = handlers.WatchedFileHandler
+args = ('sandbox.log',)
+formatter = yagi
+
+[handler_syslog]
+class = handlers.SysLogHandler
+args = ('/dev/log', handlers.SysLogHandler.LOG_USER)
+formatter = yagi
+
+[handler_null]
+class = NullHandler
+formatter = default
+args = ()
+
+[formatter_yagi]
+# substitutions available for formats are documented at:
+# https://docs.python.org/2/library/logging.html#logrecord-attributes
+format = %(name)s[%(levelname)s at %(asctime)s line: %(lineno)d] %(message)s
+
+[formatter_default]
+format = %(message)s

screenrc.oahu

@@ -6,9 +6,9 @@ stuff "cd git/quincy/quincy; gunicorn --log-file=- 'api:get_api(config_location=
 screen -t bash bash
 stuff "klugman streams\r"
 screen -t yagi1 bash
-stuff "cd git/yagi/bin; ./yagi-event --config ../../../shoebox.conf\r"
+stuff "yagi-event --config yagi.conf\r"
 screen -t yagi2 bash
-stuff "cd git/yagi/bin; ./yagi-event --config ../../../shoebox.conf\r"
+stuff "yagi-event --config yagi.conf\r"
 screen -t trigger bash
 stuff "pipeline trigger \".|oahu_config:Config\" --polling_rate=20\r"
 screen -t ready1 bash
@@ -17,6 +17,6 @@ screen -t ready2 bash
 stuff "pipeline ready \".|oahu_config:Config\" --polling_rate=20\r"
 screen -t completed bash
 stuff "pipeline completed \".|oahu_config:Config\" --polling_rate=20\r"
-screen -t gen bash 
+screen -t gen bash
 stuff "cd git/notigen/bin; python event_pump.py ../templates 2 0\r"
 

screenrc.winchester

@@ -0,0 +1,18 @@
+sessionname tach
+hardstatus alwayslastline '%{= .} %-Lw%{= .}%> %n%f %t*%{= .}%+Lw%< %-=%{g}(%{d}%H/%l%{g})'
+# not yet working w/ winchester
+#screen -t quincy bash
+#stuff "cd git/quincy/quincy; gunicorn --log-file=- 'api:get_api(config_location=\"../../../quincy.conf\")'\r"
+screen -t bash bash
+#stuff "klugman streams\r"
+screen -t yagi1 bash
+stuff "yagi-event --config yagi.conf\r"
+screen -t yagi2 bash
+stuff "yagi-event --config yagi.conf\r"
+screen -t pipeline1 bash
+stuff "pipeline_worker -c winchester.yaml\r"
+screen -t pipeline2 bash
+stuff "pipeline_worker -c winchester.yaml\r"
+screen -t gen bash
+stuff "cd git/notigen/bin; python event_pump.py ../templates 2 0\r"
+

winchester.yaml

@@ -0,0 +1,34 @@
+---
+###### This adds directories to the search path for other configfiles.
+config_path: winchester
+## It can also be a list:
+#config_path:
+#  - /etc/winchester
+#  - /home/wherever
+
+###### logging
+#log_level: debug
+## You can also use a full logging config file.
+logging_config: logging.conf
+
+###### How often to log stats
+statistics_period: 10
+
+pipeline_worker_batch_size: 1000
+pipeline_worker_delay: 10
+
+####### You can specify extra stackdistiller trait plugins here:
+#distiller_trait_plugins:
+#    test: some.module.path:TestTraitPlugin
+
+catch_all_notifications: false
+
+database:
+    url: mysql://winchester:testpasswd@localhost/winchester
+
+distiller_config: event_definitions.yaml
+trigger_definitions: triggers.yaml
+pipeline_config: pipelines.yaml
+
+pipeline_handlers:
+    logger: winchester.pipeline_handler:LoggingHandler

winchester/event_definitions.yaml

@@ -0,0 +1,63 @@
+---
+- event_type: compute.instance.*
+  traits: &instance_traits
+    tenant_id:
+      fields: payload.tenant_id
+    user_id:
+      fields: payload.user_id
+    instance_id:
+      fields: payload.instance_id
+    host:
+      fields: publisher_id
+      plugin:
+        name: split
+        parameters:
+          segment: 1
+          max_split: 1
+    service:
+      fields: publisher_id
+      plugin: split
+    memory_mb:
+      type: int
+      fields: payload.memory_mb
+    disk_gb:
+      type: int
+      fields: payload.disk_gb
+    root_gb:
+      type: int
+      fields: payload.root_gb
+    ephemeral_gb:
+      type: int
+      fields: payload.ephemeral_gb
+    vcpus:
+      type: int
+      fields: payload.vcpus
+    instance_type_id:
+      type: int
+      fields: payload.instance_type_id
+    instance_type:
+      fields: payload.instance_type
+    state:
+      fields: payload.state
+    os_architecture:
+      fields: payload.image_meta.'org.openstack__1__architecture'
+    os_version:
+      fields: payload.image_meta.'org.openstack__1__os_version'
+    os_distro:
+      fields: payload.image_meta.'org.openstack__1__os_distro'
+    launched_at:
+      type: datetime
+      fields: payload.launched_at
+    deleted_at:
+      type: datetime
+      fields: payload.deleted_at
+- event_type: compute.instance.exists
+  traits:
+    <<: *instance_traits
+    audit_period_beginning:
+      type: datetime
+      fields: payload.audit_period_beginning
+    audit_period_ending:
+      type: datetime
+      fields: payload.audit_period_ending
+

winchester/pipelines.yaml

@@ -0,0 +1,5 @@
+---
+test_pipeline:
+    - logger
+test_expire_pipeline:
+    - logger

winchester/triggers.yaml

@@ -0,0 +1,22 @@
+---
+- name: test_trigger
+  distinguished_by:
+  - instance_id
+  - timestamp: "day"
+  expiration: "$last + 1h"
+  fire_pipeline: "test_pipeline"
+  expire_pipeline: "test_expire_pipeline"
+  match_criteria:
+  - event_type:
+    - compute.instance.*
+    - "!compute.instance.exists"
+#### Traits are optional. 
+#    traits:
+#      os_distro: ubuntu
+#      memory_mb:
+#        numeric: "> 4096"
+  - event_type: compute.instance.exists
+    map_distingushed_trait:
+      timestamp: audit_period_beginning
+  fire_criteria:
+  - event_type: compute.instance.exists

yagi.conf.common

@@ -18,52 +18,47 @@ poll_delay = 0.5
 
 [logging]
 logfile = yagi.log
+config_file = logging.conf
 default_level = INFO
 
 [consumers]
 queues = monitor.info
 
-[consumer:monitor.info]
-#apps = yagi.handler.shoebox_handler.ShoeboxHandler
-apps = oahu.yagi_handler.OahuHandler
-exchange = monitor
-exchange_type = topic
-routing_key = monitor.info
-durable = True
-max_messages = 100
-
 # ------ NOTE ------
-# Yagi is launched from sandbox/git/yagi/bin, so files are
+# Yagi is launched from the sandbox/ dir, so files are
 # relative to there.
 # ------------------
 
 [oahu]
-config_class = ../../..|oahu_config:Config
+config_class = .|oahu_config:Config
+
+[winchester]
+config_file = winchester.yaml
 
 [shoebox]
 # Store in-process files in ./working
 # Move them to ./archive when full via the MoveFileCallback
 # Roll files every 1mb
 
-working_directory=../../../data/working
-destination_folder=../../../data/archive
+working_directory=data/working
+destination_folder=data/archive
 
 filename_template=events_%Y_%m_%d_%X_%f.dat
 roll_checker=shoebox.roll_checker:SizeRollChecker
 roll_size_mb=1
 
-distiller_conf=../../../distiller.conf
+distiller_conf=distiller.conf
 
 # Swift upload support
 # create a credentials file (see shoebox/bin/sample_credentials.conf)
 callback=shoebox.handlers:CallbackList
 callback_list=shoebox.handlers:MoveFileCallback, shoebox.handlers:SwiftUploadCallback, shoebox.handlers:DeleteFileCallback
 container=sandbox
-credentials_file=../../../swift_credentials.conf
+credentials_file=swift_credentials.conf
 auth_method=rackspace
 region=DFW
 
-# If you don't want Swift support, comment the above callback= 
+# If you don't want Swift support, comment the above callback=
 # entry and uncomment this one:
 #callback=shoebox.handlers:MoveFileCallback
 # which will just move the file into the archive directory.

yagi.conf.oahu

@@ -0,0 +1,10 @@
+[consumer:monitor.info]
+#apps = yagi.handler.shoebox_handler.ShoeboxHandler
+apps = oahu.yagi_handler.OahuHandler
+exchange = monitor
+exchange_type = topic
+routing_key = monitor.info
+durable = True
+max_messages = 100
+
+

yagi.conf.winchester

@@ -0,0 +1,10 @@
+[consumer:monitor.info]
+#apps = yagi.handler.shoebox_handler.ShoeboxHandler
+apps = winchester.yagi_handler.WinchesterHandler
+exchange = monitor
+exchange_type = topic
+routing_key = monitor.info
+durable = True
+max_messages = 100
+
+