pipeline configuration is now done via simport and a shared oahu.config:Config derivation

oahu_config.py

@@ -0,0 +1,28 @@
+import oahu.config
+from oahu import mongodb_sync_engine as driver
+from oahu import pipeline
+from oahu import stream_rules
+from oahu import trigger_callback
+from oahu import trigger_rule
+
+
+class Callback(object):
+    def on_trigger(self, stream):
+        print "Processing", stream
+
+
+class Config(oahu.config.Config):
+    def get_sync_engine(self, callback=None):
+        inactive = trigger_rule.Inactive(60)
+        rule_id = "request-id"  # Has to be consistent across yagi workers.
+        if not callback:
+            self.callback = Callback()
+        else:
+            self.callback = callback
+        by_request = stream_rules.StreamRule(rule_id,
+                                             ["request_id", ],
+                                             inactive, self.callback)
+        rules = [by_request, ]
+
+        return driver.MongoDBSyncEngine(rules)
+

screenrc

@@ -1,7 +1,7 @@
 sessionname tach
 hardstatus alwayslastline '%{= .} %-Lw%{= .}%> %n%f %t*%{= .}%+Lw%< %-=%{g}(%{d}%H/%l%{g})'
-screen -t gen bash 
+#screen -t gen bash 
-stuff "cd git/notigen/bin; python event_pump.py\r"
+#stuff "cd git/notigen/bin; python event_pump.py\r"
 screen -t yagi1 bash
 stuff "cd git/yagi/bin; ./yagi-event --config ../../../shoebox.conf\r"
 screen -t yagi2 bash
@@ -9,12 +9,12 @@ stuff "cd git/yagi/bin; ./yagi-event --config ../../../shoebox.conf\r"
 screen -t quincy bash
 stuff "cd git/quincy/quincy; gunicorn --log-file=- api:api\r"
 screen -t expired bash
-stuff "pipeline expired --polling_rate=2\r"
+stuff "pipeline expired \".|oahu_config:Config\" --polling_rate=2\r"
 screen -t ready1 bash
-stuff "pipeline ready --polling_rate=1\r"
+stuff "pipeline ready \".|oahu_config:Config\" --polling_rate=1\r"
 screen -t ready2 bash
-stuff "pipeline ready --polling_rate=1\r"
+stuff "pipeline ready \".|oahu_config:Config\" --polling_rate=1\r"
 screen -t completed bash
-stuff "pipeline completed --polling_rate=2\r"
+stuff "pipeline completed \".|oahu_config:Config\" --polling_rate=2\r"
 screen -t bash bash
 stuff "klugman events -h\r"

shoebox.conf

@@ -31,16 +31,19 @@ routing_key = monitor.info
 durable = True
 max_messages = 100
 
+# ------ NOTE ------
+# Yagi is launched from sandbox/git/yagi/bin, so files are
+# relative to there.
+# ------------------
+
 [oahu]
-# nothing yet
+config_class = ../../..|oahu_config:Config
 
 [shoebox]
 # Store in-process files in ./working
 # Move them to ./archive when full via the MoveFileCallback
 # Roll files every 1mb
 
-# Yagi is launched from sandbox/git/yagi/bin, so files are
-# relative to there.
 working_directory=../../../data/working
 destination_folder=../../../data/archive