Preliminary time-sync and usage support

Optionally enable STv2 event playback through the
time_sync service and early UsageHandler support.

Change-Id: I5988d320fe41df29c52ebdb8e579bf0d7f53a45f

.gitreview

@@ -0,0 +1,5 @@
+[gerrit]
+host=review.openstack.org
+port=29418
+project=stackforge/stacktach-sandbox.git
+

build.sh

@@ -35,6 +35,7 @@ cd ..
 source ./$VENV_DIR/bin/activate
 
 # Some extra required libs ...
+pip install mysql-connector-python --allow-external mysql-connector-python
 pip install gunicorn
 pip install httpie
 pip install librabbitmq

logging.conf

@@ -17,7 +17,7 @@ handlers = stderr
 qualname = yagi
 
 [logger_winchester]
-level = DEBUG
+level = INFO
 handlers = stderr
 qualname = winchester
 
@@ -68,7 +68,8 @@ args = ()
 [formatter_yagi]
 # substitutions available for formats are documented at:
 # https://docs.python.org/2/library/logging.html#logrecord-attributes
-format = %(name)s[%(levelname)s at %(asctime)s line: %(lineno)d] %(message)s
+#format = %(name)s[%(levelname)s at %(asctime)s line: %(lineno)d] %(message)s
+format = %(name)s[%(levelname)s line: %(lineno)d] %(message)s
 
 [formatter_default]
 format = %(message)s

reset.sh

@@ -0,0 +1,17 @@
+echo "This will nuke all StackTach.v3 data in mysql and rabbitmq!"
+read -p "Are you sure? " -n 1 -r
+echo    # (optional) move to a new line
+if [[ $REPLY =~ ^[Yy]$ ]]
+then
+mysql -u root --password=password < reset.sql
+sudo rabbitmqctl stop_app
+sudo rabbitmqctl reset
+sudo rabbitmqctl start_app
+fi
+
+
+
+
+
+
+

reset.sql

@@ -0,0 +1,2 @@
+drop database winchester;
+create database winchester;

screenrc.winchester

@@ -1,8 +1,13 @@
 sessionname tach
 hardstatus alwayslastline '%{= .} %-Lw%{= .}%> %n%f %t*%{= .}%+Lw%< %-=%{g}(%{d}%H/%l%{g})'
+# Enable this if you're replaying old events ...
+#screen -t time_sync bash
+#stuff "cd git/stacktach-notigen/bin; gunicorn -b 127.0.0.1:8001 --log-file=- 'time_sync:get_api()'\r"
+
 # not yet working w/ winchester
 #screen -t quincy bash
 #stuff "cd git/stacktach-quincy/quincy; gunicorn --log-file=- 'api:get_api(config_location=\"../../../quincy.conf\")'\r"
+
 screen -t bash bash
 #stuff "klugman streams\r"
 screen -t yagi1 bash
@@ -14,5 +19,6 @@ stuff "pipeline_worker -c winchester.yaml\r"
 screen -t pipeline2 bash
 stuff "pipeline_worker -c winchester.yaml\r"
 screen -t gen bash
+#stuff "cd git/stacktach-notigen/bin; python pump_from_stv2.py\r"
 stuff "cd git/stacktach-notigen/bin; python event_pump.py ../templates 2 0\r"
 

winchester.yaml

@@ -28,7 +28,16 @@ database:
 
 distiller_config: event_definitions.yaml
 trigger_definitions: triggers.yaml
+#trigger_definitions: usage_triggers.yaml
 pipeline_config: pipelines.yaml
+#pipeline_config: usage_pipelines.yaml
+
+# If you are replaying old events (timestamps in the past)
+# you will need a time service to keep the clocks
+# between yagi and the pipeline workers in sync. 
+# Don't set this if you're running with real-time events.
+#time_sync_endpoint: http://127.0.0.1:8001
 
 pipeline_handlers:
     logger: winchester.pipeline_handler:LoggingHandler
+    usage_pipeline: winchester.pipeline_handler:UsageHandler

winchester/event_definitions.yaml

@@ -1,12 +1,21 @@
----
 - event_type: compute.instance.*
   traits: &instance_traits
     tenant_id:
-      fields: payload.tenant_id
+      fields:
+        - payload.tenant_id
+        - _context_project_id
     user_id:
       fields: payload.user_id
+    request_id:
+        fields: _context_request_id
+    message: 
+        fields: payload.message
     instance_id:
-      fields: payload.instance_id
+      fields:
+        - payload.instance_uuid
+        - payload.instance_id
+        - exception.kwargs.uuid
+        - instance.uuid
     host:
       fields: publisher_id
       plugin:
@@ -17,6 +26,10 @@
     service:
       fields: publisher_id
       plugin: split
+    flavor_id:
+      fields: 
+        - payload.instance_flavor_id 
+        - payload.image_meta.instance_type_flavor_id
     memory_mb:
       type: int
       fields: payload.memory_mb
@@ -50,7 +63,9 @@
       fields: payload.launched_at
     deleted_at:
       type: datetime
-      fields: payload.deleted_at
+      fields: 
+        - payload.deleted_at
+        - payload.terminated_at
 - event_type: compute.instance.exists
   traits:
     <<: *instance_traits
@@ -60,4 +75,15 @@
     audit_period_ending:
       type: datetime
       fields: payload.audit_period_ending
-
+- event_type: snapshot_instance
+  traits:
+    <<: *instance_traits
+- event_type: scheduler.run_instance.*
+  traits:
+    <<: *instance_traits
+- event_type: keypair.import.*
+  traits:
+    <<: *instance_traits
+- event_type: rebuild_instance
+  traits:
+    <<: *instance_traits

winchester/triggers.yaml

@@ -1,4 +1,3 @@
----
 - name: test_trigger
   debug_level: 2
   distinguished_by:
@@ -9,6 +8,8 @@
   expire_pipeline: "test_expire_pipeline"
   match_criteria:
   - event_type:
+    - scheduler.run_instance.*
+    - snapshot_instance
     - compute.instance.*
     - "!compute.instance.exists"
 #### Traits are optional. 

winchester/usage_pipelines.yaml

@@ -0,0 +1,5 @@
+---
+usage_fire_pipeline:
+    - usage_pipeline
+usage_expire_pipeline:
+    - usage_pipeline

winchester/usage_triggers.yaml

@@ -0,0 +1,24 @@
+- name: usage_trigger
+  debug_level: 2
+  distinguished_by:
+  - instance_id
+  - timestamp: "day"
+  expiration: "$last + 2d"
+  fire_pipeline: "usage_fire_pipeline"
+  expire_pipeline: "usage_expire_pipeline"
+  match_criteria:
+  - event_type:
+    - compute.instance.*
+    - snapshot_instance
+    - keypair.import.*
+    - rebuild_instance
+    - compute.instance.*
+    - "!compute.instance.exists"
+  - event_type: compute.instance.exists
+    map_distingushed_by:
+      timestamp: audit_period_beginning
+  fire_criteria:
+  - event_type: compute.instance.exists
+    traits:
+      audit_period_ending:
+        datetime: $audit_period_beginning + 1d