Fix up defaults to use notabene handler and usage

Also backports some tweaks from ops:
- fixes logging.conf
- fixes pipeline_worker first time start

Change-Id: I6ab1e567594097bc23d857c05d0c1fc84262babf

ansible/roles/stv3-common/defaults/main.yaml

@@ -30,3 +30,4 @@ catch_all_notifications: false
 pipeline_handlers:
     logger: winchester.pipeline_handler:LoggingHandler
     usage: winchester.pipeline_handler:UsageHandler
+    notabene: winchester.pipeline_handler:NotabeneHandler

ansible/roles/stv3-common/files/distiller.conf

@@ -1,12 +1,22 @@
 ---
-- event_type: compute.*
+- event_type: compute.instance.*
   traits: &instance_traits
     tenant_id:
-      fields: payload.tenant_id
+      fields:
+        - payload.tenant_id
+        - _context_project_id
     user_id:
       fields: payload.user_id
+    request_id:
+        fields: _context_request_id
+    message: 
+        fields: payload.message
     instance_id:
-      fields: payload.instance_id
+      fields:
+        - payload.instance_uuid
+        - payload.instance_id
+        - exception.kwargs.uuid
+        - instance.uuid
     host:
       fields: publisher_id
       plugin:
@@ -17,14 +27,50 @@
     service:
       fields: publisher_id
       plugin: split
+    instance_flavor:
+      fields: 
+        - payload.instance_type
+        - payload.image_meta.instance_type_name
+        - payload.image_meta.instance_type_flavorid
+    instance_flavor_id:
+      type: int
+      fields: 
+        - payload.instance_flavor_id
+        - payload.image_meta.instance_type_flavor_id
+        - payload.instance_type_id
+    memory_mb:
+      type: int
+      fields: payload.memory_mb
+    disk_gb:
+      type: int
+      fields: payload.disk_gb
+    root_gb:
+      type: int
+      fields: payload.root_gb
+    ephemeral_gb:
+      type: int
+      fields: payload.ephemeral_gb
+    vcpus:
+      type: int
+      fields: payload.vcpus
+    instance_type:
+      fields: payload.instance_type
     state:
       fields: payload.state
+    os_architecture:
+      fields: payload.image_meta.'org.openstack__1__architecture'
+    os_version:
+      fields: payload.image_meta.'org.openstack__1__os_version'
+    os_distro:
+      fields: payload.image_meta.'org.openstack__1__os_distro'
     launched_at:
       type: datetime
       fields: payload.launched_at
     deleted_at:
       type: datetime
-      fields: payload.deleted_at
+      fields: 
+        - payload.deleted_at
+        - payload.terminated_at
 - event_type: compute.instance.exists
   traits:
     <<: *instance_traits
@@ -34,3 +80,15 @@
     audit_period_ending:
       type: datetime
       fields: payload.audit_period_ending
+- event_type: snapshot_instance
+  traits:
+    <<: *instance_traits
+- event_type: scheduler.run_instance.*
+  traits:
+    <<: *instance_traits
+- event_type: keypair.import.*
+  traits:
+    <<: *instance_traits
+- event_type: rebuild_instance
+  traits:
+    <<: *instance_traits

ansible/roles/stv3-common/files/pipelines.yaml

@@ -2,5 +2,33 @@
 usage_pipeline:
     - logger
     - usage
+    - name: notabene
+      params: 
+        host: localhost
+        user: guest
+        password: guest
+        port: 5672
+        vhost: /
+        library: librabbitmq
+        exchange: nova
+        exchange_type: topic
+        queue_name: monitor.info
+        env_keys:
+            - usage_notifications
+
 usage_expire_pipeline:
     - logger
+    - usage
+    - name: notabene
+      params: 
+        host: localhost
+        user: guest
+        password: guest
+        port: 5672
+        vhost: /
+        library: librabbitmq
+        exchange: nova
+        exchange_type: topic
+        queue_name: monitor.info
+        env_keys:
+            - usage_notifications

ansible/roles/stv3-common/templates/logging.conf.j2

@@ -46,7 +46,7 @@ formatter = yagi
 
 [handler_watchedfile]
 class = handlers.WatchedFileHandler
-args = ()
+args = ('/var/log/stv3/stv3.log',)
 formatter = yagi
 
 [handler_syslog]

ansible/roles/stv3-workers/tasks/main.yaml

@@ -26,13 +26,5 @@
         dest=/etc/init.d/pipeline_worker_{{item}}
         owner=stv3 group=stv3 mode=0755
   with_sequence: count={{num_pipeline_workers}}
-
-- name: yagi-event
-  debug: msg="Starting yagi-event"
-  notify: 
-    - restart yagi-event
-
-- name: pipeline-worker
-  debug: msg="Starting pipeline-worker"
   notify: 
     - restart pipeline-worker

logging.conf

@@ -12,12 +12,12 @@ level = WARNING
 handlers = null
 
 [logger_yagi]
-level = INFO
+level = DEBUG
 handlers = stderr
 qualname = yagi
 
 [logger_winchester]
-level = INFO
+level = DEBUG
 handlers = stderr
 qualname = winchester
 

winchester.yaml

@@ -28,9 +28,7 @@ database:
 
 distiller_config: event_definitions.yaml
 trigger_definitions: triggers.yaml
-#trigger_definitions: usage_triggers.yaml
 pipeline_config: pipelines.yaml
-#pipeline_config: usage_pipelines.yaml
 
 purge_completed_streams: false
 
@@ -42,4 +40,6 @@ purge_completed_streams: false
 
 pipeline_handlers:
     logger: winchester.pipeline_handler:LoggingHandler
-    usage_pipeline: winchester.pipeline_handler:UsageHandler
+    usage: winchester.pipeline_handler:UsageHandler
+    notabene: winchester.pipeline_handler:NotabeneHandler
+

winchester/pipelines.yaml

@@ -1,5 +1,36 @@
 ---
-test_pipeline:
+test_fire_pipeline:
     - logger
+    - usage
+    - name: notabene
+      params: 
+        host: localhost
+        user: guest
+        password: guest
+        port: 5672
+        vhost: /
+        library: librabbitmq
+        exchange: nova
+        exchange_type: topic
+        queue_name: monitor.info
+        env_keys:
+            - usage_notifications
+
 test_expire_pipeline:
     - logger
+    - usage
+    - name: notabene
+      params: 
+        host: localhost
+        user: guest
+        password: guest
+        port: 5672
+        vhost: /
+        library: librabbitmq
+        exchange: nova
+        exchange_type: topic
+        queue_name: monitor.info
+        env_keys:
+            - usage_notifications
+
+

winchester/triggers.yaml

@@ -4,22 +4,22 @@
   distinguished_by:
   - instance_id
   - timestamp: "day"
-  expiration: "$last + 1h"
+  expiration: "$last + 2d"
-  fire_pipeline: "test_pipeline"
+  fire_pipeline: "test_fire_pipeline"
   expire_pipeline: "test_expire_pipeline"
   match_criteria:
   - event_type:
     - compute.instance.*
-    - scheduler.run_instance.*
     - snapshot_instance
+    - keypair.import.*
+    - rebuild_instance
+    - compute.instance.*
     - "!compute.instance.exists"
-#### Traits are optional. 
-#    traits:
-#      os_distro: ubuntu
-#      memory_mb:
-#        numeric: "> 4096"
   - event_type: compute.instance.exists
-    map_distingushed_trait:
+    map_distingushed_by:
       timestamp: audit_period_beginning
   fire_criteria:
   - event_type: compute.instance.exists
+    traits:
+      audit_period_ending:
+        datetime: $audit_period_beginning + 1d

winchester/usage_pipelines.yaml

@@ -1,5 +0,0 @@
----
-usage_fire_pipeline:
-    - usage_pipeline
-usage_expire_pipeline:
-    - usage_pipeline

winchester/usage_triggers.yaml

@@ -1,24 +0,0 @@
-- name: usage_trigger
-  debug_level: 2
-  distinguished_by:
-  - instance_id
-  - timestamp: "day"
-  expiration: "$last + 2d"
-  fire_pipeline: "usage_fire_pipeline"
-  expire_pipeline: "usage_expire_pipeline"
-  match_criteria:
-  - event_type:
-    - compute.instance.*
-    - snapshot_instance
-    - keypair.import.*
-    - rebuild_instance
-    - compute.instance.*
-    - "!compute.instance.exists"
-  - event_type: compute.instance.exists
-    map_distingushed_by:
-      timestamp: audit_period_beginning
-  fire_criteria:
-  - event_type: compute.instance.exists
-    traits:
-      audit_period_ending:
-        datetime: $audit_period_beginning + 1d

yagi.conf.common

@@ -17,9 +17,9 @@ vhost = /
 poll_delay = 0.5
 
 [logging]
-logfile = yagi.log
+#logfile = yagi.log
+#default_level = DEBUG
 config_file = logging.conf
-default_level = INFO
 
 [consumers]
 queues = monitor.info