From 5d5f8ca105b1bf685ffff3a8d74a4f4f766c6954 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Tue, 10 Feb 2015 07:54:19 -0800 Subject: [PATCH] Fix up defaults to use notabene handler and usage Also backports some tweaks from ops: - fixes logging.conf - fixes pipeline_worker first time start Change-Id: I6ab1e567594097bc23d857c05d0c1fc84262babf --- ansible/roles/stv3-common/defaults/main.yaml | 1 + .../roles/stv3-common/files/distiller.conf | 66 +++++++++++++++++-- .../roles/stv3-common/files/pipelines.yaml | 28 ++++++++ .../stv3-common/templates/logging.conf.j2 | 2 +- ansible/roles/stv3-workers/tasks/main.yaml | 8 --- logging.conf | 4 +- winchester.yaml | 6 +- winchester/pipelines.yaml | 33 +++++++++- winchester/triggers.yaml | 18 ++--- winchester/usage_pipelines.yaml | 5 -- winchester/usage_triggers.yaml | 24 ------- yagi.conf.common | 4 +- 12 files changed, 140 insertions(+), 59 deletions(-) delete mode 100644 winchester/usage_pipelines.yaml delete mode 100644 winchester/usage_triggers.yaml diff --git a/ansible/roles/stv3-common/defaults/main.yaml b/ansible/roles/stv3-common/defaults/main.yaml index 3eb82e7..9e31ac3 100644 --- a/ansible/roles/stv3-common/defaults/main.yaml +++ b/ansible/roles/stv3-common/defaults/main.yaml @@ -30,3 +30,4 @@ catch_all_notifications: false pipeline_handlers: logger: winchester.pipeline_handler:LoggingHandler usage: winchester.pipeline_handler:UsageHandler + notabene: winchester.pipeline_handler:NotabeneHandler diff --git a/ansible/roles/stv3-common/files/distiller.conf b/ansible/roles/stv3-common/files/distiller.conf index 6bfd3f7..24055e6 100644 --- a/ansible/roles/stv3-common/files/distiller.conf +++ b/ansible/roles/stv3-common/files/distiller.conf @@ -1,12 +1,22 @@ --- -- event_type: compute.* +- event_type: compute.instance.* traits: &instance_traits tenant_id: - fields: payload.tenant_id + fields: + - payload.tenant_id + - _context_project_id user_id: fields: payload.user_id + request_id: + fields: _context_request_id + message: + fields: payload.message instance_id: - fields: payload.instance_id + fields: + - payload.instance_uuid + - payload.instance_id + - exception.kwargs.uuid + - instance.uuid host: fields: publisher_id plugin: @@ -17,14 +27,50 @@ service: fields: publisher_id plugin: split + instance_flavor: + fields: + - payload.instance_type + - payload.image_meta.instance_type_name + - payload.image_meta.instance_type_flavorid + instance_flavor_id: + type: int + fields: + - payload.instance_flavor_id + - payload.image_meta.instance_type_flavor_id + - payload.instance_type_id + memory_mb: + type: int + fields: payload.memory_mb + disk_gb: + type: int + fields: payload.disk_gb + root_gb: + type: int + fields: payload.root_gb + ephemeral_gb: + type: int + fields: payload.ephemeral_gb + vcpus: + type: int + fields: payload.vcpus + instance_type: + fields: payload.instance_type state: fields: payload.state + os_architecture: + fields: payload.image_meta.'org.openstack__1__architecture' + os_version: + fields: payload.image_meta.'org.openstack__1__os_version' + os_distro: + fields: payload.image_meta.'org.openstack__1__os_distro' launched_at: type: datetime fields: payload.launched_at deleted_at: type: datetime - fields: payload.deleted_at + fields: + - payload.deleted_at + - payload.terminated_at - event_type: compute.instance.exists traits: <<: *instance_traits @@ -34,3 +80,15 @@ audit_period_ending: type: datetime fields: payload.audit_period_ending +- event_type: snapshot_instance + traits: + <<: *instance_traits +- event_type: scheduler.run_instance.* + traits: + <<: *instance_traits +- event_type: keypair.import.* + traits: + <<: *instance_traits +- event_type: rebuild_instance + traits: + <<: *instance_traits diff --git a/ansible/roles/stv3-common/files/pipelines.yaml b/ansible/roles/stv3-common/files/pipelines.yaml index de96393..93ec51a 100644 --- a/ansible/roles/stv3-common/files/pipelines.yaml +++ b/ansible/roles/stv3-common/files/pipelines.yaml @@ -2,5 +2,33 @@ usage_pipeline: - logger - usage + - name: notabene + params: + host: localhost + user: guest + password: guest + port: 5672 + vhost: / + library: librabbitmq + exchange: nova + exchange_type: topic + queue_name: monitor.info + env_keys: + - usage_notifications + usage_expire_pipeline: - logger + - usage + - name: notabene + params: + host: localhost + user: guest + password: guest + port: 5672 + vhost: / + library: librabbitmq + exchange: nova + exchange_type: topic + queue_name: monitor.info + env_keys: + - usage_notifications diff --git a/ansible/roles/stv3-common/templates/logging.conf.j2 b/ansible/roles/stv3-common/templates/logging.conf.j2 index 27949fa..030b6b2 100644 --- a/ansible/roles/stv3-common/templates/logging.conf.j2 +++ b/ansible/roles/stv3-common/templates/logging.conf.j2 @@ -46,7 +46,7 @@ formatter = yagi [handler_watchedfile] class = handlers.WatchedFileHandler -args = () +args = ('/var/log/stv3/stv3.log',) formatter = yagi [handler_syslog] diff --git a/ansible/roles/stv3-workers/tasks/main.yaml b/ansible/roles/stv3-workers/tasks/main.yaml index f72a015..34b7f2e 100644 --- a/ansible/roles/stv3-workers/tasks/main.yaml +++ b/ansible/roles/stv3-workers/tasks/main.yaml @@ -26,13 +26,5 @@ dest=/etc/init.d/pipeline_worker_{{item}} owner=stv3 group=stv3 mode=0755 with_sequence: count={{num_pipeline_workers}} - -- name: yagi-event - debug: msg="Starting yagi-event" - notify: - - restart yagi-event - -- name: pipeline-worker - debug: msg="Starting pipeline-worker" notify: - restart pipeline-worker diff --git a/logging.conf b/logging.conf index b87d7ab..c6703ad 100644 --- a/logging.conf +++ b/logging.conf @@ -12,12 +12,12 @@ level = WARNING handlers = null [logger_yagi] -level = INFO +level = DEBUG handlers = stderr qualname = yagi [logger_winchester] -level = INFO +level = DEBUG handlers = stderr qualname = winchester diff --git a/winchester.yaml b/winchester.yaml index ed5d1e1..840b4e8 100644 --- a/winchester.yaml +++ b/winchester.yaml @@ -28,9 +28,7 @@ database: distiller_config: event_definitions.yaml trigger_definitions: triggers.yaml -#trigger_definitions: usage_triggers.yaml pipeline_config: pipelines.yaml -#pipeline_config: usage_pipelines.yaml purge_completed_streams: false @@ -42,4 +40,6 @@ purge_completed_streams: false pipeline_handlers: logger: winchester.pipeline_handler:LoggingHandler - usage_pipeline: winchester.pipeline_handler:UsageHandler + usage: winchester.pipeline_handler:UsageHandler + notabene: winchester.pipeline_handler:NotabeneHandler + diff --git a/winchester/pipelines.yaml b/winchester/pipelines.yaml index 85ffd4a..8969c1a 100644 --- a/winchester/pipelines.yaml +++ b/winchester/pipelines.yaml @@ -1,5 +1,36 @@ --- -test_pipeline: +test_fire_pipeline: - logger + - usage + - name: notabene + params: + host: localhost + user: guest + password: guest + port: 5672 + vhost: / + library: librabbitmq + exchange: nova + exchange_type: topic + queue_name: monitor.info + env_keys: + - usage_notifications + test_expire_pipeline: - logger + - usage + - name: notabene + params: + host: localhost + user: guest + password: guest + port: 5672 + vhost: / + library: librabbitmq + exchange: nova + exchange_type: topic + queue_name: monitor.info + env_keys: + - usage_notifications + + diff --git a/winchester/triggers.yaml b/winchester/triggers.yaml index bc5f240..d1ec7c3 100644 --- a/winchester/triggers.yaml +++ b/winchester/triggers.yaml @@ -4,22 +4,22 @@ distinguished_by: - instance_id - timestamp: "day" - expiration: "$last + 1h" - fire_pipeline: "test_pipeline" + expiration: "$last + 2d" + fire_pipeline: "test_fire_pipeline" expire_pipeline: "test_expire_pipeline" match_criteria: - event_type: - compute.instance.* - - scheduler.run_instance.* - snapshot_instance + - keypair.import.* + - rebuild_instance + - compute.instance.* - "!compute.instance.exists" -#### Traits are optional. -# traits: -# os_distro: ubuntu -# memory_mb: -# numeric: "> 4096" - event_type: compute.instance.exists - map_distingushed_trait: + map_distingushed_by: timestamp: audit_period_beginning fire_criteria: - event_type: compute.instance.exists + traits: + audit_period_ending: + datetime: $audit_period_beginning + 1d diff --git a/winchester/usage_pipelines.yaml b/winchester/usage_pipelines.yaml deleted file mode 100644 index 2d0903e..0000000 --- a/winchester/usage_pipelines.yaml +++ /dev/null @@ -1,5 +0,0 @@ ---- -usage_fire_pipeline: - - usage_pipeline -usage_expire_pipeline: - - usage_pipeline diff --git a/winchester/usage_triggers.yaml b/winchester/usage_triggers.yaml deleted file mode 100644 index 87b1b1a..0000000 --- a/winchester/usage_triggers.yaml +++ /dev/null @@ -1,24 +0,0 @@ -- name: usage_trigger - debug_level: 2 - distinguished_by: - - instance_id - - timestamp: "day" - expiration: "$last + 2d" - fire_pipeline: "usage_fire_pipeline" - expire_pipeline: "usage_expire_pipeline" - match_criteria: - - event_type: - - compute.instance.* - - snapshot_instance - - keypair.import.* - - rebuild_instance - - compute.instance.* - - "!compute.instance.exists" - - event_type: compute.instance.exists - map_distingushed_by: - timestamp: audit_period_beginning - fire_criteria: - - event_type: compute.instance.exists - traits: - audit_period_ending: - datetime: $audit_period_beginning + 1d diff --git a/yagi.conf.common b/yagi.conf.common index ff215ae..f0c4500 100644 --- a/yagi.conf.common +++ b/yagi.conf.common @@ -17,9 +17,9 @@ vhost = / poll_delay = 0.5 [logging] -logfile = yagi.log +#logfile = yagi.log +#default_level = DEBUG config_file = logging.conf -default_level = INFO [consumers] queues = monitor.info