From cc01846056d184d49032fb84e74c0c3654d4f969 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Tue, 7 Oct 2014 17:45:07 +0000 Subject: [PATCH] Preliminary time-sync and usage support Optionally enable STv2 event playback through the time_sync service and early UsageHandler support. Change-Id: I5988d320fe41df29c52ebdb8e579bf0d7f53a45f --- .gitreview | 5 +++++ build.sh | 1 + logging.conf | 5 +++-- reset.sh | 17 +++++++++++++++ reset.sql | 2 ++ screenrc.winchester | 6 ++++++ winchester.yaml | 9 ++++++++ winchester/event_definitions.yaml | 36 ++++++++++++++++++++++++++----- winchester/triggers.yaml | 3 ++- winchester/usage_pipelines.yaml | 5 +++++ winchester/usage_triggers.yaml | 24 +++++++++++++++++++++ 11 files changed, 105 insertions(+), 8 deletions(-) create mode 100644 .gitreview create mode 100755 reset.sh create mode 100644 reset.sql create mode 100644 winchester/usage_pipelines.yaml create mode 100644 winchester/usage_triggers.yaml diff --git a/.gitreview b/.gitreview new file mode 100644 index 0000000..be9342a --- /dev/null +++ b/.gitreview @@ -0,0 +1,5 @@ +[gerrit] +host=review.openstack.org +port=29418 +project=stackforge/stacktach-sandbox.git + diff --git a/build.sh b/build.sh index 537aacf..56ee3c7 100755 --- a/build.sh +++ b/build.sh @@ -35,6 +35,7 @@ cd .. source ./$VENV_DIR/bin/activate # Some extra required libs ... +pip install mysql-connector-python --allow-external mysql-connector-python pip install gunicorn pip install httpie pip install librabbitmq diff --git a/logging.conf b/logging.conf index d76a91d..b87d7ab 100644 --- a/logging.conf +++ b/logging.conf @@ -17,7 +17,7 @@ handlers = stderr qualname = yagi [logger_winchester] -level = DEBUG +level = INFO handlers = stderr qualname = winchester @@ -68,7 +68,8 @@ args = () [formatter_yagi] # substitutions available for formats are documented at: # https://docs.python.org/2/library/logging.html#logrecord-attributes -format = %(name)s[%(levelname)s at %(asctime)s line: %(lineno)d] %(message)s +#format = %(name)s[%(levelname)s at %(asctime)s line: %(lineno)d] %(message)s +format = %(name)s[%(levelname)s line: %(lineno)d] %(message)s [formatter_default] format = %(message)s diff --git a/reset.sh b/reset.sh new file mode 100755 index 0000000..1817c3b --- /dev/null +++ b/reset.sh @@ -0,0 +1,17 @@ +echo "This will nuke all StackTach.v3 data in mysql and rabbitmq!" +read -p "Are you sure? " -n 1 -r +echo # (optional) move to a new line +if [[ $REPLY =~ ^[Yy]$ ]] +then +mysql -u root --password=password < reset.sql +sudo rabbitmqctl stop_app +sudo rabbitmqctl reset +sudo rabbitmqctl start_app +fi + + + + + + + diff --git a/reset.sql b/reset.sql new file mode 100644 index 0000000..24d1a2f --- /dev/null +++ b/reset.sql @@ -0,0 +1,2 @@ +drop database winchester; +create database winchester; diff --git a/screenrc.winchester b/screenrc.winchester index 0525690..bbb3438 100644 --- a/screenrc.winchester +++ b/screenrc.winchester @@ -1,8 +1,13 @@ sessionname tach hardstatus alwayslastline '%{= .} %-Lw%{= .}%> %n%f %t*%{= .}%+Lw%< %-=%{g}(%{d}%H/%l%{g})' +# Enable this if you're replaying old events ... +#screen -t time_sync bash +#stuff "cd git/stacktach-notigen/bin; gunicorn -b 127.0.0.1:8001 --log-file=- 'time_sync:get_api()'\r" + # not yet working w/ winchester #screen -t quincy bash #stuff "cd git/stacktach-quincy/quincy; gunicorn --log-file=- 'api:get_api(config_location=\"../../../quincy.conf\")'\r" + screen -t bash bash #stuff "klugman streams\r" screen -t yagi1 bash @@ -14,5 +19,6 @@ stuff "pipeline_worker -c winchester.yaml\r" screen -t pipeline2 bash stuff "pipeline_worker -c winchester.yaml\r" screen -t gen bash +#stuff "cd git/stacktach-notigen/bin; python pump_from_stv2.py\r" stuff "cd git/stacktach-notigen/bin; python event_pump.py ../templates 2 0\r" diff --git a/winchester.yaml b/winchester.yaml index cb8400b..45ebb38 100644 --- a/winchester.yaml +++ b/winchester.yaml @@ -28,7 +28,16 @@ database: distiller_config: event_definitions.yaml trigger_definitions: triggers.yaml +#trigger_definitions: usage_triggers.yaml pipeline_config: pipelines.yaml +#pipeline_config: usage_pipelines.yaml + +# If you are replaying old events (timestamps in the past) +# you will need a time service to keep the clocks +# between yagi and the pipeline workers in sync. +# Don't set this if you're running with real-time events. +#time_sync_endpoint: http://127.0.0.1:8001 pipeline_handlers: logger: winchester.pipeline_handler:LoggingHandler + usage_pipeline: winchester.pipeline_handler:UsageHandler diff --git a/winchester/event_definitions.yaml b/winchester/event_definitions.yaml index b4f00d3..537e2d7 100644 --- a/winchester/event_definitions.yaml +++ b/winchester/event_definitions.yaml @@ -1,12 +1,21 @@ ---- - event_type: compute.instance.* traits: &instance_traits tenant_id: - fields: payload.tenant_id + fields: + - payload.tenant_id + - _context_project_id user_id: fields: payload.user_id + request_id: + fields: _context_request_id + message: + fields: payload.message instance_id: - fields: payload.instance_id + fields: + - payload.instance_uuid + - payload.instance_id + - exception.kwargs.uuid + - instance.uuid host: fields: publisher_id plugin: @@ -17,6 +26,10 @@ service: fields: publisher_id plugin: split + flavor_id: + fields: + - payload.instance_flavor_id + - payload.image_meta.instance_type_flavor_id memory_mb: type: int fields: payload.memory_mb @@ -50,7 +63,9 @@ fields: payload.launched_at deleted_at: type: datetime - fields: payload.deleted_at + fields: + - payload.deleted_at + - payload.terminated_at - event_type: compute.instance.exists traits: <<: *instance_traits @@ -60,4 +75,15 @@ audit_period_ending: type: datetime fields: payload.audit_period_ending - +- event_type: snapshot_instance + traits: + <<: *instance_traits +- event_type: scheduler.run_instance.* + traits: + <<: *instance_traits +- event_type: keypair.import.* + traits: + <<: *instance_traits +- event_type: rebuild_instance + traits: + <<: *instance_traits diff --git a/winchester/triggers.yaml b/winchester/triggers.yaml index f415bfb..4ceac2c 100644 --- a/winchester/triggers.yaml +++ b/winchester/triggers.yaml @@ -1,4 +1,3 @@ ---- - name: test_trigger debug_level: 2 distinguished_by: @@ -9,6 +8,8 @@ expire_pipeline: "test_expire_pipeline" match_criteria: - event_type: + - scheduler.run_instance.* + - snapshot_instance - compute.instance.* - "!compute.instance.exists" #### Traits are optional. diff --git a/winchester/usage_pipelines.yaml b/winchester/usage_pipelines.yaml new file mode 100644 index 0000000..2d0903e --- /dev/null +++ b/winchester/usage_pipelines.yaml @@ -0,0 +1,5 @@ +--- +usage_fire_pipeline: + - usage_pipeline +usage_expire_pipeline: + - usage_pipeline diff --git a/winchester/usage_triggers.yaml b/winchester/usage_triggers.yaml new file mode 100644 index 0000000..87b1b1a --- /dev/null +++ b/winchester/usage_triggers.yaml @@ -0,0 +1,24 @@ +- name: usage_trigger + debug_level: 2 + distinguished_by: + - instance_id + - timestamp: "day" + expiration: "$last + 2d" + fire_pipeline: "usage_fire_pipeline" + expire_pipeline: "usage_expire_pipeline" + match_criteria: + - event_type: + - compute.instance.* + - snapshot_instance + - keypair.import.* + - rebuild_instance + - compute.instance.* + - "!compute.instance.exists" + - event_type: compute.instance.exists + map_distingushed_by: + timestamp: audit_period_beginning + fire_criteria: + - event_type: compute.instance.exists + traits: + audit_period_ending: + datetime: $audit_period_beginning + 1d