diff --git a/README.md b/README.md index 9d380e1..46d1ff1 100644 --- a/README.md +++ b/README.md @@ -6,39 +6,47 @@ Dev env for StackTach development. Uses Notigen event generator and yagi. Prerequisites: make sure you have a working python dev environment (2.7+ ideally) - including virtualenv. Install rabbitmq and mongodb + including virtualenv. + Install rabbitmq. + If using the oahu pipeline engine, install mongodb. + If using the winchester pipeline engine, install MySQL. TL;DR: handle the prerequisites above. git clone https://github.com/StackTach/sandbox.git cd sandbox + If running winchester: + create a mysql database to use + set the database url appropriately in winchester.yaml ./build.sh -Tweaks: - +Tweaks: + You can create a `local.sh` to override the defaults: SOURCE_DIR=git # where the StackTach repos are cloned VENV_DIR=.venv # name of the .venv + PIPELINE_ENGINE=oahu # Name of pipeline processing library to run. The `build.sh` script will create clone each of the StackTach projects -into the `$SOURCE_DIR` directory (so you can work on them in a running env). +into the `$SOURCE_DIR` directory (so you can work on them in a running env). -The virtualenv will be created and each of the projects -(and their dependencies) installed into it. +The virtualenv will be created and each of the projects +(and their dependencies) installed into it. -A `screen` session is started, based on `screenrc` which will start the +A `screen` session is started, based on `screenrc.oahu` or +`screenrc.winchester` which will start the `notigen` event generator. The event generator simulated OpenStack notifications and pumps them into rabbitmq. `yagi-event` is also started -with the `shoebox.conf` configuration file. This will read events from +with the `yagi.conf` configuration file. This will read events from the rabbit queue and save them to local files. The working directory -and archive directory for `shoebox` is specified in `shoebox.conf`. +and archive directory for `shoebox` is specified in `yagi.conf`. The sandbox environment configures `shoebox` to upload archive files to Swift automatically. This requires you create a credentials file -in the `.../sandbox/` directory (like in +in the `.../sandbox/` directory (like in `.../git/sandbox/etc/sample_rax_credentials.conf`) Call it `swift_credentials.conf` or alter the `shoebox.conf` file accordingly. If you don't have access to a Swift server, like CloudFiles, read -the config file for details on disabling this feature. +the config file for details on disabling this feature. diff --git a/build.sh b/build.sh index d4f6513..81ab652 100755 --- a/build.sh +++ b/build.sh @@ -4,6 +4,7 @@ echo "StackTach dev env build script" SOURCE_DIR=git VENV_DIR=.venv +PIPELINE_ENGINE=oahu if [[ -f local.sh ]]; then source local.sh @@ -21,7 +22,7 @@ cd $SOURCE_DIR for file in StackTach/shoebox StackTach/simport StackTach/notigen \ StackTach/notabene StackTach/notification_utils rackerlabs/yagi \ StackTach/stackdistiller StackTach/quincy StackTach/quince \ - StackTach/klugman StackTach/oahu + StackTach/klugman StackTach/oahu StackTach/winchester do git clone https://github.com/$file done @@ -46,4 +47,12 @@ do cd ../.. done -screen -c screenrc +(cat yagi.conf.$PIPELINE_ENGINE ; cat yagi.conf.common ) > yagi.conf + +if [ $PIPELINE_ENGINE == "winchester" ] +then + winchester_db -c winchester.yaml upgrade head +fi + +screen -c screenrc.$PIPELINE_ENGINE + diff --git a/logging.conf b/logging.conf new file mode 100644 index 0000000..d76a91d --- /dev/null +++ b/logging.conf @@ -0,0 +1,74 @@ +[loggers] +keys = root, yagi, winchester, oahu + +[handlers] +keys = stderr, stdout, watchedfile, syslog, null + +[formatters] +keys = yagi, default + +[logger_root] +level = WARNING +handlers = null + +[logger_yagi] +level = INFO +handlers = stderr +qualname = yagi + +[logger_winchester] +level = DEBUG +handlers = stderr +qualname = winchester + +[logger_oahu] +level = DEBUG +handlers = stderr +qualname = oahu + +[logger_amqplib] +level = WARNING +handlers = stderr +qualname = amqplib + +[logger_sqlalchemy] +# yagi does not use sqlalchemy... yet. -mdragon +level = WARNING +handlers = stderr +qualname = sqlalchemy +# "level = INFO" logs SQL queries. +# "level = DEBUG" logs SQL queries and results. +# "level = WARNING" logs neither. (Recommended for production systems.) + +[handler_stderr] +class = StreamHandler +args = (sys.stderr,) +formatter = yagi + +[handler_stdout] +class = StreamHandler +args = (sys.stdout,) +formatter = yagi + +[handler_watchedfile] +class = handlers.WatchedFileHandler +args = ('sandbox.log',) +formatter = yagi + +[handler_syslog] +class = handlers.SysLogHandler +args = ('/dev/log', handlers.SysLogHandler.LOG_USER) +formatter = yagi + +[handler_null] +class = NullHandler +formatter = default +args = () + +[formatter_yagi] +# substitutions available for formats are documented at: +# https://docs.python.org/2/library/logging.html#logrecord-attributes +format = %(name)s[%(levelname)s at %(asctime)s line: %(lineno)d] %(message)s + +[formatter_default] +format = %(message)s diff --git a/screenrc b/screenrc.oahu similarity index 83% rename from screenrc rename to screenrc.oahu index e67621c..e8523e5 100644 --- a/screenrc +++ b/screenrc.oahu @@ -6,9 +6,9 @@ stuff "cd git/quincy/quincy; gunicorn --log-file=- 'api:get_api(config_location= screen -t bash bash stuff "klugman streams\r" screen -t yagi1 bash -stuff "cd git/yagi/bin; ./yagi-event --config ../../../shoebox.conf\r" +stuff "yagi-event --config yagi.conf\r" screen -t yagi2 bash -stuff "cd git/yagi/bin; ./yagi-event --config ../../../shoebox.conf\r" +stuff "yagi-event --config yagi.conf\r" screen -t trigger bash stuff "pipeline trigger \".|oahu_config:Config\" --polling_rate=20\r" screen -t ready1 bash @@ -17,6 +17,6 @@ screen -t ready2 bash stuff "pipeline ready \".|oahu_config:Config\" --polling_rate=20\r" screen -t completed bash stuff "pipeline completed \".|oahu_config:Config\" --polling_rate=20\r" -screen -t gen bash +screen -t gen bash stuff "cd git/notigen/bin; python event_pump.py ../templates 2 0\r" diff --git a/screenrc.winchester b/screenrc.winchester new file mode 100644 index 0000000..a6d2719 --- /dev/null +++ b/screenrc.winchester @@ -0,0 +1,18 @@ +sessionname tach +hardstatus alwayslastline '%{= .} %-Lw%{= .}%> %n%f %t*%{= .}%+Lw%< %-=%{g}(%{d}%H/%l%{g})' +# not yet working w/ winchester +#screen -t quincy bash +#stuff "cd git/quincy/quincy; gunicorn --log-file=- 'api:get_api(config_location=\"../../../quincy.conf\")'\r" +screen -t bash bash +#stuff "klugman streams\r" +screen -t yagi1 bash +stuff "yagi-event --config yagi.conf\r" +screen -t yagi2 bash +stuff "yagi-event --config yagi.conf\r" +screen -t pipeline1 bash +stuff "pipeline_worker -c winchester.yaml\r" +screen -t pipeline2 bash +stuff "pipeline_worker -c winchester.yaml\r" +screen -t gen bash +stuff "cd git/notigen/bin; python event_pump.py ../templates 2 0\r" + diff --git a/winchester.yaml b/winchester.yaml new file mode 100644 index 0000000..cb8400b --- /dev/null +++ b/winchester.yaml @@ -0,0 +1,34 @@ +--- +###### This adds directories to the search path for other configfiles. +config_path: winchester +## It can also be a list: +#config_path: +# - /etc/winchester +# - /home/wherever + +###### logging +#log_level: debug +## You can also use a full logging config file. +logging_config: logging.conf + +###### How often to log stats +statistics_period: 10 + +pipeline_worker_batch_size: 1000 +pipeline_worker_delay: 10 + +####### You can specify extra stackdistiller trait plugins here: +#distiller_trait_plugins: +# test: some.module.path:TestTraitPlugin + +catch_all_notifications: false + +database: + url: mysql://winchester:testpasswd@localhost/winchester + +distiller_config: event_definitions.yaml +trigger_definitions: triggers.yaml +pipeline_config: pipelines.yaml + +pipeline_handlers: + logger: winchester.pipeline_handler:LoggingHandler diff --git a/winchester/event_definitions.yaml b/winchester/event_definitions.yaml new file mode 100644 index 0000000..b4f00d3 --- /dev/null +++ b/winchester/event_definitions.yaml @@ -0,0 +1,63 @@ +--- +- event_type: compute.instance.* + traits: &instance_traits + tenant_id: + fields: payload.tenant_id + user_id: + fields: payload.user_id + instance_id: + fields: payload.instance_id + host: + fields: publisher_id + plugin: + name: split + parameters: + segment: 1 + max_split: 1 + service: + fields: publisher_id + plugin: split + memory_mb: + type: int + fields: payload.memory_mb + disk_gb: + type: int + fields: payload.disk_gb + root_gb: + type: int + fields: payload.root_gb + ephemeral_gb: + type: int + fields: payload.ephemeral_gb + vcpus: + type: int + fields: payload.vcpus + instance_type_id: + type: int + fields: payload.instance_type_id + instance_type: + fields: payload.instance_type + state: + fields: payload.state + os_architecture: + fields: payload.image_meta.'org.openstack__1__architecture' + os_version: + fields: payload.image_meta.'org.openstack__1__os_version' + os_distro: + fields: payload.image_meta.'org.openstack__1__os_distro' + launched_at: + type: datetime + fields: payload.launched_at + deleted_at: + type: datetime + fields: payload.deleted_at +- event_type: compute.instance.exists + traits: + <<: *instance_traits + audit_period_beginning: + type: datetime + fields: payload.audit_period_beginning + audit_period_ending: + type: datetime + fields: payload.audit_period_ending + diff --git a/winchester/pipelines.yaml b/winchester/pipelines.yaml new file mode 100644 index 0000000..85ffd4a --- /dev/null +++ b/winchester/pipelines.yaml @@ -0,0 +1,5 @@ +--- +test_pipeline: + - logger +test_expire_pipeline: + - logger diff --git a/winchester/triggers.yaml b/winchester/triggers.yaml new file mode 100644 index 0000000..6c82d5a --- /dev/null +++ b/winchester/triggers.yaml @@ -0,0 +1,22 @@ +--- +- name: test_trigger + distinguished_by: + - instance_id + - timestamp: "day" + expiration: "$last + 1h" + fire_pipeline: "test_pipeline" + expire_pipeline: "test_expire_pipeline" + match_criteria: + - event_type: + - compute.instance.* + - "!compute.instance.exists" +#### Traits are optional. +# traits: +# os_distro: ubuntu +# memory_mb: +# numeric: "> 4096" + - event_type: compute.instance.exists + map_distingushed_trait: + timestamp: audit_period_beginning + fire_criteria: + - event_type: compute.instance.exists diff --git a/shoebox.conf b/yagi.conf.common similarity index 66% rename from shoebox.conf rename to yagi.conf.common index 026e22d..ff215ae 100644 --- a/shoebox.conf +++ b/yagi.conf.common @@ -18,52 +18,47 @@ poll_delay = 0.5 [logging] logfile = yagi.log +config_file = logging.conf default_level = INFO [consumers] queues = monitor.info -[consumer:monitor.info] -#apps = yagi.handler.shoebox_handler.ShoeboxHandler -apps = oahu.yagi_handler.OahuHandler -exchange = monitor -exchange_type = topic -routing_key = monitor.info -durable = True -max_messages = 100 - # ------ NOTE ------ -# Yagi is launched from sandbox/git/yagi/bin, so files are +# Yagi is launched from the sandbox/ dir, so files are # relative to there. # ------------------ [oahu] -config_class = ../../..|oahu_config:Config +config_class = .|oahu_config:Config + +[winchester] +config_file = winchester.yaml [shoebox] # Store in-process files in ./working # Move them to ./archive when full via the MoveFileCallback # Roll files every 1mb -working_directory=../../../data/working -destination_folder=../../../data/archive +working_directory=data/working +destination_folder=data/archive filename_template=events_%Y_%m_%d_%X_%f.dat roll_checker=shoebox.roll_checker:SizeRollChecker roll_size_mb=1 -distiller_conf=../../../distiller.conf +distiller_conf=distiller.conf # Swift upload support # create a credentials file (see shoebox/bin/sample_credentials.conf) callback=shoebox.handlers:CallbackList callback_list=shoebox.handlers:MoveFileCallback, shoebox.handlers:SwiftUploadCallback, shoebox.handlers:DeleteFileCallback container=sandbox -credentials_file=../../../swift_credentials.conf +credentials_file=swift_credentials.conf auth_method=rackspace region=DFW -# If you don't want Swift support, comment the above callback= +# If you don't want Swift support, comment the above callback= # entry and uncomment this one: #callback=shoebox.handlers:MoveFileCallback # which will just move the file into the archive directory. diff --git a/yagi.conf.oahu b/yagi.conf.oahu new file mode 100644 index 0000000..93a77bf --- /dev/null +++ b/yagi.conf.oahu @@ -0,0 +1,10 @@ +[consumer:monitor.info] +#apps = yagi.handler.shoebox_handler.ShoeboxHandler +apps = oahu.yagi_handler.OahuHandler +exchange = monitor +exchange_type = topic +routing_key = monitor.info +durable = True +max_messages = 100 + + diff --git a/yagi.conf.winchester b/yagi.conf.winchester new file mode 100644 index 0000000..732160b --- /dev/null +++ b/yagi.conf.winchester @@ -0,0 +1,10 @@ +[consumer:monitor.info] +#apps = yagi.handler.shoebox_handler.ShoeboxHandler +apps = winchester.yagi_handler.WinchesterHandler +exchange = monitor +exchange_type = topic +routing_key = monitor.info +durable = True +max_messages = 100 + +