Reconcile example and mini-mon configs

Bring contents of the example config and the mini-mon config into line, add a note on log formatting, and a note in the README about the sample configurations. Change-Id: I47794b600d44a87f58825d1e11099c272f06d7aa
2015-09-30 18:16:56 +01:00 · 2015-09-30 18:16:56 +01:00 · cb99a66ee1
commit cb99a66ee1
parent de1d44e63d
3 changed files with 59 additions and 41 deletions
--- a/README.md
+++ b/README.md
@ -3,11 +3,11 @@ monasca-persister

 The Monitoring Persister consumes metrics and alarm state transitions from the Message Queue and stores them in the Metrics and Alarms database.

-Although the Persister isn't primarly a Web service it uses DropWizard, https://dropwizard.github.io/dropwizard/, which provides a nice Web appliction framework to expose an http endpoint that provides an interface through which metrics about the Persister can be queried as well as health status. 
+Although the Persister isn't primarly a Web service it uses DropWizard, https://dropwizard.github.io/dropwizard/, which provides a nice Web appliction framework to expose an http endpoint that provides an interface through which metrics about the Persister can be queried as well as health status.

 The basic design of the Persister is to have one Kafka consumer publish to a Disruptor, https://github.com/LMAX-Exchange/disruptor, that has output processors. The output processors use prepared batch statements to write to the Metrics and Alarms database.

-The number of output processors/threads in the Persister can be specified to scale to more messages. To horizontally scale and provide fault-tolerance any number of Persisters can be started as consumers from the the Message Queue. 
+The number of output processors/threads in the Persister can be specified to scale to more messages. To horizontally scale and provide fault-tolerance any number of Persisters can be started as consumers from the the Message Queue.

 # Build

@ -17,10 +17,16 @@ Requires monasca-common from https://github.com/stackforge/monasca-common. Downl
 mvn clean package
 ```

+# Configuration
+
+A sample configuration file is available in java/src/deb/etc/persister-config.yml-sample.
+
+A second configuration file is provided in java/src/main/resources/persister-config.yml for use with the [vagrant "mini-mon" development environment](https://github.com/stackforge/monasca-vagrant/).
+
 # TODO

 * Purge metrics on shutdown
-* Add more robust offset management in Kafka. Currently, the offset is advanced as each message is read. If the Persister stops after the metric has been read and prior to it being committed to the Metrics and Alarms database, the metric will be lost. 
+* Add more robust offset management in Kafka. Currently, the offset is advanced as each message is read. If the Persister stops after the metric has been read and prior to it being committed to the Metrics and Alarms database, the metric will be lost.
 * Add better handling of SQL exceptions.
 * Complete health check.
 * Specify and document the names of the metrics that are available for monitoring of the Persister.
@ -35,7 +41,7 @@ you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0
-    
+
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
--- a/java/src/deb/etc/persister-config.yml-sample
+++ b/java/src/deb/etc/persister-config.yml-sample
@ -4,25 +4,25 @@ alarmHistoryConfiguration:
  batchSize: 100
  numThreads: 2
  maxBatchTime: 15
-# See http://kafka.apache.org/documentation.html#api for semantics and defaults.
+  # See http://kafka.apache.org/documentation.html#api for semantics and defaults.
  topic: alarm-state-transitions
  groupId: 1_alarm-state-transitions
  consumerId: mini-mon
-  clientId : 1
+  clientId: 1

 metricConfiguration:
  batchSize: 10000
  numThreads: 4
  maxBatchTime: 15
-# See http://kafka.apache.org/documentation.html#api for semantics and defaults.
+  # See http://kafka.apache.org/documentation.html#api for semantics and defaults.
  topic: metrics
  groupId: 1_metrics
  consumerId: mini-mon
-  clientId : 1
+  clientId: 1

 #Kafka settings.
 kafkaConfig:
-# See http://kafka.apache.org/documentation.html#api for semantics and defaults.
+  # See http://kafka.apache.org/documentation.html#api for semantics and defaults.
  zookeeperConnect: localhost:2181
  socketTimeoutMs: 30000
  socketReceiveBufferBytes : 65536
@ -43,15 +43,17 @@ verticaMetricRepoConfig:
  maxCacheSize: 2000000

 databaseConfiguration:
-# vertica | influxdb
+  # databaseType can be (vertica | influxdb)
  databaseType: influxdb

 # Uncomment if databaseType is influxdb
 influxDbConfiguration:
  # Retention policy may be left blank to indicate default policy.
  retentionPolicy:
+  # Used only if version is V9.
  maxHttpConnections: 100
  name: mon
+  replicationFactor: 1
  url: http://localhost:8086
  user: mon_persister
  password: password
@ -63,24 +65,28 @@ influxDbConfiguration:
 #  user: dbadmin
 #  password: password
 #  properties:
-#      ssl: false
+#    ssl: false
 #  # the maximum amount of time to wait on an empty pool before throwing an exception
 #  maxWaitForConnection: 1s
+#
 #  # the SQL query to run when validating a connection's liveness
 #  validationQuery: "/* MyService Health Check */ SELECT 1"
+#
 #  # the minimum number of connections to keep open
 #  minSize: 8
+#
 #  # the maximum number of connections to keep open
 #  maxSize: 41
+#
 #  # whether or not idle connections should be validated
 #  checkConnectionWhileIdle: false
+#
 #  # the maximum lifetime of an idle connection
 #  maxConnectionAge: 1 minute

 metrics:
  frequency: 1 second

-
 # Logging settings.
 logging:

@ -90,23 +96,38 @@ logging:

  # Logger-specific levels.
  loggers:
-
-    # Sets the level for 'com.example.app' to DEBUG.
-      com.example.app: DEBUG
-    # com.hpcloud: debug
-    # com.hpcloud.mon.persister.repository: DEBUG
+    monasca: DEBUG

  appenders:
+    # Uncomment to enable logging to the console:
+    #- type: console
+    #  threshold: DEBUG
+    #  timeZone: UTC
+    #  target: stdout

    - type: file
      threshold: INFO
      archive: true
+      # The file to which current statements will be logged.
      currentLogFilename: /var/log/monasca/persister/monasca-persister.log
+
+      # When the log file rotates, the archived log will be renamed to this and gzipped. The
+      # %d is replaced with the previous day (yyyy-MM-dd). Custom rolling windows can be created
+      # by passing a SimpleDateFormat-compatible format as an argument: "%d{yyyy-MM-dd-hh}".
      archivedLogFilenamePattern: /var/log/monasca/persister/monasca-persister.log-%d.log.gz
+
+      # The number of archived files to keep.
      archivedFileCount: 5
+
      # The timezone used to format dates. HINT: USE THE DEFAULT, UTC.
      timeZone: UTC

+      # Uncomment to approximately match the default log format of the python
+      # Openstack components. %pid is unavoidably formatted with [brackets],
+      # which are hard-coded in dropwizard's logging module.
+      # See http://logback.qos.ch/manual/layouts.html#conversionWord for details of the format string
+      # logFormat: "%app%pid: %d{YYYY-MM-dd HH:mm:ss.SSS} %pid %level %logger [-] [%thread] %msg %ex{1}"
+
 # Set the persister ports to 8090/8091 to avoid conflict with the api
 server:
  applicationConnectors:
--- a/java/src/main/resources/persister-config.yml
+++ b/java/src/main/resources/persister-config.yml
@ -4,7 +4,7 @@ alarmHistoryConfiguration:
  batchSize: 100
  numThreads: 1
  maxBatchTime: 15
-# See http://kafka.apache.org/documentation.html#api for semantics and defaults.
+  # See http://kafka.apache.org/documentation.html#api for semantics and defaults.
  topic:  alarm-state-transitions
  groupId: persister_alarms
  consumerId: 1
@ -14,7 +14,7 @@ metricConfiguration:
  batchSize: 1000
  numThreads: 2
  maxBatchTime: 30
-# See http://kafka.apache.org/documentation.html#api for semantics and defaults.
+  # See http://kafka.apache.org/documentation.html#api for semantics and defaults.
  topic:  metrics
  groupId: persister_metrics
  consumerId: 1
@ -22,8 +22,7 @@ metricConfiguration:

 #Kafka settings.
 kafkaConfig:
-  #zookeeperConnect: localhost:2181
-# See http://kafka.apache.org/documentation.html#api for semantics and defaults.
+  # See http://kafka.apache.org/documentation.html#api for semantics and defaults.
  zookeeperConnect: 192.168.10.4:2181
  socketTimeoutMs: 30000
  socketReceiveBufferBytes : 65536
@ -44,10 +43,10 @@ verticaMetricRepoConfig:
  maxCacheSize: 2000000

 databaseConfiguration:
-# databaseType can be (vertica | influxdb)
+  # databaseType can be (vertica | influxdb)
  databaseType: influxdb
-#  databaseType: vertica

+# Uncomment if databaseType is influxdb
 influxDbConfiguration:
  # Retention policy may be left blank to indicate default policy.
  retentionPolicy:
@ -56,19 +55,14 @@ influxDbConfiguration:
  name: mon
  replicationFactor: 1
  url: http://192.168.10.4:8086
-#  url: http://10.10.10.2:8086
  user:  root
  password: root

+# Uncomment if databaseType is vertica
 #dataSourceFactory:
 #  driverClass: com.vertica.jdbc.Driver
-##  url: jdbc:vertica://mini-mon:5433/monasca
-#  # my Vertica DB in the cloud
-#  #url: jdbc:vertica://15.185.94.245:5433/som
-#  url: jdbc:vertica://192.168.10.4:5433/mon
-##  user: persister
-#  #user: mon_persister
-#  user: dbadmin
+#  url: jdbc:vertica://192.168.10.4:5433/monasca
+#  user: monasca_persister
 #  password: password
 #  properties:
 #    ssl: false
@ -98,17 +92,14 @@ logging:

  # The default level of all loggers. Can be OFF, ERROR, WARN, INFO,
  # DEBUG, TRACE, or ALL.
-  level: info
+  level: INFO

  # Logger-specific levels.
  loggers:
-
-    # Sets the level for 'com.example.app' to DEBUG.
-    com.example.app: DEBUG
    monasca: DEBUG
-#    monasca.persister.repository: DEBUG

  appenders:
+
    - type: console
      threshold: DEBUG
      timeZone: UTC
@ -117,11 +108,6 @@ logging:
    - type: file
      threshold: DEBUG
      archive: true
-      currentLogFilename: ./logs/monasca-persister.log
-      archivedLogFilenamePattern: ./logs/monasca-persister-%d.log.gz
-      archivedFileCount: 5
-
-    - type: file
      # The file to which current statements will be logged.
      currentLogFilename: ./logs/monasca-persister.log

@ -136,3 +122,8 @@ logging:
      # The timezone used to format dates. HINT: USE THE DEFAULT, UTC.
      timeZone: UTC

+      # Uncomment to approximately match the default log format of the python
+      # Openstack components. %pid is unavoidably formatted with [brackets],
+      # which are hard-coded in dropwizard's logging module.
+      # See http://logback.qos.ch/manual/layouts.html#conversionWord for details of the format string
+      # logFormat: "%app%pid: %d{YYYY-MM-dd HH:mm:ss.SSS} %pid %level %logger [-] [%thread] %msg %ex{1}"