File based publisher

Ceilometer logs metering data into a database by default. In some cases
it is ideal to have the metering data go into a file which can be
examined easily. This patch adds the file based publisher. One can
make changes to pipeline.yaml file to add the file publisher to
an existing pipeline or a new pipeline so that metering data can not
only be saved in database but also can be saved in a file.

The following example shows how it can be configured in pipeline.

    -
        name: meter_file
        interval: 600
        counters:
            - "*"
        transformers:
        publishers:
            - file:///tmp/meters?max_bytes=10000000&backup_count=5

With the above configuration, a set of files named meters can be
found in the /tmp directory. The file names should be meters, meters.1,
meters.2 etc. These files will contain the metering data. Since the
implementation uses rotating log file when max_bytes and backup_count
are specified, one can increase the backup_count and max_bytes to keep
the file for a longer period or use the path to point the file to a
desired location. If no file path specified, the publisher will log an
error and not record any metering data.

Change-Id: If85ad95f837b2d178527eb7de16707f8af2c4ce2

ceilometer/publisher/file.py

@@ -0,0 +1,97 @@
+# -*- encoding: utf-8 -*-
+#
+# Copyright 2013 IBM Corp
+#
+# Author: Tong Li <litong01@us.ibm.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import logging
+import logging.handlers
+import urlparse
+
+from ceilometer import publisher
+from ceilometer.openstack.common import log
+
+LOG = log.getLogger(__name__)
+
+
+class FilePublisher(publisher.PublisherBase):
+    """Publisher metering data to file.
+
+    The publisher which records metering data into a file. The file name and
+    location should be configured in ceilometer pipeline configuration file.
+    If a file name and location is not specified, this File Publisher will not
+    log any meters other than log a warning in Ceilometer log file.
+
+    To enable this publisher, add the following section to file
+    /etc/ceilometer/publisher.yaml or simply add it to an existing pipeline.
+
+        -
+            name: meter_file
+            interval: 600
+            counters:
+                - "*"
+            transformers:
+            publishers:
+                - file:///var/test?max_bytes=10000000&backup_count=5
+
+    File path is required for this publisher to work properly. If max_bytes
+    or backup_count is missing, FileHandler will be used to save the metering
+    data. If max_bytes and backup_count are present, RotatingFileHandler will
+    be used to save the metering data.
+    """
+
+    def __init__(self, parsed_url):
+        super(FilePublisher, self).__init__(parsed_url)
+
+        self.publisher_logger = None
+        path = parsed_url.path
+        if not path or path.lower() == 'file':
+            LOG.error('The path for the file publisher is required')
+            return
+
+        rfh = None
+        max_bytes = 0
+        backup_count = 0
+        # Handling other configuration options in the query string
+        if parsed_url.query:
+            params = urlparse.parse_qs(parsed_url.query)
+            if params.get('max_bytes') and params.get('backup_count'):
+                try:
+                    max_bytes = int(params.get('max_bytes')[0])
+                    backup_count = int(params.get('backup_count')[0])
+                except ValueError:
+                    LOG.error('max_bytes and backup_count should be '
+                              'numbers.')
+                    return
+        # create rotating file handler
+        rfh = logging.handlers.RotatingFileHandler(
+            path, encoding='utf8', maxBytes=max_bytes,
+            backupCount=backup_count)
+
+        self.publisher_logger = logging.Logger('publisher.file')
+        self.publisher_logger.propagate = False
+        self.publisher_logger.setLevel(logging.INFO)
+        rfh.setLevel(logging.INFO)
+        self.publisher_logger.addHandler(rfh)
+
+    def publish_counters(self, context, counters, source):
+        """Send a metering message for publishing
+
+        :param context: Execution context from the service or RPC call
+        :param counter: Counter from pipeline after transformation
+        :param source: counter source
+        """
+        if self.publisher_logger:
+            self.publisher_logger.info(counters)

setup.cfg

@@ -96,6 +96,7 @@ ceilometer.publisher =
     meter = ceilometer.publisher.rpc:RPCPublisher
     rpc = ceilometer.publisher.rpc:RPCPublisher
     udp = ceilometer.publisher.udp:UDPPublisher
+    file = ceilometer.publisher.file:FilePublisher
 
 ceilometer.alarm =
     threshold_eval = ceilometer.alarm.threshold_evaluation:Evaluator

tests/publisher/test_file.py

@@ -0,0 +1,115 @@
+# -*- encoding: utf-8 -*-
+#
+# Copyright © 2013 eNovance
+#
+# Author: Julien Danjou <julien@danjou.info>
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+"""Tests for ceilometer/publisher/udp.py
+"""
+
+import datetime
+import os
+import logging
+import logging.handlers
+from ceilometer import counter
+from ceilometer.publisher import file
+from ceilometer.tests import base
+from ceilometer.openstack.common.network_utils import urlsplit
+
+
+class TestFilePublisher(base.TestCase):
+
+    test_data = [
+        counter.Counter(
+            name='test',
+            type=counter.TYPE_CUMULATIVE,
+            unit='',
+            volume=1,
+            user_id='test',
+            project_id='test',
+            resource_id='test_run_tasks',
+            timestamp=datetime.datetime.utcnow().isoformat(),
+            resource_metadata={'name': 'TestPublish'},
+        ),
+        counter.Counter(
+            name='test2',
+            type=counter.TYPE_CUMULATIVE,
+            unit='',
+            volume=1,
+            user_id='test',
+            project_id='test',
+            resource_id='test_run_tasks',
+            timestamp=datetime.datetime.utcnow().isoformat(),
+            resource_metadata={'name': 'TestPublish'},
+        ),
+        counter.Counter(
+            name='test2',
+            type=counter.TYPE_CUMULATIVE,
+            unit='',
+            volume=1,
+            user_id='test',
+            project_id='test',
+            resource_id='test_run_tasks',
+            timestamp=datetime.datetime.utcnow().isoformat(),
+            resource_metadata={'name': 'TestPublish'},
+        ),
+    ]
+
+    COUNTER_SOURCE = 'testsource'
+
+    def test_file_publisher(self):
+        # Test valid configurations
+        parsed_url = urlsplit(
+            'file:///tmp/log_file?max_bytes=50&backup_count=3')
+        publisher = file.FilePublisher(parsed_url)
+        publisher.publish_counters(None,
+                                   self.test_data,
+                                   self.COUNTER_SOURCE)
+
+        handler = publisher.publisher_logger.handlers[0]
+        self.assertTrue(isinstance(handler,
+                                   logging.handlers.RotatingFileHandler))
+        self.assertEqual([handler.maxBytes, handler.baseFilename,
+                          handler.backupCount],
+                         [50, '/tmp/log_file', 3])
+        # The rotating file gets created since only allow 50 bytes.
+        self.assertTrue(os.path.exists('/tmp/log_file.1'))
+
+        # Test missing max bytes, backup count configurations
+        parsed_url = urlsplit(
+            'file:///tmp/log_file_plain')
+        publisher = file.FilePublisher(parsed_url)
+        publisher.publish_counters(None,
+                                   self.test_data,
+                                   self.COUNTER_SOURCE)
+
+        handler = publisher.publisher_logger.handlers[0]
+        self.assertTrue(isinstance(handler,
+                                   logging.handlers.RotatingFileHandler))
+        self.assertEqual([handler.maxBytes, handler.baseFilename,
+                          handler.backupCount],
+                         [0, '/tmp/log_file_plain', 0])
+
+        # The rotating file gets created since only allow 50 bytes.
+        self.assertTrue(os.path.exists('/tmp/log_file_plain'))
+
+        # Test invalid max bytes, backup count configurations
+        parsed_url = urlsplit(
+            'file:///tmp/log_file_bad?max_bytes=yus&backup_count=5y')
+        publisher = file.FilePublisher(parsed_url)
+        publisher.publish_counters(None,
+                                   self.test_data,
+                                   self.COUNTER_SOURCE)
+
+        self.assertIsNone(publisher.publisher_logger)