From ba006c4f1cdd9a072347171a9d5d1cf6be8935be Mon Sep 17 00:00:00 2001 From: Tong Li Date: Mon, 15 Jul 2013 13:44:53 -0400 Subject: [PATCH] File based publisher Ceilometer logs metering data into a database by default. In some cases it is ideal to have the metering data go into a file which can be examined easily. This patch adds the file based publisher. One can make changes to pipeline.yaml file to add the file publisher to an existing pipeline or a new pipeline so that metering data can not only be saved in database but also can be saved in a file. The following example shows how it can be configured in pipeline. - name: meter_file interval: 600 counters: - "*" transformers: publishers: - file:///tmp/meters?max_bytes=10000000&backup_count=5 With the above configuration, a set of files named meters can be found in the /tmp directory. The file names should be meters, meters.1, meters.2 etc. These files will contain the metering data. Since the implementation uses rotating log file when max_bytes and backup_count are specified, one can increase the backup_count and max_bytes to keep the file for a longer period or use the path to point the file to a desired location. If no file path specified, the publisher will log an error and not record any metering data. Change-Id: If85ad95f837b2d178527eb7de16707f8af2c4ce2 --- ceilometer/publisher/file.py | 97 +++++++++++++++++++++++++++++ setup.cfg | 1 + tests/publisher/test_file.py | 115 +++++++++++++++++++++++++++++++++++ 3 files changed, 213 insertions(+) create mode 100644 ceilometer/publisher/file.py create mode 100644 tests/publisher/test_file.py diff --git a/ceilometer/publisher/file.py b/ceilometer/publisher/file.py new file mode 100644 index 000000000..fb046ee7c --- /dev/null +++ b/ceilometer/publisher/file.py @@ -0,0 +1,97 @@ +# -*- encoding: utf-8 -*- +# +# Copyright 2013 IBM Corp +# +# Author: Tong Li +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging +import logging.handlers +import urlparse + +from ceilometer import publisher +from ceilometer.openstack.common import log + +LOG = log.getLogger(__name__) + + +class FilePublisher(publisher.PublisherBase): + """Publisher metering data to file. + + The publisher which records metering data into a file. The file name and + location should be configured in ceilometer pipeline configuration file. + If a file name and location is not specified, this File Publisher will not + log any meters other than log a warning in Ceilometer log file. + + To enable this publisher, add the following section to file + /etc/ceilometer/publisher.yaml or simply add it to an existing pipeline. + + - + name: meter_file + interval: 600 + counters: + - "*" + transformers: + publishers: + - file:///var/test?max_bytes=10000000&backup_count=5 + + File path is required for this publisher to work properly. If max_bytes + or backup_count is missing, FileHandler will be used to save the metering + data. If max_bytes and backup_count are present, RotatingFileHandler will + be used to save the metering data. + """ + + def __init__(self, parsed_url): + super(FilePublisher, self).__init__(parsed_url) + + self.publisher_logger = None + path = parsed_url.path + if not path or path.lower() == 'file': + LOG.error('The path for the file publisher is required') + return + + rfh = None + max_bytes = 0 + backup_count = 0 + # Handling other configuration options in the query string + if parsed_url.query: + params = urlparse.parse_qs(parsed_url.query) + if params.get('max_bytes') and params.get('backup_count'): + try: + max_bytes = int(params.get('max_bytes')[0]) + backup_count = int(params.get('backup_count')[0]) + except ValueError: + LOG.error('max_bytes and backup_count should be ' + 'numbers.') + return + # create rotating file handler + rfh = logging.handlers.RotatingFileHandler( + path, encoding='utf8', maxBytes=max_bytes, + backupCount=backup_count) + + self.publisher_logger = logging.Logger('publisher.file') + self.publisher_logger.propagate = False + self.publisher_logger.setLevel(logging.INFO) + rfh.setLevel(logging.INFO) + self.publisher_logger.addHandler(rfh) + + def publish_counters(self, context, counters, source): + """Send a metering message for publishing + + :param context: Execution context from the service or RPC call + :param counter: Counter from pipeline after transformation + :param source: counter source + """ + if self.publisher_logger: + self.publisher_logger.info(counters) diff --git a/setup.cfg b/setup.cfg index e2a1724f7..816227e46 100644 --- a/setup.cfg +++ b/setup.cfg @@ -96,6 +96,7 @@ ceilometer.publisher = meter = ceilometer.publisher.rpc:RPCPublisher rpc = ceilometer.publisher.rpc:RPCPublisher udp = ceilometer.publisher.udp:UDPPublisher + file = ceilometer.publisher.file:FilePublisher ceilometer.alarm = threshold_eval = ceilometer.alarm.threshold_evaluation:Evaluator diff --git a/tests/publisher/test_file.py b/tests/publisher/test_file.py new file mode 100644 index 000000000..caf6aad8e --- /dev/null +++ b/tests/publisher/test_file.py @@ -0,0 +1,115 @@ +# -*- encoding: utf-8 -*- +# +# Copyright © 2013 eNovance +# +# Author: Julien Danjou +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +"""Tests for ceilometer/publisher/udp.py +""" + +import datetime +import os +import logging +import logging.handlers +from ceilometer import counter +from ceilometer.publisher import file +from ceilometer.tests import base +from ceilometer.openstack.common.network_utils import urlsplit + + +class TestFilePublisher(base.TestCase): + + test_data = [ + counter.Counter( + name='test', + type=counter.TYPE_CUMULATIVE, + unit='', + volume=1, + user_id='test', + project_id='test', + resource_id='test_run_tasks', + timestamp=datetime.datetime.utcnow().isoformat(), + resource_metadata={'name': 'TestPublish'}, + ), + counter.Counter( + name='test2', + type=counter.TYPE_CUMULATIVE, + unit='', + volume=1, + user_id='test', + project_id='test', + resource_id='test_run_tasks', + timestamp=datetime.datetime.utcnow().isoformat(), + resource_metadata={'name': 'TestPublish'}, + ), + counter.Counter( + name='test2', + type=counter.TYPE_CUMULATIVE, + unit='', + volume=1, + user_id='test', + project_id='test', + resource_id='test_run_tasks', + timestamp=datetime.datetime.utcnow().isoformat(), + resource_metadata={'name': 'TestPublish'}, + ), + ] + + COUNTER_SOURCE = 'testsource' + + def test_file_publisher(self): + # Test valid configurations + parsed_url = urlsplit( + 'file:///tmp/log_file?max_bytes=50&backup_count=3') + publisher = file.FilePublisher(parsed_url) + publisher.publish_counters(None, + self.test_data, + self.COUNTER_SOURCE) + + handler = publisher.publisher_logger.handlers[0] + self.assertTrue(isinstance(handler, + logging.handlers.RotatingFileHandler)) + self.assertEqual([handler.maxBytes, handler.baseFilename, + handler.backupCount], + [50, '/tmp/log_file', 3]) + # The rotating file gets created since only allow 50 bytes. + self.assertTrue(os.path.exists('/tmp/log_file.1')) + + # Test missing max bytes, backup count configurations + parsed_url = urlsplit( + 'file:///tmp/log_file_plain') + publisher = file.FilePublisher(parsed_url) + publisher.publish_counters(None, + self.test_data, + self.COUNTER_SOURCE) + + handler = publisher.publisher_logger.handlers[0] + self.assertTrue(isinstance(handler, + logging.handlers.RotatingFileHandler)) + self.assertEqual([handler.maxBytes, handler.baseFilename, + handler.backupCount], + [0, '/tmp/log_file_plain', 0]) + + # The rotating file gets created since only allow 50 bytes. + self.assertTrue(os.path.exists('/tmp/log_file_plain')) + + # Test invalid max bytes, backup count configurations + parsed_url = urlsplit( + 'file:///tmp/log_file_bad?max_bytes=yus&backup_count=5y') + publisher = file.FilePublisher(parsed_url) + publisher.publish_counters(None, + self.test_data, + self.COUNTER_SOURCE) + + self.assertIsNone(publisher.publisher_logger)