diff --git a/ceilometer/objectstore/swift_middleware.py b/ceilometer/objectstore/swift_middleware.py new file mode 100644 index 000000000..18e3afdb9 --- /dev/null +++ b/ceilometer/objectstore/swift_middleware.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +# +# Copyright © 2012 eNovance +# +# Author: Julien Danjou +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import + +from ceilometer import publish +from ceilometer import counter +from ceilometer.openstack.common import cfg +from ceilometer.openstack.common import context +from ceilometer.openstack.common import timeutils + +from swift.common.swob import Request +from swift.common.utils import split_path +try: + # Swift > 1.7.5 + from swift.common.utils import InputProxy +except ImportError: + # Swift <= 1.7.5 + from swift.common.middleware.proxy_logging import InputProxy + + +class CeilometerMiddleware(object): + """ + Ceilometer middleware used for counting requests. + """ + + def __init__(self, app, conf): + self.app = app + cfg.CONF([], project='ceilometer') + + def __call__(self, env, start_response): + start_response_args = [None] + input_proxy = InputProxy(env['wsgi.input']) + env['wsgi.input'] = input_proxy + + def my_start_response(status, headers, exc_info=None): + start_response_args[0] = (status, list(headers), exc_info) + + def iter_response(iterable): + if start_response_args[0]: + start_response(*start_response_args[0]) + bytes_sent = 0 + try: + for chunk in iterable: + if chunk: + bytes_sent += len(chunk) + yield chunk + finally: + self.publish_counter(env, + input_proxy.bytes_received, + bytes_sent) + + try: + iterable = self.app(env, my_start_response) + except Exception: + self.publish_counter(env, input_proxy.bytes_received, 0) + raise + else: + return iter_response(iterable) + + @staticmethod + def publish_counter(env, bytes_received, bytes_sent): + req = Request(env) + version, account, container, obj = split_path(req.path, 1, 4, True) + now = timeutils.utcnow().isoformat() + + if bytes_received: + publish.publish_counter(context.get_admin_context(), + counter.Counter( + name='storage.objects.incoming.bytes', + type='delta', + volume=bytes_received, + user_id=env.get('HTTP_X_USER_ID'), + project_id=env.get('HTTP_X_TENANT_ID'), + resource_id=account.partition( + 'AUTH_')[2], + timestamp=now, + resource_metadata={ + "path": req.path, + "version": version, + "container": container, + "object": obj, + }), + cfg.CONF.metering_topic, + cfg.CONF.metering_secret, + cfg.CONF.counter_source) + + if bytes_sent: + publish.publish_counter(context.get_admin_context(), + counter.Counter( + name='storage.objects.outgoing.bytes', + type='delta', + volume=bytes_sent, + user_id=env.get('HTTP_X_USER_ID'), + project_id=env.get('HTTP_X_TENANT_ID'), + resource_id=account.partition( + 'AUTH_')[2], + timestamp=now, + resource_metadata={ + "path": req.path, + "version": version, + "container": container, + "object": obj, + }), + cfg.CONF.metering_topic, + cfg.CONF.metering_secret, + cfg.CONF.counter_source) + + +def filter_factory(global_conf, **local_conf): + conf = global_conf.copy() + conf.update(local_conf) + + def ceilometer_filter(app): + return CeilometerMiddleware(app, conf) + return ceilometer_filter diff --git a/doc/source/install.rst b/doc/source/install.rst index 77d6e5cf1..ee0306eba 100644 --- a/doc/source/install.rst +++ b/doc/source/install.rst @@ -98,6 +98,15 @@ Installing the Collector --user_id $CEILOMETER_USER \ --role_id 462fa46c13fd4798a95a3bfbe27b5e54 + You'll also need to add the Ceilometer middleware to Swift to account for + incoming and outgoing traffic, adding this lines to + ``/etc/swift/proxy-server.conf``:: + + [filter:ceilometer] + use = egg:ceilometer#swift + + And adding ``ceilometer`` in the ``pipeline`` of that same file. + 4. Install MongoDB. Follow the instructions to install the MongoDB_ package for your diff --git a/doc/source/measurements.rst b/doc/source/measurements.rst index 445ec6a4e..7b71d4e83 100644 --- a/doc/source/measurements.rst +++ b/doc/source/measurements.rst @@ -109,13 +109,15 @@ volume.size Gauge GB vol ID Size of volume Object Storage (Swift) ====================== -========================== ========== ========== ======== ================================================== -Name Type Volume Resource Note -========================== ========== ========== ======== ================================================== -storage.objects Gauge objects store ID Number of objects -storage.objects.size Gauge bytes store ID Total size of stored objects -storage.objects.containers Gauge containers store ID Number of containers -========================== ========== ========== ======== ================================================== +========================== ========== ========== ======== ================================================== +Name Type Volume Resource Note +========================== ========== ========== ======== ================================================== +storage.objects Gauge objects store ID Number of objects +storage.objects.size Gauge bytes store ID Total size of stored objects +storage.objects.containers Gauge containers store ID Number of containers +storage.objects.incoming.bytes Delta bytes store ID Number of incoming bytes +storage.objects.outgoing.bytes Delta bytes store ID Number of outgoing bytes +============================== ========== ========== ======== ================================================== Dynamically retrieving the Meters via ceilometer client ======================================================= diff --git a/setup.py b/setup.py index fa345f824..7a79cb96b 100755 --- a/setup.py +++ b/setup.py @@ -132,5 +132,8 @@ setuptools.setup( [ceilometer.compute.virt] libvirt = ceilometer.compute.virt.libvirt.inspector:LibvirtInspector + + [paste.filter_factory] + swift=ceilometer.objectstore.swift_middleware:filter_factory """), ) diff --git a/tests/objectstore/test_swift_middleware.py b/tests/objectstore/test_swift_middleware.py new file mode 100644 index 000000000..7436e4174 --- /dev/null +++ b/tests/objectstore/test_swift_middleware.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +# +# Copyright © 2012 eNovance +# +# Author: Julien Danjou +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import cStringIO as StringIO +from webob import Request + +from ceilometer.tests import base +from ceilometer.objectstore import swift_middleware +from ceilometer.openstack.common import rpc + + +class FakeApp(object): + def __init__(self, body=['This string is 28 bytes long']): + self.body = body + + def __call__(self, env, start_response): + start_response('200 OK', [('Content-Type', 'text/plain'), + ('Content-Length', str(sum(map(len, self.body))))]) + while env['wsgi.input'].read(5): + pass + return self.body + + +class TestSwiftMiddleware(base.TestCase): + + def setUp(self): + super(TestSwiftMiddleware, self).setUp() + self.notifications = [] + self.stubs.Set(rpc, 'cast', self._faux_notify) + + @staticmethod + def start_response(*args): + pass + + def _faux_notify(self, context, topic, msg): + self.notifications.append((topic, msg)) + + def test_get(self): + app = swift_middleware.CeilometerMiddleware(FakeApp(), {}) + req = Request.blank('/1.0/account/container/obj', + environ={'REQUEST_METHOD': 'GET'}) + resp = app(req.environ, self.start_response) + self.assertEqual(list(resp), ["This string is 28 bytes long"]) + self.assertEqual(len(self.notifications), 2) + data = self.notifications[0][1]['args']['data'] + self.assertEqual(data['counter_volume'], 28) + self.assertEqual(data['resource_metadata']['version'], '1.0') + self.assertEqual(data['resource_metadata']['container'], 'container') + self.assertEqual(data['resource_metadata']['object'], 'obj') + + def test_put(self): + app = swift_middleware.CeilometerMiddleware(FakeApp(body=['']), {}) + req = Request.blank('/1.0/account/container/obj', + environ={'REQUEST_METHOD': 'GET', + 'wsgi.input': + StringIO.StringIO('some stuff')}) + resp = list(app(req.environ, self.start_response)) + self.assertEqual(len(self.notifications), 2) + data = self.notifications[0][1]['args']['data'] + self.assertEqual(data['counter_volume'], 10) + self.assertEqual(data['resource_metadata']['version'], '1.0') + self.assertEqual(data['resource_metadata']['container'], 'container') + self.assertEqual(data['resource_metadata']['object'], 'obj') + + def test_post(self): + app = swift_middleware.CeilometerMiddleware(FakeApp(body=['']), {}) + req = Request.blank('/1.0/account/container/obj', + environ={'REQUEST_METHOD': 'POST', + 'wsgi.input': + StringIO.StringIO('some other stuff')}) + resp = list(app(req.environ, self.start_response)) + self.assertEqual(len(self.notifications), 2) + data = self.notifications[0][1]['args']['data'] + self.assertEqual(data['counter_volume'], 16) + self.assertEqual(data['resource_metadata']['version'], '1.0') + self.assertEqual(data['resource_metadata']['container'], 'container') + self.assertEqual(data['resource_metadata']['object'], 'obj') + + def test_get_container(self): + app = swift_middleware.CeilometerMiddleware(FakeApp(), {}) + req = Request.blank('/1.0/account/container', + environ={'REQUEST_METHOD': 'GET'}) + resp = list(app(req.environ, self.start_response)) + self.assertEqual(len(self.notifications), 2) + data = self.notifications[0][1]['args']['data'] + self.assertEqual(data['counter_volume'], 28) + self.assertEqual(data['resource_metadata']['version'], '1.0') + self.assertEqual(data['resource_metadata']['container'], 'container') + self.assertEqual(data['resource_metadata']['object'], None) diff --git a/tools/test-requires b/tools/test-requires index 2935d2376..ad7ffd13f 100644 --- a/tools/test-requires +++ b/tools/test-requires @@ -22,3 +22,7 @@ setuptools-git>=0.4 # very soon. hg+https://bitbucket.org/cdevienne/wsme pecan +# We should use swift>1.7.5, but it's not yet available +swift +# Swift dep that is not necessary if we depend on swift>1.7.5 +netifaces diff --git a/tools/test-requires-folsom b/tools/test-requires-folsom index 202e24722..82dd0531f 100644 --- a/tools/test-requires-folsom +++ b/tools/test-requires-folsom @@ -20,4 +20,8 @@ setuptools-git>=0.4 # checkout on bitbucket. I hope to have that resolved # very soon. hg+https://bitbucket.org/cdevienne/wsme -pecan \ No newline at end of file +pecan +# We should use swift>1.7.5, but it's not yet available +swift +# Swift dep that is not necessary if we depend on swift>1.7.5 +netifaces