openstack/aodh
Code Issues Proposed changes

Add support for Swift incoming/outgoing trafic metering

Browse Source 
This adds a middleware for Swift that meters incoming and outgoing bytes.

This is part of blueprint pollster-swift.

Change-Id: I94f330ee4cf5df8a743c77fcfae9efd505568060
Signed-off-by: Julien Danjou <julien@danjou.info>
This commit is contained in:
Julien Danjou 2012-12-11 16:14:42 +01:00
parent ade5b235b0
commit 6200b9d1ed
7 changed files with 268 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

133
ceilometer/objectstore/swift_middleware.py Normal file
View File

@ -0,0 +1,133 @@
#!/usr/bin/env python
# -*- encoding: utf-8 -*-
#
# Copyright © 2012 eNovance <licensing@enovance.com>
#
# Author: Julien Danjou <julien@danjou.info>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import absolute_import
from ceilometer import publish
from ceilometer import counter
from ceilometer.openstack.common import cfg
from ceilometer.openstack.common import context
from ceilometer.openstack.common import timeutils
from swift.common.swob import Request
from swift.common.utils import split_path
try:
# Swift > 1.7.5
from swift.common.utils import InputProxy
except ImportError:
# Swift <= 1.7.5
from swift.common.middleware.proxy_logging import InputProxy
class CeilometerMiddleware(object):
"""
Ceilometer middleware used for counting requests.
"""
def __init__(self, app, conf):
self.app = app
cfg.CONF([], project='ceilometer')
def __call__(self, env, start_response):
start_response_args = [None]
input_proxy = InputProxy(env['wsgi.input'])
env['wsgi.input'] = input_proxy
def my_start_response(status, headers, exc_info=None):
start_response_args[0] = (status, list(headers), exc_info)
def iter_response(iterable):
if start_response_args[0]:
start_response(*start_response_args[0])
bytes_sent = 0
try:
for chunk in iterable:
if chunk:
bytes_sent += len(chunk)
yield chunk
finally:
self.publish_counter(env,
input_proxy.bytes_received,
bytes_sent)
try:
iterable = self.app(env, my_start_response)
except Exception:
self.publish_counter(env, input_proxy.bytes_received, 0)
raise
else:
return iter_response(iterable)
@staticmethod
def publish_counter(env, bytes_received, bytes_sent):
req = Request(env)
version, account, container, obj = split_path(req.path, 1, 4, True)
now = timeutils.utcnow().isoformat()
if bytes_received:
publish.publish_counter(context.get_admin_context(),
counter.Counter(
name='storage.objects.incoming.bytes',
type='delta',
volume=bytes_received,
user_id=env.get('HTTP_X_USER_ID'),
project_id=env.get('HTTP_X_TENANT_ID'),
resource_id=account.partition(
'AUTH_')[2],
timestamp=now,
resource_metadata={
"path": req.path,
"version": version,
"container": container,
"object": obj,
}),
cfg.CONF.metering_topic,
cfg.CONF.metering_secret,
cfg.CONF.counter_source)
if bytes_sent:
publish.publish_counter(context.get_admin_context(),
counter.Counter(
name='storage.objects.outgoing.bytes',
type='delta',
volume=bytes_sent,
user_id=env.get('HTTP_X_USER_ID'),
project_id=env.get('HTTP_X_TENANT_ID'),
resource_id=account.partition(
'AUTH_')[2],
timestamp=now,
resource_metadata={
"path": req.path,
"version": version,
"container": container,
"object": obj,
}),
cfg.CONF.metering_topic,
cfg.CONF.metering_secret,
cfg.CONF.counter_source)
def filter_factory(global_conf, **local_conf):
conf = global_conf.copy()
conf.update(local_conf)
def ceilometer_filter(app):
return CeilometerMiddleware(app, conf)
return ceilometer_filter

9
doc/source/install.rst
View File

@ -98,6 +98,15 @@ Installing the Collector
--user_id $CEILOMETER_USER \
--role_id 462fa46c13fd4798a95a3bfbe27b5e54
You'll also need to add the Ceilometer middleware to Swift to account for
incoming and outgoing traffic, adding this lines to
``/etc/swift/proxy-server.conf``::
[filter:ceilometer]
use = egg:ceilometer#swift
And adding ``ceilometer`` in the ``pipeline`` of that same file.
4. Install MongoDB.
Follow the instructions to install the MongoDB_ package for your

16
doc/source/measurements.rst
View File

@ -109,13 +109,15 @@ volume.size Gauge GB vol ID Size of volume
Object Storage (Swift)
======================
========================== ========== ========== ======== ==================================================
Name Type Volume Resource Note
========================== ========== ========== ======== ==================================================
storage.objects Gauge objects store ID Number of objects
storage.objects.size Gauge bytes store ID Total size of stored objects
storage.objects.containers Gauge containers store ID Number of containers
========================== ========== ========== ======== ==================================================
========================== ========== ========== ======== ==================================================
Name Type Volume Resource Note
========================== ========== ========== ======== ==================================================
storage.objects Gauge objects store ID Number of objects
storage.objects.size Gauge bytes store ID Total size of stored objects
storage.objects.containers Gauge containers store ID Number of containers
storage.objects.incoming.bytes Delta bytes store ID Number of incoming bytes
storage.objects.outgoing.bytes Delta bytes store ID Number of outgoing bytes
============================== ========== ========== ======== ==================================================
Dynamically retrieving the Meters via ceilometer client
=======================================================

3
setup.py
View File

@ -132,5 +132,8 @@ setuptools.setup(
[ceilometer.compute.virt]
libvirt = ceilometer.compute.virt.libvirt.inspector:LibvirtInspector
[paste.filter_factory]
swift=ceilometer.objectstore.swift_middleware:filter_factory
"""),
)

105
tests/objectstore/test_swift_middleware.py Normal file
View File

@ -0,0 +1,105 @@
#!/usr/bin/env python
# -*- encoding: utf-8 -*-
#
# Copyright © 2012 eNovance <licensing@enovance.com>
#
# Author: Julien Danjou <julien@danjou.info>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import cStringIO as StringIO
from webob import Request
from ceilometer.tests import base
from ceilometer.objectstore import swift_middleware
from ceilometer.openstack.common import rpc
class FakeApp(object):
def __init__(self, body=['This string is 28 bytes long']):
self.body = body
def __call__(self, env, start_response):
start_response('200 OK', [('Content-Type', 'text/plain'),
('Content-Length', str(sum(map(len, self.body))))])
while env['wsgi.input'].read(5):
pass
return self.body
class TestSwiftMiddleware(base.TestCase):
def setUp(self):
super(TestSwiftMiddleware, self).setUp()
self.notifications = []
self.stubs.Set(rpc, 'cast', self._faux_notify)
@staticmethod
def start_response(*args):
pass
def _faux_notify(self, context, topic, msg):
self.notifications.append((topic, msg))
def test_get(self):
app = swift_middleware.CeilometerMiddleware(FakeApp(), {})
req = Request.blank('/1.0/account/container/obj',
environ={'REQUEST_METHOD': 'GET'})
resp = app(req.environ, self.start_response)
self.assertEqual(list(resp), ["This string is 28 bytes long"])
self.assertEqual(len(self.notifications), 2)
data = self.notifications[0][1]['args']['data']
self.assertEqual(data['counter_volume'], 28)
self.assertEqual(data['resource_metadata']['version'], '1.0')
self.assertEqual(data['resource_metadata']['container'], 'container')
self.assertEqual(data['resource_metadata']['object'], 'obj')
def test_put(self):
app = swift_middleware.CeilometerMiddleware(FakeApp(body=['']), {})
req = Request.blank('/1.0/account/container/obj',
environ={'REQUEST_METHOD': 'GET',
'wsgi.input':
StringIO.StringIO('some stuff')})
resp = list(app(req.environ, self.start_response))
self.assertEqual(len(self.notifications), 2)
data = self.notifications[0][1]['args']['data']
self.assertEqual(data['counter_volume'], 10)
self.assertEqual(data['resource_metadata']['version'], '1.0')
self.assertEqual(data['resource_metadata']['container'], 'container')
self.assertEqual(data['resource_metadata']['object'], 'obj')
def test_post(self):
app = swift_middleware.CeilometerMiddleware(FakeApp(body=['']), {})
req = Request.blank('/1.0/account/container/obj',
environ={'REQUEST_METHOD': 'POST',
'wsgi.input':
StringIO.StringIO('some other stuff')})
resp = list(app(req.environ, self.start_response))
self.assertEqual(len(self.notifications), 2)
data = self.notifications[0][1]['args']['data']
self.assertEqual(data['counter_volume'], 16)
self.assertEqual(data['resource_metadata']['version'], '1.0')
self.assertEqual(data['resource_metadata']['container'], 'container')
self.assertEqual(data['resource_metadata']['object'], 'obj')
def test_get_container(self):
app = swift_middleware.CeilometerMiddleware(FakeApp(), {})
req = Request.blank('/1.0/account/container',
environ={'REQUEST_METHOD': 'GET'})
resp = list(app(req.environ, self.start_response))
self.assertEqual(len(self.notifications), 2)
data = self.notifications[0][1]['args']['data']
self.assertEqual(data['counter_volume'], 28)
self.assertEqual(data['resource_metadata']['version'], '1.0')
self.assertEqual(data['resource_metadata']['container'], 'container')
self.assertEqual(data['resource_metadata']['object'], None)

4
tools/test-requires
View File

@ -22,3 +22,7 @@ setuptools-git>=0.4
# very soon.
hg+https://bitbucket.org/cdevienne/wsme
pecan
# We should use swift>1.7.5, but it's not yet available
swift
# Swift dep that is not necessary if we depend on swift>1.7.5
netifaces

6
tools/test-requires-folsom
View File

@ -20,4 +20,8 @@ setuptools-git>=0.4
# checkout on bitbucket. I hope to have that resolved
# very soon.
hg+https://bitbucket.org/cdevienne/wsme
pecan
pecan
# We should use swift>1.7.5, but it's not yet available
swift
# Swift dep that is not necessary if we depend on swift>1.7.5
netifaces