Implement object store metering

Implements these pollsters for swift :
  - storage.objects (per-tenant number of objects)
  - storage.objects.size (per-tenant total size of stored objects)
  - storage.objects.containers (per-tenant number of containers)

Implements: blueprint pollster-swift
Fixes: bug #1004450
Change-Id: Ie27abd0b5803043f06899f14b394f2a7ab691bca

ceilometer/objectstore/swift.py

@@ -0,0 +1,104 @@
+# -*- encoding: utf-8 -*-
+#
+# Copyright © 2012 eNovance
+#
+# Author: Guillaume Pernot <gpernot@praksys.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+"""Common code for working with object stores
+"""
+
+from __future__ import absolute_import
+
+import abc
+
+from keystoneclient.v2_0 import client as ksclient
+from swiftclient import client as swift
+
+from ceilometer import plugin
+from ceilometer import counter
+from ceilometer.openstack.common import cfg
+from ceilometer.openstack.common import timeutils
+from ceilometer.openstack.common import log
+
+LOG = log.getLogger(__name__)
+
+OPTS = [
+    cfg.StrOpt('reseller_prefix',
+               default='AUTH_',
+               help="Swift reseller prefix. Must be on par with "\
+                   "reseller_prefix in proxy-server.conf."),
+]
+
+cfg.CONF.register_opts(OPTS)
+
+
+class _Base(plugin.PollsterBase):
+
+    __metaclass__ = abc.ABCMeta
+
+    @staticmethod
+    @abc.abstractmethod
+    def iter_accounts():
+        """Iterate over all accounts, yielding (tenant_id, stats) tuples."""
+
+    def get_counters(self, manager, context):
+        for tenant, account in self.iter_accounts():
+            yield counter.Counter(
+                name='storage.objects',
+                type=counter.TYPE_GAUGE,
+                volume=int(account['x-account-object-count']),
+                user_id=None,
+                project_id=tenant,
+                resource_id=tenant,
+                timestamp=timeutils.isotime(),
+                resource_metadata=None,
+            )
+            yield counter.Counter(
+                name='storage.objects.size',
+                type=counter.TYPE_GAUGE,
+                volume=int(account['x-account-bytes-used']),
+                user_id=None,
+                project_id=tenant,
+                resource_id=tenant,
+                timestamp=timeutils.isotime(),
+                resource_metadata=None,
+            )
+            yield counter.Counter(
+                name='storage.objects.containers',
+                type=counter.TYPE_GAUGE,
+                volume=int(account['x-account-container-count']),
+                user_id=None,
+                project_id=tenant,
+                resource_id=tenant,
+                timestamp=timeutils.isotime(),
+                resource_metadata=None,
+            )
+
+
+class SwiftPollster(_Base):
+    """Iterate over all accounts, using keystone.
+    """
+
+    @staticmethod
+    def iter_accounts():
+        ks = ksclient.Client(username=cfg.CONF.os_username,
+                             password=cfg.CONF.os_password,
+                             tenant_name=cfg.CONF.os_tenant_name,
+                             auth_url=cfg.CONF.os_auth_url)
+        endpoint = ks.service_catalog.url_for(service_type='object-store',
+                                              endpoint_type='adminURL')
+        base_url = '%s/v1/%s' % (endpoint, cfg.CONF.reseller_prefix)
+        for t in ks.tenants.list():
+            yield (t.id, swift.head_account('%s%s' % (base_url, t.id),
+                                            ks.auth_token))

doc/source/configuration.rst

@@ -50,6 +50,7 @@ metering_api_port                8777                                  The port
 disabled_central_pollsters                                             List of central pollsters to skip loading
 disabled_compute_pollsters                                             List of compute pollsters to skip loading
 disabled_notification_listeners                                        List of notification listeners to skip loading
+reseller_prefix                  AUTH\_                                Prefix used by swift for reseller token
 ===============================  ====================================  ==============================================================
 
 SQL Alchemy

doc/source/install.rst

@@ -83,23 +83,38 @@ Installing the Collector
    to ``rabbit`` or ``qpid`` in ``glance-api.conf`` and restarting the
    service.
 
-3. Install MongoDB.
+3. In order to retrieve object store statistics, ceilometer needs an
+   access to swift with ``ResellerAdmin`` role. You should give this
+   role to your ``os_username`` user for tenant ``os_tenant_name``::
+
+   $ keystone role-create --name=ResellerAdmin
+   +----------+----------------------------------+
+   | Property |              Value               |
+   +----------+----------------------------------+
+   |    id    | 462fa46c13fd4798a95a3bfbe27b5e54 |
+   |   name   |          ResellerAdmin           |
+   +----------+----------------------------------+
+   $ keystone user-role-add --tenant_id $SERVICE_TENANT \
+                            --user_id $CEILOMETER_USER \
+                            --role_id 462fa46c13fd4798a95a3bfbe27b5e54
+
+4. Install MongoDB.
 
    Follow the instructions to install the MongoDB_ package for your
    operating system, then start the service.
 
-4. Clone the ceilometer git repository to the management server::
+5. Clone the ceilometer git repository to the management server::
 
    $ cd /opt/stack
    $ git clone https://github.com/openstack/ceilometer.git
 
-5. As a user with ``root`` permissions or ``sudo`` privileges, run the
+6. As a user with ``root`` permissions or ``sudo`` privileges, run the
    ceilometer installer::
 
    $ cd ceilometer
    $ sudo python setup.py install
 
-6. Configure ceilometer.
+7. Configure ceilometer.
 
    Ceilometer needs to know about some of the nova configuration
    options, so the simplest way to start is copying
@@ -112,7 +127,7 @@ Installing the Collector
    Refer to :doc:`configuration` for details about any other options
    you might want to modify before starting the service.
 
-7. Start the collector.
+8. Start the collector.
 
    ::
 

doc/source/measurements.rst

@@ -106,6 +106,17 @@ volume                    Gauge             1  vol ID    Duration of volune
 volume.size               Gauge            GB  vol ID    Size of volume
 ========================  ==========  =======  ========  =======================================================
 
+Object Storage (Swift)
+======================
+
+==========================  ==========  ==========  ========  ==================================================
+Name                        Type        Volume      Resource  Note
+==========================  ==========  ==========  ========  ==================================================
+storage.objects             Gauge          objects  store ID  Number of objects
+storage.objects.size        Gauge            bytes  store ID  Total size of stored objects
+storage.objects.containers  Gauge       containers  store ID  Number of containers
+==========================  ==========  ==========  ========  ==================================================
+
 Naming convention
 =================
 If you plan on adding meters, please follow the convention bellow:

setup.py

@@ -119,6 +119,7 @@ setuptools.setup(
     network_floatingip = ceilometer.network.floatingip:FloatingIPPollster
     image = ceilometer.image.glance:ImagePollster
     image_size = ceilometer.image.glance:ImageSizePollster
+    objectstore = ceilometer.objectstore.swift:SwiftPollster
 
     [ceilometer.storage]
     log = ceilometer.storage.impl_log:LogStorage

tests/objectstore/test_swift.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python
+# -*- encoding: utf-8 -*-
+#
+# Copyright © 2012 eNovance <licensing@enovance.com>
+#
+# Author: Guillaume Pernot <gpernot@praksys.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from ceilometer.objectstore import swift
+from ceilometer.tests import base
+
+ACCOUNTS = [('tenant-000', {'x-account-object-count': 12,
+                            'x-account-bytes-used': 321321321,
+                            'x-account-container-count': 7,
+                            }),
+            ('tenant-001', {'x-account-object-count': 34,
+                            'x-account-bytes-used': 9898989898,
+                            'x-account-container-count': 17,
+                            })]
+
+
+class TestSwiftPollster(base.TestCase):
+
+    @staticmethod
+    def fake_iter_accounts(_dummy):
+        for i in ACCOUNTS:
+            yield i
+
+    def setUp(self):
+        super(TestSwiftPollster, self).setUp()
+        self.pollster = swift.SwiftPollster()
+        self.stubs.Set(swift.SwiftPollster, 'iter_accounts',
+                       self.fake_iter_accounts)
+
+    def test_objectstore_metering(self):
+        counters = list(self.pollster.get_counters(None, None))
+        self.assertEqual(len(counters), 6)

tools/pip-requires

@@ -14,3 +14,4 @@ stevedore>=0.6
 python-glanceclient
 python-novaclient>=2.6.10
 python-keystoneclient>=0.2,<0.3
+python-swiftclient