add an option to disable non-metric meters

a large chunk of meters don't measure anything and are really
healthcheck events. these should be stored as events and not
samples. this patch adds an option to disable false meters and
warn if not disabled.

Change-Id: I654f657dd36967ad3ec99bbe06209e7745432e12

ceilometer/agent/plugin_base.py

@@ -187,6 +187,16 @@ class NotificationBase(PluginBase):
                 p(list(self.process_notification(notification)))
 
 
+class NonMetricNotificationBase(object):
+    """Use to mark non-measurement meters
+
+    There are a number of historical non-measurement meters that should really
+    be captured as events. This common base allows us to disable these invalid
+    meters.
+    """
+    pass
+
+
 class ExtensionLoadError(Exception):
     """Error of loading pollster plugin.
 

ceilometer/compute/notifications/instance.py

@@ -20,6 +20,7 @@ import abc
 
 import six
 
+from ceilometer.agent import plugin_base
 from ceilometer.compute import notifications
 from ceilometer.compute import util
 from ceilometer import sample
@@ -47,7 +48,8 @@ class UserMetadataAwareInstanceNotificationBase(
         """Derive sample from notification payload."""
 
 
-class InstanceScheduled(UserMetadataAwareInstanceNotificationBase):
+class InstanceScheduled(UserMetadataAwareInstanceNotificationBase,
+                        plugin_base.NonMetricNotificationBase):
     event_types = ['scheduler.run_instance.scheduled']
 
     def get_instance_properties(self, message):
@@ -73,7 +75,8 @@ class ComputeInstanceNotificationBase(
     event_types = ['compute.instance.*']
 
 
-class Instance(ComputeInstanceNotificationBase):
+class Instance(ComputeInstanceNotificationBase,
+               plugin_base.NonMetricNotificationBase):
     def get_sample(self, message):
         yield sample.Sample.from_notification(
             name='instance',
@@ -138,7 +141,8 @@ class EphemeralDiskSize(ComputeInstanceNotificationBase):
             message=message)
 
 
-class InstanceFlavor(ComputeInstanceNotificationBase):
+class InstanceFlavor(ComputeInstanceNotificationBase,
+                     plugin_base.NonMetricNotificationBase):
     def get_sample(self, message):
         instance_type = message.get('payload', {}).get('instance_type')
         if instance_type:

ceilometer/data_processing/notifications.py

@@ -30,7 +30,8 @@ cfg.CONF.register_opts(OPTS)
 SERVICE = 'sahara'
 
 
-class DataProcessing(plugin_base.NotificationBase):
+class DataProcessing(plugin_base.NotificationBase,
+                     plugin_base.NonMetricNotificationBase):
 
     resource_name = '%s.cluster' % SERVICE
 

ceilometer/identity/notifications.py

@@ -29,7 +29,8 @@ cfg.CONF.register_opts(OPTS)
 SERVICE = 'identity'
 
 
-class _Base(plugin_base.NotificationBase):
+class _Base(plugin_base.NotificationBase,
+            plugin_base.NonMetricNotificationBase):
     """Convert identity notification into Samples."""
 
     resource_type = None

ceilometer/image/notifications.py

@@ -55,7 +55,7 @@ class ImageCRUDBase(ImageBase):
     ]
 
 
-class ImageCRUD(ImageCRUDBase):
+class ImageCRUD(ImageCRUDBase, plugin_base.NonMetricNotificationBase):
     def process_notification(self, message):
         yield sample.Sample.from_notification(
             name=message['event_type'],
@@ -68,7 +68,7 @@ class ImageCRUD(ImageCRUDBase):
             message=message)
 
 
-class Image(ImageCRUDBase):
+class Image(ImageCRUDBase, plugin_base.NonMetricNotificationBase):
     def process_notification(self, message):
         yield sample.Sample.from_notification(
             name='image',

ceilometer/key_value_storage/notifications.py

@@ -41,7 +41,7 @@ class _Base(plugin_base.NotificationBase):
                 for topic in conf.notification_topics]
 
 
-class Table(_Base):
+class Table(_Base, plugin_base.NonMetricNotificationBase):
 
     event_types = [
         'magnetodb.table.create.end',

ceilometer/network/notifications.py

@@ -112,7 +112,7 @@ class NetworkNotificationBase(plugin_base.NotificationBase):
                     message=resource_message)
 
 
-class Network(NetworkNotificationBase):
+class Network(NetworkNotificationBase, plugin_base.NonMetricNotificationBase):
     """Listen for Neutron network notifications.
 
     Handle network.{create.end|update.*|exists} notifications from neutron.
@@ -120,7 +120,7 @@ class Network(NetworkNotificationBase):
     resource_name = 'network'
 
 
-class Subnet(NetworkNotificationBase):
+class Subnet(NetworkNotificationBase, plugin_base.NonMetricNotificationBase):
     """Listen for Neutron notifications.
 
     Handle subnet.{create.end|update.*|exists} notifications from neutron.
@@ -128,7 +128,7 @@ class Subnet(NetworkNotificationBase):
     resource_name = 'subnet'
 
 
-class Port(NetworkNotificationBase):
+class Port(NetworkNotificationBase, plugin_base.NonMetricNotificationBase):
     """Listen for Neutron notifications.
 
     Handle port.{create.end|update.*|exists} notifications from neutron.
@@ -136,7 +136,7 @@ class Port(NetworkNotificationBase):
     resource_name = 'port'
 
 
-class Router(NetworkNotificationBase):
+class Router(NetworkNotificationBase, plugin_base.NonMetricNotificationBase):
     """Listen for Neutron notifications.
 
     Handle router.{create.end|update.*|exists} notifications from neutron.
@@ -144,7 +144,8 @@ class Router(NetworkNotificationBase):
     resource_name = 'router'
 
 
-class FloatingIP(NetworkNotificationBase):
+class FloatingIP(NetworkNotificationBase,
+                 plugin_base.NonMetricNotificationBase):
     """Listen for Neutron notifications.
 
     Handle floatingip.{create.end|update.*|exists} notifications from neutron.
@@ -173,7 +174,7 @@ class Bandwidth(NetworkNotificationBase):
             message=message)
 
 
-class Pool(NetworkNotificationBase):
+class Pool(NetworkNotificationBase, plugin_base.NonMetricNotificationBase):
     """Listen for Neutron notifications.
 
     Handle pool.{create.end|update.*|exists} notifications from neutron.
@@ -182,7 +183,7 @@ class Pool(NetworkNotificationBase):
     counter_name = 'network.services.lb.pool'
 
 
-class Vip(NetworkNotificationBase):
+class Vip(NetworkNotificationBase, plugin_base.NonMetricNotificationBase):
     """Listen for Neutron notifications.
 
     Handle vip.{create.end|update.*|exists} notifications from neutron.
@@ -191,7 +192,7 @@ class Vip(NetworkNotificationBase):
     counter_name = 'network.services.lb.vip'
 
 
-class Member(NetworkNotificationBase):
+class Member(NetworkNotificationBase, plugin_base.NonMetricNotificationBase):
     """Listen for Neutron notifications.
 
     Handle member.{create.end|update.*|exists} notifications from neutron.
@@ -200,7 +201,8 @@ class Member(NetworkNotificationBase):
     counter_name = 'network.services.lb.member'
 
 
-class HealthMonitor(NetworkNotificationBase):
+class HealthMonitor(NetworkNotificationBase,
+                    plugin_base.NonMetricNotificationBase):
     """Listen for Neutron notifications.
 
     Handle health_monitor.{create.end|update.*|exists} notifications
@@ -210,7 +212,7 @@ class HealthMonitor(NetworkNotificationBase):
     counter_name = 'network.services.lb.health_monitor'
 
 
-class Firewall(NetworkNotificationBase):
+class Firewall(NetworkNotificationBase, plugin_base.NonMetricNotificationBase):
     """Listen for Neutron notifications.
 
     Handle firewall.{create.end|update.*|exists} notifications from neutron.
@@ -219,7 +221,8 @@ class Firewall(NetworkNotificationBase):
     counter_name = 'network.services.firewall'
 
 
-class FirewallPolicy(NetworkNotificationBase):
+class FirewallPolicy(NetworkNotificationBase,
+                     plugin_base.NonMetricNotificationBase):
     """Listen for Neutron notifications.
 
     Handle firewall_policy.{create.end|update.*|exists} notifications
@@ -229,7 +232,8 @@ class FirewallPolicy(NetworkNotificationBase):
     counter_name = 'network.services.firewall.policy'
 
 
-class FirewallRule(NetworkNotificationBase):
+class FirewallRule(NetworkNotificationBase,
+                   plugin_base.NonMetricNotificationBase):
     """Listen for Neutron notifications.
 
     Handle firewall_rule.{create.end|update.*|exists} notifications
@@ -239,7 +243,8 @@ class FirewallRule(NetworkNotificationBase):
     counter_name = 'network.services.firewall.rule'
 
 
-class VPNService(NetworkNotificationBase):
+class VPNService(NetworkNotificationBase,
+                 plugin_base.NonMetricNotificationBase):
     """Listen for Neutron notifications.
 
     Handle vpnservice.{create.end|update.*|exists} notifications from neutron.
@@ -248,7 +253,8 @@ class VPNService(NetworkNotificationBase):
     counter_name = 'network.services.vpn'
 
 
-class IPSecPolicy(NetworkNotificationBase):
+class IPSecPolicy(NetworkNotificationBase,
+                  plugin_base.NonMetricNotificationBase):
     """Listen for Neutron notifications.
 
     Handle pool.{create.end|update.*|exists} notifications from neutron.
@@ -257,7 +263,8 @@ class IPSecPolicy(NetworkNotificationBase):
     counter_name = 'network.services.vpn.ipsecpolicy'
 
 
-class IKEPolicy(NetworkNotificationBase):
+class IKEPolicy(NetworkNotificationBase,
+                plugin_base.NonMetricNotificationBase):
     """Listen for Neutron notifications.
 
     Handle ikepolicy.{create.end|update.*|exists} notifications from neutron.
@@ -266,7 +273,8 @@ class IKEPolicy(NetworkNotificationBase):
     counter_name = 'network.services.vpn.ikepolicy'
 
 
-class IPSecSiteConnection(NetworkNotificationBase):
+class IPSecSiteConnection(NetworkNotificationBase,
+                          plugin_base.NonMetricNotificationBase):
     """Listen for Neutron notifications.
 
     Handle ipsec_site_connection.{create.end|update.*|exists}

ceilometer/notification.py

@@ -18,9 +18,10 @@ from oslo_config import cfg
 from oslo_context import context
 from stevedore import extension
 
+from ceilometer.agent import plugin_base as base
 from ceilometer import coordination
 from ceilometer.event import endpoint as event_endpoint
-from ceilometer.i18n import _
+from ceilometer.i18n import _, _LW
 from ceilometer import messaging
 from ceilometer.openstack.common import log
 from ceilometer.openstack.common import service as os_service
@@ -40,6 +41,14 @@ OPTS = [
                 deprecated_group='collector',
                 default=False,
                 help='Save event details.'),
+    cfg.BoolOpt('disable_non_metric_meters',
+                default=False,
+                help='WARNING: Ceilometer historically offered the ability to '
+                     'store events as meters. This usage is NOT advised as it '
+                     'can flood the metering database and cause performance '
+                     'degradation. This option disables the collection of '
+                     'non-metric meters and will be the default behavior in '
+                     'Liberty.'),
     cfg.BoolOpt('workload_partitioning',
                 default=False,
                 help='Enable workload partitioning, allowing multiple '
@@ -141,6 +150,10 @@ class NotificationService(os_service.Service):
             self.tg.add_timer(cfg.CONF.coordination.check_watchers,
                               self.partition_coordinator.run_watchers)
 
+        if not cfg.CONF.notification.disable_non_metric_meters:
+            LOG.warning(_LW('Non-metric meters may be collected. It is highly '
+                            'advisable to disable these meters using '
+                            'ceilometer.conf or the pipeline.yaml'))
         # Add a dummy thread to have wait() working
         self.tg.add_timer(604800, lambda: None)
 
@@ -160,6 +173,9 @@ class NotificationService(os_service.Service):
         targets = []
         for ext in notification_manager:
             handler = ext.obj
+            if (cfg.CONF.notification.disable_non_metric_meters and
+                    isinstance(handler, base.NonMetricNotificationBase)):
+                continue
             LOG.debug(_('Event types from %(name)s: %(type)s'
                         ' (ack_on_error=%(error)s)') %
                       {'name': ext.name,

ceilometer/objectstore/notifications.py

@@ -44,7 +44,7 @@ class _Base(plugin_base.NotificationBase):
                 for topic in conf.notification_topics]
 
 
-class SwiftWsgiMiddleware(_Base):
+class SwiftWsgiMiddleware(_Base, plugin_base.NonMetricNotificationBase):
 
     @property
     def event_types(self):

ceilometer/orchestration/notifications.py

@@ -30,7 +30,8 @@ cfg.CONF.register_opts(OPTS)
 SERVICE = 'orchestration'
 
 
-class StackCRUD(plugin_base.NotificationBase):
+class StackCRUD(plugin_base.NotificationBase,
+                plugin_base.NonMetricNotificationBase):
 
     resource_name = '%s.stack' % SERVICE
 

ceilometer/tests/test_notification.py

@@ -91,6 +91,8 @@ class TestNotification(tests_base.BaseTestCase):
         self.CONF = self.useFixture(fixture_config.Config()).conf
         self.CONF.set_override("connection", "log://", group='database')
         self.CONF.set_override("store_events", False, group="notification")
+        self.CONF.set_override("disable_non_metric_meters", False,
+                               group="notification")
         self.setup_messaging(self.CONF)
         self.srv = notification.NotificationService()
 
@@ -187,6 +189,8 @@ class BaseRealNotification(tests_base.BaseTestCase):
         self.CONF.set_override("pipeline_cfg_file", pipeline_cfg_file)
 
         self.CONF.set_override("store_events", True, group="notification")
+        self.CONF.set_override("disable_non_metric_meters", False,
+                               group="notification")
         ev_pipeline = yaml.dump({
             'sources': [{
                 'name': 'test_event',
@@ -257,6 +261,16 @@ class TestRealNotification(BaseRealNotification):
         self.srv.stop()
         self.assertEqual(self.expected_events, len(self.publisher.events))
 
+    @mock.patch('ceilometer.publisher.test.TestPublisher')
+    def test_notification_disable_non_metrics(self, fake_publisher_cls):
+        self.CONF.set_override("disable_non_metric_meters", True,
+                               group="notification")
+        # instance is a not a metric. we should only get back memory
+        self.expected_samples = 1
+        fake_publisher_cls.return_value = self.publisher
+        self._check_notification_service()
+        self.assertEqual('memory', self.publisher.samples[0].name)
+
     @mock.patch('ceilometer.coordination.PartitionCoordinator')
     @mock.patch('ceilometer.publisher.test.TestPublisher')
     def test_ha_configured_agent_coord_disabled(self, fake_publisher_cls,

ceilometer/volume/notifications.py

@@ -61,7 +61,7 @@ class VolumeCRUDBase(VolumeBase):
     ]
 
 
-class VolumeCRUD(VolumeCRUDBase):
+class VolumeCRUD(VolumeCRUDBase, plugin_base.NonMetricNotificationBase):
     def process_notification(self, message):
         yield sample.Sample.from_notification(
             name=message['event_type'],
@@ -74,7 +74,7 @@ class VolumeCRUD(VolumeCRUDBase):
             message=message)
 
 
-class Volume(VolumeCRUDBase):
+class Volume(VolumeCRUDBase, plugin_base.NonMetricNotificationBase):
     def process_notification(self, message):
         yield sample.Sample.from_notification(
             name='volume',
@@ -111,7 +111,7 @@ class SnapshotCRUDBase(VolumeBase):
     ]
 
 
-class SnapshotCRUD(SnapshotCRUDBase):
+class SnapshotCRUD(SnapshotCRUDBase, plugin_base.NonMetricNotificationBase):
     def process_notification(self, message):
         yield sample.Sample.from_notification(
             name=message['event_type'],
@@ -124,7 +124,7 @@ class SnapshotCRUD(SnapshotCRUDBase):
             message=message)
 
 
-class Snapshot(SnapshotCRUDBase):
+class Snapshot(SnapshotCRUDBase, plugin_base.NonMetricNotificationBase):
     def process_notification(self, message):
         yield sample.Sample.from_notification(
             name='snapshot',

doc/source/measurements.rst

@@ -523,6 +523,12 @@ Using notification plugin profiler/notifications.py.
 Creating New Meters
 ===================
 
+.. note::
+
+  Meters in Ceilometer should represent a standard of measurement. If tracking
+  the general state of a resource, the datapoint should be modelled as an Event
+  rather than a Meter.
+
 Naming convention
 -----------------