openstack/aodh
Code Issues Proposed changes

Add pollster data cache

Browse Source 
Add a dictionary as a data cache for the pollsters,
creating it outside of the loop where they are
invoked and passing it to them. Future changesets will
include changes to the pollsters to put data in the
cache.

blueprint one-meter-per-plugin

Change-Id: Ie65526dfe65a8880ad8683b62fae62f8e7f9e69b
Signed-off-by: Doug Hellmann <doug.hellmann@dreamhost.com>
This commit is contained in:
Doug Hellmann 2013-07-02 17:08:49 -04:00
parent 2f1378ebea
commit a10773e413
17 changed files with 52 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
ceilometer/central/manager.py
View File

@ -37,11 +37,15 @@ class PollingTask(agent.PollingTask):
with self.publish_context as publisher:
# TODO(yjiang5) passing counters into get_counters to avoid
# polling all counters one by one
cache = {}
for pollster in self.pollsters:
try:
LOG.info("Polling pollster %s", pollster.name)
publisher(list(pollster.obj.get_counters(
self.manager)))
counters = list(pollster.obj.get_counters(
self.manager,
cache,
))
publisher(counters)
except Exception as err:
LOG.warning('Continue after error from %s: %s',
pollster.name, err)

5
ceilometer/compute/manager.py
View File

@ -37,12 +37,15 @@ class PollingTask(agent.PollingTask):
if getattr(instance, 'OS-EXT-STS:vm_state', None) != 'error':
# TODO(yjiang5) passing counters to get_counters to avoid
# polling all counters one by one
cache = {}
for pollster in self.pollsters:
try:
LOG.info("Polling pollster %s", pollster.name)
publisher(list(pollster.obj.get_counters(
self.manager,
instance)))
cache,
instance,
)))
except Exception as err:
LOG.warning('Continue after error from %s: %s',
pollster.name, err)

6
ceilometer/compute/nova_notifier.py
View File

@ -55,12 +55,14 @@ class DeletedInstanceStatsGatherer(object):
self.mgr = extensions
self.inspector = inspector.get_hypervisor_inspector()
def _get_counters_from_plugin(self, ext, instance, *args, **kwds):
def _get_counters_from_plugin(self, ext, cache, instance, *args, **kwds):
"""Used with the extenaion manager map() method."""
return ext.obj.get_counters(self, instance)
return ext.obj.get_counters(self, cache, instance)
def __call__(self, instance):
cache = {}
counters = self.mgr.map(self._get_counters_from_plugin,
cache=cache,
instance=instance,
)
# counters is a list of lists, so flatten it before returning

6
ceilometer/compute/plugin.py
View File

@ -30,6 +30,10 @@ class ComputePollster(plugin.PollsterBase):
__metaclass__ = abc.ABCMeta
@abc.abstractmethod
def get_counters(self, manager, context):
def get_counters(self, manager, cache, instance):
"""Return a sequence of Counter instances from polling the resources.
:param manager: The service manager invoking the plugin
:param cache: A dictionary for passing data between plugins
:param instance: The instance to examine
"""

8
ceilometer/compute/pollsters.py
View File

@ -57,7 +57,7 @@ class InstancePollster(plugin.ComputePollster):
# variable. We don't need such format in future
return ['instance', 'instance:*']
def get_counters(self, manager, instance):
def get_counters(self, manager, cache, instance):
yield make_counter_from_instance(instance,
name='instance',
type=counter.TYPE_GAUGE,
@ -91,7 +91,7 @@ class DiskIOPollster(plugin.ComputePollster):
'disk.write.requests',
'disk.write.bytes']
def get_counters(self, manager, instance):
def get_counters(self, manager, cache, instance):
instance_name = _instance_name(instance)
try:
r_bytes = 0
@ -164,7 +164,7 @@ class CPUPollster(plugin.ComputePollster):
def get_counter_names():
return ['cpu', 'cpu_util']
def get_counters(self, manager, instance):
def get_counters(self, manager, cache, instance):
self.LOG.info('checking instance %s', instance.id)
instance_name = _instance_name(instance)
try:
@ -237,7 +237,7 @@ class NetPollster(plugin.ComputePollster):
'network.outgoing.bytes',
'network.outgoing.packets']
def get_counters(self, manager, instance):
def get_counters(self, manager, cache, instance):
instance_name = _instance_name(instance)
self.LOG.info('checking instance %s', instance.id)
try:

2
ceilometer/energy/kwapi.py
View File

@ -77,7 +77,7 @@ class KwapiPollster(_Base):
def get_counter_names():
return ['energy', 'power']
def get_counters(self, manager):
def get_counters(self, manager, cache):
"""Returns all counters."""
for probe in self.iter_probes(manager.keystone):
yield counter.Counter(

2
ceilometer/image/glance.py
View File

@ -99,7 +99,7 @@ class ImagePollster(_Base):
def get_counter_names():
return ['image', 'image.size']
def get_counters(self, manager):
def get_counters(self, manager, cache):
for image in self.iter_images(manager.keystone):
yield counter.Counter(
name='image',

2
ceilometer/network/floatingip.py
View File

@ -35,7 +35,7 @@ class FloatingIPPollster(plugin.CentralPollster):
def get_counter_names():
return ['ip.floating']
def get_counters(self, manager):
def get_counters(self, manager, cache):
nv = nova_client.Client()
for ip in nv.floating_ip_get_all():
self.LOG.info("FLOATING IP USAGE: %s" % ip.ip)

2
ceilometer/objectstore/swift.py
View File

@ -55,7 +55,7 @@ class _Base(plugin.PollsterBase):
def iter_accounts(ksclient):
"""Iterate over all accounts, yielding (tenant_id, stats) tuples."""
def get_counters(self, manager):
def get_counters(self, manager, cache):
for tenant, account in self.iter_accounts(manager.keystone):
yield counter.Counter(
name='storage.objects',

8
ceilometer/plugin.py
View File

@ -88,7 +88,13 @@ class PollsterBase(PluginBase):
"""Return a sequence of Counter names supported by the pollster."""
@abc.abstractmethod
def get_counters(self, manager, instance):
def get_counters(self, manager, cache):
"""Return a sequence of Counter instances from polling the resources.
:param manager: The service manager class invoking the plugin.
:param cache: A dictionary to allow pollsters to pass data
between themselves when recomputing it would be
expensive (e.g., asking another service for a
list of objects).
"""

2
nova_tests/test_notifier.py
View File

@ -77,7 +77,7 @@ class TestNovaNotifier(base.TestCase):
},
)
def get_counters(self, manager, instance):
def get_counters(self, manager, cache, instance):
self.instances.append((manager, instance))
return [self.test_data]

6
tests/agentbase.py
View File

@ -52,17 +52,17 @@ class TestPollster:
def get_counter_names(self):
return [self.test_data.name]
def get_counters(self, manager, instance=None):
def get_counters(self, manager, cache, instance=None):
self.counters.append((manager, instance))
return [self.test_data]
class TestPollsterException(TestPollster):
def get_counters(self, manager, instance=None):
def get_counters(self, manager, cache, instance=None):
# Put an instance parameter here so that it can be used
# by both central manager and compute manager
# In future, we possibly don't need such hack if we
# combin the get_counters() function again
# combine the get_counters() function again
self.counters.append((manager, instance))
raise Exception()

8
tests/compute/test_pollsters.py
View File

@ -56,7 +56,7 @@ class TestInstancePollster(TestPollsterBase):
mgr = manager.AgentManager()
pollster = pollsters.InstancePollster()
counters = list(pollster.get_counters(mgr, self.instance))
counters = list(pollster.get_counters(mgr, {}, self.instance))
self.assertEquals(len(counters), 2)
self.assertEqual(counters[0].name, 'instance')
self.assertEqual(counters[1].name, 'instance:m1.small')
@ -80,7 +80,7 @@ class TestDiskIOPollster(TestPollsterBase):
mgr = manager.AgentManager()
pollster = pollsters.DiskIOPollster()
counters = list(pollster.get_counters(mgr, self.instance))
counters = list(pollster.get_counters(mgr, {}, self.instance))
assert counters
self.assertEqual(set([c.name for c in counters]),
@ -143,7 +143,7 @@ class TestNetPollster(TestPollsterBase):
mgr = manager.AgentManager()
pollster = pollsters.NetPollster()
counters = list(pollster.get_counters(mgr, self.instance))
counters = list(pollster.get_counters(mgr, {}, self.instance))
assert counters
self.assertEqual(set([c.name for c in counters]),
set(pollster.get_counter_names()))
@ -203,7 +203,7 @@ class TestCPUPollster(TestPollsterBase):
pollster = pollsters.CPUPollster()
def _verify_cpu_metering(zero, expected_time):
counters = list(pollster.get_counters(mgr, self.instance))
counters = list(pollster.get_counters(mgr, {}, self.instance))
self.assertEquals(len(counters), 2)
self.assertEqual(set([c.name for c in counters]),
set(pollster.get_counter_names()))

6
tests/energy/test_kwapi.py
View File

@ -76,13 +76,13 @@ class TestKwapiPollster(base.TestCase):
self.stubs.Set(kwapi._Base, 'get_kwapi_client',
self.fake_kwapi_get_kwapi_client)
counters = list(kwapi.KwapiPollster().get_counters(self.manager))
counters = list(kwapi.KwapiPollster().get_counters(self.manager, {}))
self.assertEqual(len(counters), 0)
def test_kwapi_counter(self):
self.stubs.Set(kwapi._Base, 'iter_probes', self.fake_kwapi_iter_probes)
counters = list(kwapi.KwapiPollster().get_counters(self.manager))
counters = list(kwapi.KwapiPollster().get_counters(self.manager, {}))
self.assertEqual(len(counters), 6)
energy_counters = [counter for counter in counters
if counter.name == "energy"]
@ -104,6 +104,6 @@ class TestKwapiPollster(base.TestCase):
def test_kwapi_counter_list(self):
self.stubs.Set(kwapi._Base, 'iter_probes', self.fake_kwapi_iter_probes)
counters = list(kwapi.KwapiPollster().get_counters(self.manager))
counters = list(kwapi.KwapiPollster().get_counters(self.manager, {}))
self.assertEqual(set([c.name for c in counters]),
set(kwapi.KwapiPollster().get_counter_names()))

4
tests/image/test_glance.py
View File

@ -140,7 +140,7 @@ class TestImagePollster(base.TestCase):
self.assertEqual(len(images), len(set(image.id for image in images)))
def test_glance_image_counter(self):
counters = list(glance.ImagePollster().get_counters(self.manager))
counters = list(glance.ImagePollster().get_counters(self.manager, {}))
self.assertEqual(len(counters), 6)
for counter in [c for c in counters if c.name == 'image']:
self.assertEqual(counter.volume, 1)
@ -150,6 +150,6 @@ class TestImagePollster(base.TestCase):
counters)))
def test_get_counter_names(self):
counters = list(glance.ImagePollster().get_counters(self.manager))
counters = list(glance.ImagePollster().get_counters(self.manager, {}))
self.assertEqual(set([c.name for c in counters]),
set(glance.ImagePollster().get_counter_names()))

4
tests/network/test_floatingip.py
View File

@ -65,7 +65,7 @@ class TestFloatingIPPollster(base.TestCase):
# assert False, 'Should have seen an error'
def test_get_counters_not_empty(self):
counters = list(self.pollster.get_counters(self.manager))
counters = list(self.pollster.get_counters(self.manager, {}))
self.assertEqual(len(counters), 3)
# It's necessary to verify all the attributes extracted by Nova
# API /os-floating-ips to make sure they're available and correct.
@ -82,6 +82,6 @@ class TestFloatingIPPollster(base.TestCase):
self.assertEqual(counters[2].resource_metadata["pool"], "public")
def test_get_counter_names(self):
counters = list(self.pollster.get_counters(self.manager))
counters = list(self.pollster.get_counters(self.manager, {}))
self.assertEqual(set([c.name for c in counters]),
set(self.pollster.get_counter_names()))

6
tests/objectstore/test_swift.py
View File

@ -77,18 +77,18 @@ class TestSwiftPollster(base.TestCase):
def test_objectstore_metering(self):
self.stubs.Set(swift.SwiftPollster, 'iter_accounts',
self.fake_iter_accounts)
counters = list(self.pollster.get_counters(self.manager))
counters = list(self.pollster.get_counters(self.manager, {}))
self.assertEqual(len(counters), 6)
def test_objectstore_get_counter_names(self):
self.stubs.Set(swift.SwiftPollster, 'iter_accounts',
self.fake_iter_accounts)
counters = list(self.pollster.get_counters(self.manager))
counters = list(self.pollster.get_counters(self.manager, {}))
self.assertEqual(set([c.name for c in counters]),
set(self.pollster.get_counter_names()))
def test_objectstore_endpoint_notfound(self):
self.stubs.Set(self.manager.keystone.service_catalog, 'url_for',
self.fake_ks_service_catalog_url_for)
counters = list(self.pollster.get_counters(self.manager))
counters = list(self.pollster.get_counters(self.manager, {}))
self.assertEqual(len(counters), 0)