openstack/aodh
Code Issues Proposed changes

Merge "Add pollster data cache"

Browse Source
This commit is contained in:
Jenkins 2013-07-04 23:54:10 +00:00 committed by Gerrit Code Review
commit ad99ae85c9
17 changed files with 52 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
ceilometer/central/manager.py
View File

@ -37,11 +37,15 @@ class PollingTask(agent.PollingTask):
with self.publish_context as publisher:
# TODO(yjiang5) passing counters into get_counters to avoid
# polling all counters one by one
cache = {}
for pollster in self.pollsters:
try:
LOG.info("Polling pollster %s", pollster.name)
publisher(list(pollster.obj.get_counters(
self.manager)))
counters = list(pollster.obj.get_counters(
self.manager,
cache,
))
publisher(counters)
except Exception as err:
LOG.warning('Continue after error from %s: %s',
pollster.name, err)

5
ceilometer/compute/manager.py
View File

@ -37,12 +37,15 @@ class PollingTask(agent.PollingTask):
if getattr(instance, 'OS-EXT-STS:vm_state', None) != 'error':
# TODO(yjiang5) passing counters to get_counters to avoid
# polling all counters one by one
cache = {}
for pollster in self.pollsters:
try:
LOG.info("Polling pollster %s", pollster.name)
publisher(list(pollster.obj.get_counters(
self.manager,
instance)))
cache,
instance,
)))
except Exception as err:
LOG.warning('Continue after error from %s: %s',
pollster.name, err)

6
ceilometer/compute/nova_notifier.py
View File

@ -55,12 +55,14 @@ class DeletedInstanceStatsGatherer(object):
self.mgr = extensions
self.inspector = inspector.get_hypervisor_inspector()
def _get_counters_from_plugin(self, ext, instance, *args, **kwds):
def _get_counters_from_plugin(self, ext, cache, instance, *args, **kwds):
"""Used with the extenaion manager map() method."""
return ext.obj.get_counters(self, instance)
return ext.obj.get_counters(self, cache, instance)
def __call__(self, instance):
cache = {}
counters = self.mgr.map(self._get_counters_from_plugin,
cache=cache,
instance=instance,
)
# counters is a list of lists, so flatten it before returning

6
ceilometer/compute/plugin.py
View File

@ -30,6 +30,10 @@ class ComputePollster(plugin.PollsterBase):
__metaclass__ = abc.ABCMeta
@abc.abstractmethod
def get_counters(self, manager, context):
def get_counters(self, manager, cache, instance):
"""Return a sequence of Counter instances from polling the resources.
:param manager: The service manager invoking the plugin
:param cache: A dictionary for passing data between plugins
:param instance: The instance to examine
"""

8
ceilometer/compute/pollsters.py
View File

@ -57,7 +57,7 @@ class InstancePollster(plugin.ComputePollster):
# variable. We don't need such format in future
return ['instance', 'instance:*']
def get_counters(self, manager, instance):
def get_counters(self, manager, cache, instance):
yield make_counter_from_instance(instance,
name='instance',
type=counter.TYPE_GAUGE,
@ -91,7 +91,7 @@ class DiskIOPollster(plugin.ComputePollster):
'disk.write.requests',
'disk.write.bytes']
def get_counters(self, manager, instance):
def get_counters(self, manager, cache, instance):
instance_name = _instance_name(instance)
try:
r_bytes = 0
@ -164,7 +164,7 @@ class CPUPollster(plugin.ComputePollster):
def get_counter_names():
return ['cpu', 'cpu_util']
def get_counters(self, manager, instance):
def get_counters(self, manager, cache, instance):
self.LOG.info('checking instance %s', instance.id)
instance_name = _instance_name(instance)
try:
@ -237,7 +237,7 @@ class NetPollster(plugin.ComputePollster):
'network.outgoing.bytes',
'network.outgoing.packets']
def get_counters(self, manager, instance):
def get_counters(self, manager, cache, instance):
instance_name = _instance_name(instance)
self.LOG.info('checking instance %s', instance.id)
try:

2
ceilometer/energy/kwapi.py
View File

@ -77,7 +77,7 @@ class KwapiPollster(_Base):
def get_counter_names():
return ['energy', 'power']
def get_counters(self, manager):
def get_counters(self, manager, cache):
"""Returns all counters."""
for probe in self.iter_probes(manager.keystone):
yield counter.Counter(

2
ceilometer/image/glance.py
View File

@ -99,7 +99,7 @@ class ImagePollster(_Base):
def get_counter_names():
return ['image', 'image.size']
def get_counters(self, manager):
def get_counters(self, manager, cache):
for image in self.iter_images(manager.keystone):
yield counter.Counter(
name='image',

2
ceilometer/network/floatingip.py
View File

@ -35,7 +35,7 @@ class FloatingIPPollster(plugin.CentralPollster):
def get_counter_names():
return ['ip.floating']
def get_counters(self, manager):
def get_counters(self, manager, cache):
nv = nova_client.Client()
for ip in nv.floating_ip_get_all():
self.LOG.info("FLOATING IP USAGE: %s" % ip.ip)

2
ceilometer/objectstore/swift.py
View File

@ -55,7 +55,7 @@ class _Base(plugin.PollsterBase):
def iter_accounts(ksclient):
"""Iterate over all accounts, yielding (tenant_id, stats) tuples."""
def get_counters(self, manager):
def get_counters(self, manager, cache):
for tenant, account in self.iter_accounts(manager.keystone):
yield counter.Counter(
name='storage.objects',

8
ceilometer/plugin.py
View File

@ -88,7 +88,13 @@ class PollsterBase(PluginBase):
"""Return a sequence of Counter names supported by the pollster."""
@abc.abstractmethod
def get_counters(self, manager, instance):
def get_counters(self, manager, cache):
"""Return a sequence of Counter instances from polling the resources.
:param manager: The service manager class invoking the plugin.
:param cache: A dictionary to allow pollsters to pass data
between themselves when recomputing it would be
expensive (e.g., asking another service for a
list of objects).
"""

2
nova_tests/test_notifier.py
View File

@ -77,7 +77,7 @@ class TestNovaNotifier(base.TestCase):
},
)
def get_counters(self, manager, instance):
def get_counters(self, manager, cache, instance):
self.instances.append((manager, instance))
return [self.test_data]

6
tests/agentbase.py
View File

@ -52,17 +52,17 @@ class TestPollster:
def get_counter_names(self):
return [self.test_data.name]
def get_counters(self, manager, instance=None):
def get_counters(self, manager, cache, instance=None):
self.counters.append((manager, instance))
return [self.test_data]
class TestPollsterException(TestPollster):
def get_counters(self, manager, instance=None):
def get_counters(self, manager, cache, instance=None):
# Put an instance parameter here so that it can be used
# by both central manager and compute manager
# In future, we possibly don't need such hack if we
# combin the get_counters() function again
# combine the get_counters() function again
self.counters.append((manager, instance))
raise Exception()

8
tests/compute/test_pollsters.py
View File

@ -56,7 +56,7 @@ class TestInstancePollster(TestPollsterBase):
mgr = manager.AgentManager()
pollster = pollsters.InstancePollster()
counters = list(pollster.get_counters(mgr, self.instance))
counters = list(pollster.get_counters(mgr, {}, self.instance))
self.assertEquals(len(counters), 2)
self.assertEqual(counters[0].name, 'instance')
self.assertEqual(counters[1].name, 'instance:m1.small')
@ -80,7 +80,7 @@ class TestDiskIOPollster(TestPollsterBase):
mgr = manager.AgentManager()
pollster = pollsters.DiskIOPollster()
counters = list(pollster.get_counters(mgr, self.instance))
counters = list(pollster.get_counters(mgr, {}, self.instance))
assert counters
self.assertEqual(set([c.name for c in counters]),
@ -143,7 +143,7 @@ class TestNetPollster(TestPollsterBase):
mgr = manager.AgentManager()
pollster = pollsters.NetPollster()
counters = list(pollster.get_counters(mgr, self.instance))
counters = list(pollster.get_counters(mgr, {}, self.instance))
assert counters
self.assertEqual(set([c.name for c in counters]),
set(pollster.get_counter_names()))
@ -203,7 +203,7 @@ class TestCPUPollster(TestPollsterBase):
pollster = pollsters.CPUPollster()
def _verify_cpu_metering(zero, expected_time):
counters = list(pollster.get_counters(mgr, self.instance))
counters = list(pollster.get_counters(mgr, {}, self.instance))
self.assertEquals(len(counters), 2)
self.assertEqual(set([c.name for c in counters]),
set(pollster.get_counter_names()))

6
tests/energy/test_kwapi.py
View File

@ -76,13 +76,13 @@ class TestKwapiPollster(base.TestCase):
self.stubs.Set(kwapi._Base, 'get_kwapi_client',
self.fake_kwapi_get_kwapi_client)
counters = list(kwapi.KwapiPollster().get_counters(self.manager))
counters = list(kwapi.KwapiPollster().get_counters(self.manager, {}))
self.assertEqual(len(counters), 0)
def test_kwapi_counter(self):
self.stubs.Set(kwapi._Base, 'iter_probes', self.fake_kwapi_iter_probes)
counters = list(kwapi.KwapiPollster().get_counters(self.manager))
counters = list(kwapi.KwapiPollster().get_counters(self.manager, {}))
self.assertEqual(len(counters), 6)
energy_counters = [counter for counter in counters
if counter.name == "energy"]
@ -104,6 +104,6 @@ class TestKwapiPollster(base.TestCase):
def test_kwapi_counter_list(self):
self.stubs.Set(kwapi._Base, 'iter_probes', self.fake_kwapi_iter_probes)
counters = list(kwapi.KwapiPollster().get_counters(self.manager))
counters = list(kwapi.KwapiPollster().get_counters(self.manager, {}))
self.assertEqual(set([c.name for c in counters]),
set(kwapi.KwapiPollster().get_counter_names()))

4
tests/image/test_glance.py
View File

@ -140,7 +140,7 @@ class TestImagePollster(base.TestCase):
self.assertEqual(len(images), len(set(image.id for image in images)))
def test_glance_image_counter(self):
counters = list(glance.ImagePollster().get_counters(self.manager))
counters = list(glance.ImagePollster().get_counters(self.manager, {}))
self.assertEqual(len(counters), 6)
for counter in [c for c in counters if c.name == 'image']:
self.assertEqual(counter.volume, 1)
@ -150,6 +150,6 @@ class TestImagePollster(base.TestCase):
counters)))
def test_get_counter_names(self):
counters = list(glance.ImagePollster().get_counters(self.manager))
counters = list(glance.ImagePollster().get_counters(self.manager, {}))
self.assertEqual(set([c.name for c in counters]),
set(glance.ImagePollster().get_counter_names()))

4
tests/network/test_floatingip.py
View File

@ -65,7 +65,7 @@ class TestFloatingIPPollster(base.TestCase):
# assert False, 'Should have seen an error'
def test_get_counters_not_empty(self):
counters = list(self.pollster.get_counters(self.manager))
counters = list(self.pollster.get_counters(self.manager, {}))
self.assertEqual(len(counters), 3)
# It's necessary to verify all the attributes extracted by Nova
# API /os-floating-ips to make sure they're available and correct.
@ -82,6 +82,6 @@ class TestFloatingIPPollster(base.TestCase):
self.assertEqual(counters[2].resource_metadata["pool"], "public")
def test_get_counter_names(self):
counters = list(self.pollster.get_counters(self.manager))
counters = list(self.pollster.get_counters(self.manager, {}))
self.assertEqual(set([c.name for c in counters]),
set(self.pollster.get_counter_names()))

6
tests/objectstore/test_swift.py
View File

@ -77,18 +77,18 @@ class TestSwiftPollster(base.TestCase):
def test_objectstore_metering(self):
self.stubs.Set(swift.SwiftPollster, 'iter_accounts',
self.fake_iter_accounts)
counters = list(self.pollster.get_counters(self.manager))
counters = list(self.pollster.get_counters(self.manager, {}))
self.assertEqual(len(counters), 6)
def test_objectstore_get_counter_names(self):
self.stubs.Set(swift.SwiftPollster, 'iter_accounts',
self.fake_iter_accounts)
counters = list(self.pollster.get_counters(self.manager))
counters = list(self.pollster.get_counters(self.manager, {}))
self.assertEqual(set([c.name for c in counters]),
set(self.pollster.get_counter_names()))
def test_objectstore_endpoint_notfound(self):
self.stubs.Set(self.manager.keystone.service_catalog, 'url_for',
self.fake_ks_service_catalog_url_for)
counters = list(self.pollster.get_counters(self.manager))
counters = list(self.pollster.get_counters(self.manager, {}))
self.assertEqual(len(counters), 0)