From 9f0cfce9a6a30e03746ae1f207b6c5e3cd0ba7bc Mon Sep 17 00:00:00 2001 From: Ladislav Smola Date: Mon, 8 Sep 2014 18:09:13 +0200 Subject: [PATCH] Allow to pass dict from resource discovery Allowing to pass dict from resource discovery and storing it as metadata. Before it was only possible to pass IP address, which is not enough. For TripleO case, important metadata like image name and flavor from Nova are passed and stored in resource, allowing to make aggregate queries according to them. Change-Id: I4dabf79f248842398e1126d310d110b1fe95fad5 Closes-Bug: #1366871 --- ceilometer/hardware/discovery.py | 22 +++++++++-- ceilometer/hardware/inspector/base.py | 7 ++-- ceilometer/hardware/inspector/snmp.py | 8 ++-- ceilometer/hardware/plugin.py | 39 +++++++++++++++++-- ceilometer/hardware/pollsters/disk.py | 2 +- ceilometer/hardware/pollsters/net.py | 2 +- ceilometer/hardware/pollsters/util.py | 7 ++-- ceilometer/tests/hardware/pollsters/base.py | 2 +- .../tests/hardware/pollsters/test_util.py | 7 +++- 9 files changed, 74 insertions(+), 22 deletions(-) diff --git a/ceilometer/hardware/discovery.py b/ceilometer/hardware/discovery.py index 643830879..8848eb576 100644 --- a/ceilometer/hardware/discovery.py +++ b/ceilometer/hardware/discovery.py @@ -41,22 +41,36 @@ class NodesDiscoveryTripleO(plugin.DiscoveryBase): super(NodesDiscoveryTripleO, self).__init__() self.nova_cli = nova_client.Client() + @staticmethod + def _address(instance, field): + return instance.addresses['ctlplane'][0].get(field) + def discover(self, param=None): """Discover resources to monitor.""" instances = self.nova_cli.instance_get_all() - ip_addresses = [] + resources = [] for instance in instances: try: - ip_address = instance.addresses['ctlplane'][0]['addr'] + ip_address = self._address(instance, 'addr') final_address = ( cfg.CONF.hardware.url_scheme + cfg.CONF.hardware.readonly_user_name + ':' + cfg.CONF.hardware.readonly_user_password + '@' + ip_address) - ip_addresses.append(final_address) + + resource = { + 'resource_id': instance.id, + 'resource_url': final_address, + 'mac_addr': self._address(instance, + 'OS-EXT-IPS-MAC:mac_addr'), + 'image_id': instance.image['id'], + 'flavor_id': instance.flavor['id'] + } + + resources.append(resource) except KeyError: LOG.error(_("Couldn't obtain IP address of" "instance %s") % instance.id) - return ip_addresses + return resources diff --git a/ceilometer/hardware/inspector/base.py b/ceilometer/hardware/inspector/base.py index 872efc5d6..71bc270d0 100644 --- a/ceilometer/hardware/inspector/base.py +++ b/ceilometer/hardware/inspector/base.py @@ -25,14 +25,15 @@ import six @six.add_metaclass(abc.ABCMeta) class Inspector(object): @abc.abstractmethod - def inspect_generic(self, host, identifier, cache): + def inspect_generic(self, host, identifier, cache, extra_metadata=None): """A generic inspect function. :param host: the target host :param identifier: the identifier of the metric :param cache: cache passed from the pollster + :param extra_metadata: extra dict to be used as metadata :return: an iterator of (value, metadata, extra) :return value: the sample value :return metadata: dict to construct sample's metadata - :return extra: dict of extra info to help constructing sample - """ \ No newline at end of file + :return extra: dict of extra metadata to help constructing sample + """ diff --git a/ceilometer/hardware/inspector/snmp.py b/ceilometer/hardware/inspector/snmp.py index 826fff12e..f5f53e3e1 100644 --- a/ceilometer/hardware/inspector/snmp.py +++ b/ceilometer/hardware/inspector/snmp.py @@ -354,7 +354,7 @@ class SNMPInspector(base.Inspector): new_oids.append(metadata[0]) return new_oids - def inspect_generic(self, host, identifier, cache): + def inspect_generic(self, host, identifier, cache, extra_metadata=None): # the snmp definition for the corresponding meter meter_def = self.MAPPING[identifier] # collect oids that needs to be queried @@ -373,6 +373,7 @@ class SNMPInspector(base.Inspector): meter_def['metric_oid'][0], meter_def['matching_type'], False) + extra_metadata = extra_metadata or {} for oid in oids_for_sample_values: suffix = oid[len(meter_def['metric_oid'][0]):] value = self.get_oid_value(oid_cache, @@ -382,15 +383,14 @@ class SNMPInspector(base.Inspector): metadata = self.construct_metadata(oid_cache, meter_def['metadata'], suffix) - extra = {} # call post_op for special cases if meter_def['post_op']: func = getattr(self, meter_def['post_op'], None) if func: value = func(host, cache, meter_def, - value, metadata, extra, + value, metadata, extra_metadata, suffix) - yield (value, metadata, extra) + yield (value, metadata, extra_metadata) def _post_op_net(self, host, cache, meter_def, value, metadata, extra, suffix): diff --git a/ceilometer/hardware/plugin.py b/ceilometer/hardware/plugin.py index 7ed634ce7..5e9dc9c60 100644 --- a/ceilometer/hardware/plugin.py +++ b/ceilometer/hardware/plugin.py @@ -47,7 +47,36 @@ class HardwarePollster(plugin.CentralPollster): def default_discovery(self): return 'tripleo_overcloud_nodes' - def get_samples(self, manager, cache, resources): + @staticmethod + def _parse_resource(res): + """Parse resource from discovery. + + Either URL can be given or dict. Dict has to contain at least + keys 'resource_id' and 'resource_url', all the dict keys will be stored + as metadata. + + :param res: URL or dict containing all resource info. + :return parsed_url, resource_id, metadata: Returns parsed URL used for + SNMP query, unique identifier of the resource and metadata + of the resource. + """ + + if isinstance(res, dict): + if 'resource_url' not in res or 'resource_id' not in res: + LOG.exception(_('Passed resource dict must contain keys ' + 'resource_id and resource_url.')) + + metadata = res + parsed_url = netutils.urlsplit(res['resource_url']) + resource_id = res['resource_id'] + else: + metadata = {} + parsed_url = netutils.urlsplit(res) + resource_id = res + + return parsed_url, resource_id, metadata + + def get_samples(self, manager, cache, resources=None): """Return an iterable of Sample instances from polling the resources. :param manager: The service manager invoking the plugin @@ -57,17 +86,19 @@ class HardwarePollster(plugin.CentralPollster): resources = resources or [] h_cache = cache.setdefault(self.CACHE_KEY, {}) sample_iters = [] - for res in resources: - parsed_url = netutils.urlsplit(res) + for resource in resources: + parsed_url, res, extra_metadata = self._parse_resource(resource) ins = self._get_inspector(parsed_url) try: # Call hardware inspector to poll for the data i_cache = h_cache.setdefault(res, {}) + if self.IDENTIFIER not in i_cache: i_cache[self.IDENTIFIER] = list(ins.inspect_generic( parsed_url, self.IDENTIFIER, - i_cache)) + i_cache, + extra_metadata)) # Generate samples if i_cache[self.IDENTIFIER]: sample_iters.append(self.generate_samples( diff --git a/ceilometer/hardware/pollsters/disk.py b/ceilometer/hardware/pollsters/disk.py index 0ab23edfc..67735e6f2 100644 --- a/ceilometer/hardware/pollsters/disk.py +++ b/ceilometer/hardware/pollsters/disk.py @@ -29,7 +29,7 @@ class _Base(plugin.HardwarePollster): def generate_one_sample(self, host, c_data): value, metadata, extra = c_data - res_id = host.hostname + res_id = extra.get('resource_id') or host.hostname if metadata.get('device'): res_id = res_id + ".%s" % metadata.get('device') return util.make_sample_from_host(host, diff --git a/ceilometer/hardware/pollsters/net.py b/ceilometer/hardware/pollsters/net.py index baa5ee610..bb9ba3e92 100644 --- a/ceilometer/hardware/pollsters/net.py +++ b/ceilometer/hardware/pollsters/net.py @@ -29,7 +29,7 @@ class _Base(plugin.HardwarePollster): def generate_one_sample(self, host, c_data): value, metadata, extra = c_data - res_id = host.hostname + res_id = extra.get('resource_id') or host.hostname if metadata.get('name'): res_id = res_id + ".%s" % metadata.get('name') return util.make_sample_from_host(host, diff --git a/ceilometer/hardware/pollsters/util.py b/ceilometer/hardware/pollsters/util.py index 298314b45..d2f807be4 100644 --- a/ceilometer/hardware/pollsters/util.py +++ b/ceilometer/hardware/pollsters/util.py @@ -44,15 +44,16 @@ def make_sample_from_host(host_url, name, sample_type, unit, volume, res_metadata=None, extra={}): resource_metadata = make_resource_metadata(res_metadata, host_url) + resource_metadata.update(extra) - res_id = extra.get('resource_id') or resource_id or host_url.hostname + res_id = resource_id or extra.get('resource_id') or host_url.hostname return sample.Sample( name='hardware.' + name, type=sample_type, unit=unit, volume=volume, - user_id=extra.get('user_id') or user_id, - project_id=extra.get('project_id') or project_id, + user_id=user_id or extra.get('user_id'), + project_id=project_id or extra.get('project_id'), resource_id=res_id, timestamp=timeutils.isotime(), resource_metadata=resource_metadata, diff --git a/ceilometer/tests/hardware/pollsters/base.py b/ceilometer/tests/hardware/pollsters/base.py index 0a8748e82..72240efab 100644 --- a/ceilometer/tests/hardware/pollsters/base.py +++ b/ceilometer/tests/hardware/pollsters/base.py @@ -42,7 +42,7 @@ class FakeInspector(inspector_base.Inspector): 'disk.size.used': (90, disk_metadata, {}), } - def inspect_generic(self, host, identifier, cache): + def inspect_generic(self, host, identifier, cache, extra_metadata=None): yield self.DATA[identifier] diff --git a/ceilometer/tests/hardware/pollsters/test_util.py b/ceilometer/tests/hardware/pollsters/test_util.py index 194d04f48..d6e5c736d 100644 --- a/ceilometer/tests/hardware/pollsters/test_util.py +++ b/ceilometer/tests/hardware/pollsters/test_util.py @@ -53,4 +53,9 @@ class TestPollsterUtils(test_base.BaseTestCase): extra=extra) self.assertEqual(None, s.user_id) self.assertEqual('project', s.project_id) - self.assertEqual('resource', s.resource_id) \ No newline at end of file + self.assertEqual('resource', s.resource_id) + self.assertEqual({'resource_url': 'snmp://127.0.0.1:161', + 'project_id': 'project', + 'resource_id': + 'resource'}, + s.resource_metadata)