openstack/aodh
Code Issues Proposed changes

Ensure user metadata mapped for instance notifications

Browse Source 
Fixes bug 1284073

For pollster-originated samples related to instances, we apply a
mapping to the user metadata so as to ensure that there are no
embedded periods in metadata keys and also to exclude any metadata
not matching the configured reserved namespace.

Now this logic is also applied to user metadata for samples derived
from instance-related notifications in order to avoid failures when
persisting these data in mongodb.

Change-Id: I77eab59023fa23c5879b05c948bd4cb1db7fa177
This commit is contained in:
Eoghan Glynn 2014-03-04 18:46:29 +00:00
parent cae5b179e5
commit ddeb54bb0d
5 changed files with 303 additions and 84 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
ceilometer/compute/notifications/instance.py
View File

@ -20,14 +20,45 @@
"""Converters for producing compute sample messages from notification events.
"""
import abc
import six
from ceilometer.compute import notifications
from ceilometer.compute import util
from ceilometer import sample
class InstanceScheduled(notifications.ComputeNotificationBase):
event_types = ['scheduler.run_instance.scheduled']
@six.add_metaclass(abc.ABCMeta)
class UserMetadataAwareInstanceNotificationBase(
notifications.ComputeNotificationBase):
"""Consumes notifications containing instance user metadata.
"""
def process_notification(self, message):
instance_properties = self.get_instance_properties(message)
if 'metadata' in instance_properties:
src_metadata = instance_properties['metadata']
del instance_properties['metadata']
util.add_reserved_user_metadata(src_metadata, instance_properties)
return self.get_sample(message)
def get_instance_properties(self, message):
"""Retrieve instance properties from notification payload."""
return message['payload']
@abc.abstractmethod
def get_sample(self, message):
"""Derive sample from notification payload."""
class InstanceScheduled(UserMetadataAwareInstanceNotificationBase):
event_types = ['scheduler.run_instance.scheduled']
def get_instance_properties(self, message):
"""Retrieve instance properties from notification payload."""
return message['payload']['request_spec']['instance_properties']
def get_sample(self, message):
yield sample.Sample.from_notification(
name='instance.scheduled',
type=sample.TYPE_DELTA,
@ -41,14 +72,15 @@ class InstanceScheduled(notifications.ComputeNotificationBase):
message=message)
class ComputeInstanceNotificationBase(notifications.ComputeNotificationBase):
class ComputeInstanceNotificationBase(
UserMetadataAwareInstanceNotificationBase):
"""Convert compute.instance.* notifications into Samples
"""
event_types = ['compute.instance.*']
class Instance(ComputeInstanceNotificationBase):
def process_notification(self, message):
def get_sample(self, message):
yield sample.Sample.from_notification(
name='instance',
type=sample.TYPE_GAUGE,
@ -61,7 +93,7 @@ class Instance(ComputeInstanceNotificationBase):
class Memory(ComputeInstanceNotificationBase):
def process_notification(self, message):
def get_sample(self, message):
yield sample.Sample.from_notification(
name='memory',
type=sample.TYPE_GAUGE,
@ -74,7 +106,7 @@ class Memory(ComputeInstanceNotificationBase):
class VCpus(ComputeInstanceNotificationBase):
def process_notification(self, message):
def get_sample(self, message):
yield sample.Sample.from_notification(
name='vcpus',
type=sample.TYPE_GAUGE,
@ -87,7 +119,7 @@ class VCpus(ComputeInstanceNotificationBase):
class RootDiskSize(ComputeInstanceNotificationBase):
def process_notification(self, message):
def get_sample(self, message):
yield sample.Sample.from_notification(
name='disk.root.size',
type=sample.TYPE_GAUGE,
@ -100,7 +132,7 @@ class RootDiskSize(ComputeInstanceNotificationBase):
class EphemeralDiskSize(ComputeInstanceNotificationBase):
def process_notification(self, message):
def get_sample(self, message):
yield sample.Sample.from_notification(
name='disk.ephemeral.size',
type=sample.TYPE_GAUGE,
@ -113,7 +145,7 @@ class EphemeralDiskSize(ComputeInstanceNotificationBase):
class InstanceFlavor(ComputeInstanceNotificationBase):
def process_notification(self, message):
def get_sample(self, message):
instance_type = message.get('payload', {}).get('instance_type')
if instance_type:
yield sample.Sample.from_notification(
@ -134,7 +166,7 @@ class InstanceDelete(ComputeInstanceNotificationBase):
event_types = ['compute.instance.delete.samples']
def process_notification(self, message):
def get_sample(self, message):
for s in message['payload'].get('samples', []):
yield sample.Sample.from_notification(
name=s['name'],

34
ceilometer/compute/pollsters/util.py
View File

@ -18,8 +18,7 @@
# License for the specific language governing permissions and limitations
# under the License.
from oslo.config import cfg
from ceilometer.compute import util as compute_util
from ceilometer.openstack.common import timeutils
from ceilometer import sample
@ -35,35 +34,6 @@ INSTANCE_PROPERTIES = [
'ramdisk_id',
]
OPTS = [
cfg.ListOpt('reserved_metadata_namespace',
default=['metering.'],
help='List of metadata prefixes reserved for metering use.'),
cfg.IntOpt('reserved_metadata_length',
default=256,
help='Limit on length of reserved metadata values.'),
]
cfg.CONF.register_opts(OPTS)
def _add_reserved_user_metadata(instance, metadata):
limit = cfg.CONF.reserved_metadata_length
user_metadata = {}
for prefix in cfg.CONF.reserved_metadata_namespace:
md = dict(
(k[len(prefix):].replace('.', '_'),
v[:limit] if isinstance(v, basestring) else v)
for k, v in instance.metadata.items()
if (k.startswith(prefix) and
k[len(prefix):].replace('.', '_') not in metadata)
)
user_metadata.update(md)
if user_metadata:
metadata['user_metadata'] = user_metadata
return metadata
def _get_metadata_from_object(instance):
"""Return a metadata dictionary for the instance.
@ -97,7 +67,7 @@ def _get_metadata_from_object(instance):
metadata['root_gb'] = int(metadata['disk_gb']) - \
int(metadata['ephemeral_gb'])
return _add_reserved_user_metadata(instance, metadata)
return compute_util.add_reserved_user_metadata(instance.metadata, metadata)
def make_sample_from_instance(instance, name, type, unit, volume,

49
ceilometer/compute/util.py Normal file
View File

@ -0,0 +1,49 @@
# -*- encoding: utf-8 -*-
#
# Copyright © 2014 Red Hat, Inc
#
# Author: Eoghan Glynn <eglynn@redhat.com>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslo.config import cfg
OPTS = [
cfg.ListOpt('reserved_metadata_namespace',
default=['metering.'],
help='List of metadata prefixes reserved for metering use.'),
cfg.IntOpt('reserved_metadata_length',
default=256,
help='Limit on length of reserved metadata values.'),
]
cfg.CONF.register_opts(OPTS)
def add_reserved_user_metadata(src_metadata, dest_metadata):
limit = cfg.CONF.reserved_metadata_length
user_metadata = {}
for prefix in cfg.CONF.reserved_metadata_namespace:
md = dict(
(k[len(prefix):].replace('.', '_'),
v[:limit] if isinstance(v, basestring) else v)
for k, v in src_metadata.items()
if (k.startswith(prefix) and
k[len(prefix):].replace('.', '_') not in dest_metadata)
)
user_metadata.update(md)
if user_metadata:
dest_metadata['user_metadata'] = user_metadata
return dest_metadata

250
ceilometer/tests/compute/notifications/test_instance.py
View File

@ -201,7 +201,9 @@ INSTANCE_FINISH_RESIZE_END = {
u'hostname': u's1',
u'state': u'resized',
u'launched_at': u'2013-01-04T15:10:14.923939',
u'metadata': [],
u'metadata': {u'metering.server_group': u'Group_A',
u'AutoScalingGroupName': u'tyky-Group_Awste7',
u'metering.foo.bar': u'true'},
u'ramdisk_id': u'5f23128e-5525-46d8-bc66-9c30cd87141a',
u'access_ip_v6': None,
u'disk_gb': 20,
@ -258,7 +260,9 @@ INSTANCE_RESIZE_REVERT_END = {
u'hostname': u's1',
u'state': u'resized',
u'launched_at': u'2013-01-04T15:10:14.000000',
u'metadata': [],
u'metadata': {u'metering.server_group': u'Group_A',
u'AutoScalingGroupName': u'tyky-Group_A-wste7',
u'metering.foo.bar': u'true'},
u'ramdisk_id': u'5f23128e-5525-46d8-bc66-9c30cd87141a',
u'access_ip_v6': None,
u'disk_gb': 0,
@ -315,7 +319,9 @@ INSTANCE_DELETE_SAMPLES = {
u'hostname': u's1',
u'state': u'resized',
u'launched_at': u'2013-01-04T15:10:14.000000',
u'metadata': [],
u'metadata': {u'metering.server_group': u'Group_A',
u'AutoScalingGroupName': u'tyky-Group_A-wste7',
u'metering.foo.bar': u'true'},
u'ramdisk_id': u'5f23128e-5525-46d8-bc66-9c30cd87141a',
u'access_ip_v6': None,
u'disk_gb': 0,
@ -352,49 +358,190 @@ INSTANCE_DELETE_SAMPLES = {
u'priority': u'INFO'
}
INSTANCE_SCHEDULED = {
u'_context_roles': [u'admin'],
u'_context_request_id': u'req-9da1d714-dabe-42fd-8baa-583e57cd4f1a',
u'_context_request_id': u'req-f28a836a-32bf-4cc3-940a-3515878c181f',
u'_context_quota_class': None,
u'event_type': u'scheduler.run_instance.scheduled',
u'_context_user_name': u'admin',
u'_context_project_name': u'admin',
u'timestamp': u'2013-01-04 15:20:32.009532',
u'_context_is_admin': True,
u'message_id': u'c48deeba-d0c3-4154-b3db-47480b52267a',
u'_context_auth_token': None,
u'_context_instance_lock_checked': False,
u'_context_project_id': u'cea4b25edb484e5392727181b7721d29',
u'_context_timestamp': u'2013-01-04T15:19:51.018218',
u'_context_read_deleted': u'no',
u'_context_user_id': u'01b83a5e23f24a6fb6cd073c0aee6eed',
u'_context_remote_address': u'10.147.132.184',
u'publisher_id': u'compute.ip-10-147-132-184.ec2.internal',
u'_context_service_catalog': [{
u'endpoints': [{
u'adminURL':
u'http://172.16.12.21:8776/v1/2bd766a095b44486bf07cf7f666997eb',
u'region': u'RegionOne',
u'internalURL':
u'http://172.16.12.21:8776/v1/2bd766a095b44486bf07cf7f666997eb',
u'id': u'30cb904fdc294eea9b225e06b2d0d4eb',
u'publicURL':
u'http://172.16.12.21:8776/v1/2bd766a095b44486bf07cf7f666997eb'}],
u'endpoints_links': [],
u'type': u'volume',
u'name': u'cinder'}],
u'_context_auth_token': u'TOK',
u'_context_user_id': u'0a757cd896b64b65ba3784afef564116',
u'payload': {
'instance_id': 'fake-uuid1-1',
'weighted_host': {
'host': 'host3',
'weight': 3.0,
},
'request_spec': {
'instance_properties': {
'root_gb': 512,
'ephemeral_gb': 0,
'launch_index': 0,
'memory_mb': 512,
'vcpus': 1,
'os_type': 'Linux',
'project_id': 1,
'system_metadata': {'system': 'metadata'}},
'instance_type': {'memory_mb': 512,
'vcpus': 1,
'root_gb': 512,
'ephemeral_gb': 0},
'instance_uuids': ['fake-uuid1-1'],
},
},
u'priority': u'INFO'
u'weighted_host': {u'host': u'eglynn-f19-devstack3', u'weight': 1.0},
u'request_spec': {
u'num_instances': 1,
u'block_device_mapping': [{
u'instance_uuid': u'9206baae-c3b6-41bc-96f2-2c0726ff51c8',
u'guest_format': None,
u'boot_index': 0,
u'no_device': None,
u'connection_info': None,
u'volume_id': None,
u'volume_size': None,
u'device_name': None,
u'disk_bus': None,
u'image_id': u'0560ac3f-3bcd-434d-b012-8dd7a212b73b',
u'source_type': u'image',
u'device_type': u'disk',
u'snapshot_id': None,
u'destination_type': u'local',
u'delete_on_termination': True}],
u'image': {
u'status': u'active',
u'name': u'cirros-0.3.1-x86_64-uec',
u'deleted': False,
u'container_format': u'ami',
u'created_at': u'2014-02-18T13:16:26.000000',
u'disk_format': u'ami',
u'updated_at': u'2014-02-18T13:16:27.000000',
u'properties': {
u'kernel_id': u'c8794c1a-4158-42cc-9f97-d0d250c9c6a4',
u'ramdisk_id': u'4999726c-545c-4a9e-bfc0-917459784275'},
u'min_disk': 0,
u'min_ram': 0,
u'checksum': u'f8a2eeee2dc65b3d9b6e63678955bd83',
u'owner': u'2bd766a095b44486bf07cf7f666997eb',
u'is_public': True,
u'deleted_at': None,
u'id': u'0560ac3f-3bcd-434d-b012-8dd7a212b73b',
u'size': 25165824},
u'instance_type': {
u'root_gb': 1,
u'name': u'm1.tiny',
u'ephemeral_gb': 0,
u'memory_mb': 512,
u'vcpus': 1,
u'extra_specs': {},
u'swap': 0,
u'rxtx_factor': 1.0,
u'flavorid': u'1',
u'vcpu_weight': None,
u'id': 2},
u'instance_properties': {
u'vm_state': u'building',
u'availability_zone': None,
u'terminated_at': None,
u'ephemeral_gb': 0,
u'instance_type_id': 2,
u'user_data': None,
u'cleaned': False,
u'vm_mode': None,
u'deleted_at': None,
u'reservation_id': u'r-ven5q6om',
u'id': 15,
u'security_groups': [{
u'deleted_at': None,
u'user_id': u'0a757cd896b64b65ba3784afef564116',
u'description': u'default',
u'deleted': False,
u'created_at': u'2014-02-19T11:02:31.000000',
u'updated_at': None,
u'project_id': u'2bd766a095b44486bf07cf7f666997eb',
u'id': 1,
u'name': u'default'}],
u'disable_terminate': False,
u'root_device_name': None,
u'display_name': u'new',
u'uuid': u'9206baae-c3b6-41bc-96f2-2c0726ff51c8',
u'default_swap_device': None,
u'info_cache': {
u'instance_uuid': u'9206baae-c3b6-41bc-96f2-2c0726ff51c8',
u'deleted': False,
u'created_at': u'2014-03-05T12:44:00.000000',
u'updated_at': None,
u'network_info': [],
u'deleted_at': None},
u'hostname': u'new',
u'launched_on': None,
u'display_description': u'new',
u'key_data': None,
u'deleted': False,
u'config_drive': u'',
u'power_state': 0,
u'default_ephemeral_device': None,
u'progress': 0,
u'project_id': u'2bd766a095b44486bf07cf7f666997eb',
u'launched_at': None,
u'scheduled_at': None,
u'node': None,
u'ramdisk_id': u'4999726c-545c-4a9e-bfc0-917459784275',
u'access_ip_v6': None,
u'access_ip_v4': None,
u'kernel_id': u'c8794c1a-4158-42cc-9f97-d0d250c9c6a4',
u'key_name': None,
u'updated_at': None,
u'host': None,
u'root_gb': 1,
u'user_id': u'0a757cd896b64b65ba3784afef564116',
u'system_metadata': {
u'image_kernel_id':
u'c8794c1a-4158-42cc-9f97-d0d250c9c6a4',
u'image_min_disk': u'1',
u'instance_type_memory_mb': u'512',
u'instance_type_swap': u'0',
u'instance_type_vcpu_weight': None,
u'instance_type_root_gb': u'1',
u'instance_type_name': u'm1.tiny',
u'image_ramdisk_id':
u'4999726c-545c-4a9e-bfc0-917459784275',
u'instance_type_id': u'2',
u'instance_type_ephemeral_gb': u'0',
u'instance_type_rxtx_factor': u'1.0',
u'instance_type_flavorid': u'1',
u'instance_type_vcpus': u'1',
u'image_container_format': u'ami',
u'image_min_ram': u'0',
u'image_disk_format': u'ami',
u'image_base_image_ref':
u'0560ac3f-3bcd-434d-b012-8dd7a212b73b'},
u'task_state': u'scheduling',
u'shutdown_terminate': False,
u'cell_name': None,
u'ephemeral_key_uuid': None,
u'locked': False,
u'name': u'instance-0000000f',
u'created_at': u'2014-03-05T12:44:00.000000',
u'locked_by': None,
u'launch_index': 0,
u'memory_mb': 512,
u'vcpus': 1,
u'image_ref': u'0560ac3f-3bcd-434d-b012-8dd7a212b73b',
u'architecture': None,
u'auto_disk_config': False,
u'os_type': None,
u'metadata': {u'metering.server_group': u'Group_A',
u'AutoScalingGroupName': u'tyky-Group_Awste7',
u'metering.foo.bar': u'true'}},
u'security_group': [u'default'],
u'instance_uuids': [u'9206baae-c3b6-41bc-96f2-2c0726ff51c8']}},
u'priority': u'INFO',
u'_context_is_admin': True,
u'_context_timestamp': u'2014-03-05T12:44:00.135674',
u'publisher_id': u'scheduler.eglynn-f19-devstack3',
u'message_id': u'd6c1ae63-a26b-47c7-8397-8794216e09dd',
u'_context_remote_address': u'172.16.12.21',
u'_context_roles': [u'_member_', u'admin'],
u'timestamp': u'2014-03-05 12:44:00.733758',
u'_context_user': u'0a757cd896b64b65ba3784afef564116',
u'_unique_id': u'2af47cbdde604ff794bb046f3f9db1e2',
u'_context_project_name': u'admin',
u'_context_read_deleted': u'no',
u'_context_tenant': u'2bd766a095b44486bf07cf7f666997eb',
u'_context_instance_lock_checked': False,
u'_context_project_id': u'2bd766a095b44486bf07cf7f666997eb',
u'_context_user_name': u'admin'
}
@ -424,6 +571,14 @@ class TestNotifications(test.BaseTestCase):
def _find_counter(counters, name):
return filter(lambda counter: counter.name == name, counters)[0]
def _verify_user_metadata(self, metadata):
self.assertIn('user_metadata', metadata)
user_meta = metadata['user_metadata']
self.assertEqual(user_meta.get('server_group'), 'Group_A')
self.assertNotIn('AutoScalingGroupName', user_meta)
self.assertIn('foo_bar', user_meta)
self.assertNotIn('foo.bar', user_meta)
def test_instance_create_instance(self):
ic = instance.Instance()
counters = list(ic.process_notification(INSTANCE_CREATE_END))
@ -493,6 +648,7 @@ class TestNotifications(test.BaseTestCase):
self.assertEqual(len(counters), 1)
c = counters[0]
self.assertEqual(c.volume, 1)
self._verify_user_metadata(c.resource_metadata)
def test_instance_finish_resize_flavor(self):
ic = instance.InstanceFlavor()
@ -501,6 +657,7 @@ class TestNotifications(test.BaseTestCase):
c = counters[0]
self.assertEqual(c.volume, 1)
self.assertEqual(c.name, 'instance:m1.small')
self._verify_user_metadata(c.resource_metadata)
def test_instance_finish_resize_memory(self):
ic = instance.Memory()
@ -509,6 +666,7 @@ class TestNotifications(test.BaseTestCase):
c = counters[0]
self.assertEqual(c.volume,
INSTANCE_FINISH_RESIZE_END['payload']['memory_mb'])
self._verify_user_metadata(c.resource_metadata)
def test_instance_finish_resize_vcpus(self):
ic = instance.VCpus()
@ -517,6 +675,7 @@ class TestNotifications(test.BaseTestCase):
c = counters[0]
self.assertEqual(c.volume,
INSTANCE_FINISH_RESIZE_END['payload']['vcpus'])
self._verify_user_metadata(c.resource_metadata)
def test_instance_resize_finish_instance(self):
ic = instance.Instance()
@ -524,6 +683,7 @@ class TestNotifications(test.BaseTestCase):
self.assertEqual(len(counters), 1)
c = counters[0]
self.assertEqual(c.volume, 1)
self._verify_user_metadata(c.resource_metadata)
def test_instance_resize_finish_flavor(self):
ic = instance.InstanceFlavor()
@ -532,6 +692,7 @@ class TestNotifications(test.BaseTestCase):
c = counters[0]
self.assertEqual(c.volume, 1)
self.assertEqual(c.name, 'instance:m1.tiny')
self._verify_user_metadata(c.resource_metadata)
def test_instance_resize_finish_memory(self):
ic = instance.Memory()
@ -540,6 +701,7 @@ class TestNotifications(test.BaseTestCase):
c = counters[0]
self.assertEqual(c.volume,
INSTANCE_RESIZE_REVERT_END['payload']['memory_mb'])
self._verify_user_metadata(c.resource_metadata)
def test_instance_resize_finish_vcpus(self):
ic = instance.VCpus()
@ -548,6 +710,7 @@ class TestNotifications(test.BaseTestCase):
c = counters[0]
self.assertEqual(c.volume,
INSTANCE_RESIZE_REVERT_END['payload']['vcpus'])
self._verify_user_metadata(c.resource_metadata)
def test_instance_delete_samples(self):
ic = instance.InstanceDelete()
@ -555,6 +718,8 @@ class TestNotifications(test.BaseTestCase):
self.assertEqual(len(counters), 2)
names = [c.name for c in counters]
self.assertEqual(names, ['sample-name1', 'sample-name2'])
c = counters[0]
self._verify_user_metadata(c.resource_metadata)
def test_instance_scheduled(self):
ic = instance.InstanceScheduled()
@ -568,3 +733,6 @@ class TestNotifications(test.BaseTestCase):
self.assertEqual(names, ['instance.scheduled'])
rid = [c.resource_id for c in counters]
self.assertEqual(rid, ['fake-uuid1-1'])
c = counters[0]
metadata = c.resource_metadata['request_spec']['instance_properties']
self._verify_user_metadata(metadata)

2
etc/ceilometer/ceilometer.conf.sample
View File

@ -62,7 +62,7 @@
#
# Options defined in ceilometer.compute.pollsters.util
# Options defined in ceilometer.compute.util
#
# List of metadata prefixes reserved for metering use. (list