openstack/aodh
Code Issues Proposed changes

Adds disk iops metrics implementation in Hyper-V Inspector

Browse Source 
Regarding the disk metrics, Hyper-V cannot collect read_requests or
write_requests metrics as the other inspectors do, but it can collect
the disk IOPS instead. This patch is targeted to Windows
Hyper-V / Server 2012 R2, Windows 8.1 or newer, since they introduced
disk metrics collection in Hyper-V.

This patch adds iops metrics implementation in Hyper-V Inspector as
well as disk iops and per disk iops pollsters.

DocImpact

Change-Id: I1c6b5d1f3d8b490f0c839124be18deac86979ae7
Implements: blueprint hyper-v-disk-iops-metrics
This commit is contained in:
Adelina Tuvenie 2015-01-28 09:30:23 -08:00 committed by Eoghan Glynn
parent 9874290a50
commit e5cc81dc15
10 changed files with 219 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

85
ceilometer/compute/pollsters/disk.py
View File

@ -46,6 +46,10 @@ DiskLatencyData = collections.namedtuple('DiskLatencyData',
['disk_latency',
'per_disk_latency'])
DiskIOPSData = collections.namedtuple('DiskIOPSData',
['iops_count',
'per_disk_iops'])
@six.add_metaclass(abc.ABCMeta)
class _Base(pollsters.BaseComputePollster):
@ -550,3 +554,84 @@ class PerDeviceDiskLatencyPollster(_DiskLatencyPollsterBase):
resource_id="%s-%s" % (instance.id, disk)
))
return samples
class _DiskIOPSPollsterBase(pollsters.BaseComputePollster):
CACHE_KEY_DISK_IOPS = 'disk-iops'
def _populate_cache(self, inspector, cache, instance):
i_cache = cache.setdefault(self.CACHE_KEY_DISK_IOPS, {})
if instance.id not in i_cache:
iops = 0
per_device_iops = {}
disk_iops_count = inspector.inspect_disk_iops(instance)
for disk, stats in disk_iops_count:
iops += stats.iops_count
per_device_iops[disk.device] = (stats.iops_count)
per_disk_iops = {
'iops_count': per_device_iops
}
i_cache[instance.id] = DiskIOPSData(
iops,
per_disk_iops
)
return i_cache[instance.id]
@abc.abstractmethod
def _get_samples(self, instance, disk_rates_info):
"""Return one or more Sample."""
def get_samples(self, manager, cache, resources):
for instance in resources:
try:
disk_iops_info = self._populate_cache(
self.inspector,
cache,
instance,
)
for disk_iops in self._get_samples(instance,
disk_iops_info):
yield disk_iops
except virt_inspector.InstanceNotFoundException as err:
# Instance was deleted while getting samples. Ignore it.
LOG.debug(_('Exception while getting samples %s'), err)
except ceilometer.NotImplementedError:
# Selected inspector does not implement this pollster.
LOG.debug(_('%(inspector)s does not provide data for '
'%(pollster)s'),
{'inspector': self.inspector.__class__.__name__,
'pollster': self.__class__.__name__})
except Exception as err:
instance_name = util.instance_name(instance)
LOG.exception(_('Ignoring instance %(name)s: %(error)s'),
{'name': instance_name, 'error': err})
class DiskIOPSPollster(_DiskIOPSPollsterBase):
def _get_samples(self, instance, disk_iops_info):
return [util.make_sample_from_instance(
instance,
name='disk.iops',
type=sample.TYPE_GAUGE,
unit='count/s',
volume=disk_iops_info.iops_count
)]
class PerDeviceDiskIOPSPollster(_DiskIOPSPollsterBase):
def _get_samples(self, instance, disk_iops_info):
samples = []
for disk, value in six.iteritems(disk_iops_info.per_disk_iops[
'iops_count']):
samples.append(util.make_sample_from_instance(
instance,
name='disk.device.iops',
type=sample.TYPE_GAUGE,
unit='count/s',
volume=value,
resource_id="%s-%s" % (instance.id, disk)
))
return samples

9
ceilometer/compute/virt/hyperv/inspector.py
View File

@ -90,3 +90,12 @@ class HyperVInspector(virt_inspector.Inspector):
disk_latency=disk_metrics['disk_latency'])
yield (disk, stats)
def inspect_disk_iops(self, instance):
instance_name = util.instance_name(instance)
for disk_metrics in self._utils.get_disk_iops_count(instance_name):
disk = virt_inspector.Disk(device=disk_metrics['instance_id'])
stats = virt_inspector.DiskIOPSStats(
iops_count=disk_metrics['iops_count'])
yield (disk, stats)

15
ceilometer/compute/virt/hyperv/utilsv2.py
View File

@ -58,6 +58,7 @@ class UtilsV2(object):
_DISK_RD_METRIC_NAME = 'Disk Data Read'
_DISK_WR_METRIC_NAME = 'Disk Data Written'
_DISK_LATENCY_METRIC_NAME = 'Average Disk Latency'
_DISK_IOPS_METRIC_NAME = 'Average Normalized Disk Throughput'
def __init__(self, host='.'):
if sys.platform == 'win32':
@ -167,6 +168,20 @@ class UtilsV2(object):
'instance_id': disk.InstanceID,
}
def get_disk_iops_count(self, vm_name):
vm = self._lookup_vm(vm_name)
metric_def_iops = self._get_metric_def(self._DISK_IOPS_METRIC_NAME)
disks = self._get_vm_resources(vm, self._STORAGE_ALLOC)
for disk in disks:
metric_values = self._get_metric_values(
disk, [metric_def_iops])
yield {
'iops_count': metric_values[0],
'instance_id': disk.InstanceID,
}
@staticmethod
def _sum_metric_values(metrics):
tot_metric_val = 0

15
ceilometer/compute/virt/inspector.py
View File

@ -135,6 +135,13 @@ DiskRateStats = collections.namedtuple('DiskRateStats',
DiskLatencyStats = collections.namedtuple('DiskLatencyStats',
['disk_latency'])
# Named tuple representing disk iops statistics.
#
# iops: number of iops per second
#
DiskIOPSStats = collections.namedtuple('DiskIOPSStats',
['iops_count'])
# Exception types
#
@ -235,6 +242,14 @@ class Inspector(object):
"""
raise ceilometer.NotImplementedError
def inspect_disk_iops(self, instance):
"""Inspect the disk statistics as rates for an instance.
:param instance: the target instance
:return: for each disk, the number of iops per second
"""
raise ceilometer.NotImplementedError
def get_hypervisor_inspector():
try:

4
ceilometer/tests/agent/test_manager.py
View File

@ -35,8 +35,8 @@ class TestManager(base.BaseTestCase):
def test_load_plugins_pollster_list(self):
mgr = manager.AgentManager(pollster_list=['disk.*'])
# currently we do have 18 disk-related pollsters
self.assertEqual(18, len(list(mgr.extensions)))
# currently we do have 20 disk-related pollsters
self.assertEqual(20, len(list(mgr.extensions)))
def test_load_plugins_no_intersection(self):
# Let's test nothing will be polled if namespace and pollsters

48
ceilometer/tests/compute/pollsters/test_diskio.py
View File

@ -318,3 +318,51 @@ class TestDiskLatencyPollsters(TestBaseDiskIO):
self._check_per_device_samples(disk.PerDeviceDiskLatencyPollster,
'disk.device.latency', 2, 'disk2')
class TestDiskIOPSPollsters(TestBaseDiskIO):
DISKS = [
(virt_inspector.Disk(device='disk1'),
virt_inspector.DiskIOPSStats(10)),
(virt_inspector.Disk(device='disk2'),
virt_inspector.DiskIOPSStats(20)),
]
TYPE = 'gauge'
def setUp(self):
super(TestDiskIOPSPollsters, self).setUp()
self.inspector.inspect_disk_iops = mock.Mock(return_value=self.DISKS)
@mock.patch('ceilometer.pipeline.setup_pipeline', mock.MagicMock())
def _check_get_samples(self, factory, sample_name,
expected_count=2):
pollster = factory()
mgr = manager.AgentManager()
cache = {}
samples = list(pollster.get_samples(mgr, cache, self.instance))
self.assertIsNotNone(samples)
self.assertIsNotEmpty(samples)
self.assertIn(pollster.CACHE_KEY_DISK_IOPS, cache)
for instance in self.instance:
self.assertIn(instance.id, cache[pollster.CACHE_KEY_DISK_IOPS])
self.assertEqual(set([sample_name]), set([s.name for s in samples]))
match = [s for s in samples if s.name == sample_name]
self.assertEqual(expected_count, len(match),
'missing counter %s' % sample_name)
return match
def test_disk_iops(self):
self._check_aggregate_samples(disk.DiskIOPSPollster,
'disk.iops', 30L)
def test_per_device_iops(self):
self._check_per_device_samples(disk.PerDeviceDiskIOPSPollster,
'disk.device.iops', 10L, 'disk1')
self._check_per_device_samples(disk.PerDeviceDiskIOPSPollster,
'disk.device.iops', 20L, 'disk2')

20
ceilometer/tests/compute/virt/hyperv/test_inspector.py
View File

@ -134,3 +134,23 @@ class TestHyperVInspection(base.BaseTestCase):
self.assertEqual(fake_instance_id, inspected_disk.device)
self.assertEqual(fake_disk_latency, inspected_stats.disk_latency)
def test_inspect_disk_iops_count(self):
fake_instance_name = mock.sentinel.INSTANCE_NAME
fake_disk_iops_count = mock.sentinel.DISK_IOPS_COUNT
fake_instance_id = mock.sentinel.INSTANCE_ID
self._inspector._utils.get_disk_iops_count.return_value = [{
'iops_count': fake_disk_iops_count,
'instance_id': fake_instance_id}]
inspected_disks = list(self._inspector.inspect_disk_iops(
fake_instance_name))
self.assertEqual(1, len(inspected_disks))
self.assertEqual(2, len(inspected_disks[0]))
inspected_disk, inspected_stats = inspected_disks[0]
self.assertEqual(fake_instance_id, inspected_disk.device)
self.assertEqual(fake_disk_iops_count, inspected_stats.iops_count)

21
ceilometer/tests/compute/virt/hyperv/test_utilsv2.py
View File

@ -193,6 +193,27 @@ class TestUtilsV2(base.BaseTestCase):
self.assertEqual(fake_latency, disk_metrics[0]['disk_latency'])
self.assertEqual(fake_instance_id, disk_metrics[0]['instance_id'])
def test_get_disk_iops_metrics(self):
fake_vm_name = mock.sentinel.VM_NAME
fake_instance_id = mock.sentinel.FAKE_INSTANCE_ID
fake_iops_count = mock.sentinel.FAKE_IOPS_COUNT
self._utils._lookup_vm = mock.MagicMock()
mock_disk = mock.MagicMock()
mock_disk.InstanceID = fake_instance_id
self._utils._get_vm_resources = mock.MagicMock(
return_value=[mock_disk])
self._utils._get_metric_values = mock.MagicMock(
return_value=[fake_iops_count])
disk_metrics = list(self._utils.get_disk_iops_count(fake_vm_name))
self.assertEqual(1, len(disk_metrics))
self.assertEqual(fake_iops_count, disk_metrics[0]['iops_count'])
self.assertEqual(fake_instance_id, disk_metrics[0]['instance_id'])
def test_get_metric_value_instances(self):
mock_el1 = mock.MagicMock()
mock_associator = mock.MagicMock()

2
doc/source/measurements.rst
View File

@ -84,6 +84,7 @@ disk.read.bytes.rate g B/s inst ID p 1, 2, 3,
disk.write.bytes c B inst ID p 1, 2 Volume of writes
disk.write.bytes.rate g B/s inst ID p 1, 2, 3, 4 Average volume of writes
disk.latency g ms inst ID p 2 Average disk latency
disk.iops g count/s inst ID p 2 Average disk iops
disk.device.read.requests c request disk ID p 1, 2 Number of read requests
disk.device.read.requests.rate g request/s disk ID p 1, 2, 3 Average rate of read requests
disk.device.write.requests c request disk ID p 1, 2 Number of write requests
@ -93,6 +94,7 @@ disk.device.read.bytes.rate g B/s disk ID p 1, 2, 3
disk.device.write.bytes c B disk ID p 1, 2 Volume of writes
disk.device.write.bytes.rate g B/s disk ID p 1, 2, 3 Average volume of writes
disk.device.latency g ms disk ID p 2 Average disk latency per device
disk.device.iops g count/s disk ID p 2 Average disk iops per device
disk.root.size g GB inst ID n 1, 2 Size of root disk
disk.ephemeral.size g GB inst ID n 1, 2 Size of ephemeral disk
network.incoming.bytes c B iface ID p 1, 2 Number of incoming bytes

2
setup.cfg
View File

@ -126,6 +126,8 @@ ceilometer.poll.compute =
disk.device.write.bytes.rate = ceilometer.compute.pollsters.disk:PerDeviceWriteBytesRatePollster
disk.latency = ceilometer.compute.pollsters.disk:DiskLatencyPollster
disk.device.latency = ceilometer.compute.pollsters.disk:PerDeviceDiskLatencyPollster
disk.iops = ceilometer.compute.pollsters.disk:DiskIOPSPollster
disk.device.iops = ceilometer.compute.pollsters.disk:PerDeviceDiskIOPSPollster
cpu = ceilometer.compute.pollsters.cpu:CPUPollster
cpu_util = ceilometer.compute.pollsters.cpu:CPUUtilPollster
network.incoming.bytes = ceilometer.compute.pollsters.net:IncomingBytesPollster