VMware vSphere support: Performance Mgr APIs

(*) Added a dependency on 'oslo.vmware' library, in requirements.txt.

(*) Using the vmware library, implemented a wrapper class -
VsphereOperations - for querying performance stats from VC.

(*) Wrote unit test cases for VsphereOperations.

Change-Id: I445bcdb3f8ac240d1799bfd31f8b888982d7d97f
Implements: blueprint vmware-vcenter-server

ceilometer/compute/virt/vmware/vsphere_operations.py

@@ -0,0 +1,217 @@
+# Copyright (c) 2014 VMware, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from oslo.vmware import vim_util
+
+
+PERF_MANAGER_TYPE = "PerformanceManager"
+PERF_COUNTER_PROPERTY = "perfCounter"
+VM_INSTANCE_ID_PROPERTY = 'config.extraConfig["nvp.vm-uuid"].value'
+
+# ESXi Servers sample performance data every 20 seconds. 20-second interval
+# data is called instance data or real-time data. To retrieve instance data,
+# we need to specify a value of 20 seconds for the "PerfQuerySpec.intervalId"
+# property. In that case the "QueryPerf" method operates as a raw data feed
+# that bypasses the vCenter database and instead retrieves performance data
+# from an ESXi host.
+# The following value is time interval for real-time performance stats
+# in seconds and it is not configurable.
+VC_REAL_TIME_SAMPLING_INTERVAL = 20
+
+
+class VsphereOperations(object):
+    """Class to invoke vSphere APIs calls required by various
+       pollsters, collecting data from VMware infrastructure.
+    """
+    def __init__(self, api_session, max_objects):
+        self._api_session = api_session
+        self._max_objects = max_objects
+        # Mapping between "VM's Nova instance Id" -> "VM's MOID"
+        # In case a VM is deployed by Nova, then its name is instance ID.
+        # So this map essentially has VM names as keys.
+        self._vm_moid_lookup_map = {}
+
+        # Mapping from full name -> ID, for VC Performance counters
+        self._perf_counter_id_lookup_map = None
+
+    def _init_vm_moid_lookup_map(self):
+        session = self._api_session
+        result = session.invoke_api(vim_util, "get_objects", session.vim,
+                                    "VirtualMachine", self._max_objects,
+                                    [VM_INSTANCE_ID_PROPERTY],
+                                    False)
+        while result:
+            for vm_object in result.objects:
+                vm_moid = vm_object.obj.value
+                vm_instance_id = vm_object.propSet[0].val
+                if vm_instance_id:
+                    self._vm_moid_lookup_map[vm_instance_id] = vm_moid
+
+            result = session.invoke_api(vim_util, "continue_retrieval",
+                                        session.vim, result)
+
+    def get_vm_moid(self, vm_instance_id):
+        """Method returns VC MOID of the VM by its NOVA instance ID.
+        """
+        if vm_instance_id not in self._vm_moid_lookup_map:
+            self._init_vm_moid_lookup_map()
+
+        return self._vm_moid_lookup_map.get(vm_instance_id, None)
+
+    def _init_perf_counter_id_lookup_map(self):
+
+        # Query details of all the performance counters from VC
+        session = self._api_session
+        client_factory = session.vim.client.factory
+        perf_manager = session.vim.service_content.perfManager
+
+        prop_spec = vim_util.build_property_spec(
+            client_factory, PERF_MANAGER_TYPE, [PERF_COUNTER_PROPERTY])
+
+        obj_spec = vim_util.build_object_spec(
+            client_factory, perf_manager, None)
+
+        filter_spec = vim_util.build_property_filter_spec(
+            client_factory, [prop_spec], [obj_spec])
+
+        options = client_factory.create('ns0:RetrieveOptions')
+        options.maxObjects = 1
+
+        prop_collector = session.vim.service_content.propertyCollector
+        result = session.invoke_api(session.vim, "RetrievePropertiesEx",
+                                    prop_collector, specSet=[filter_spec],
+                                    options=options)
+
+        perf_counter_infos = result.objects[0].propSet[0].val.PerfCounterInfo
+
+        # Extract the counter Id for each counter and populate the map
+        self._perf_counter_id_lookup_map = {}
+        for perf_counter_info in perf_counter_infos:
+
+            counter_group = perf_counter_info.groupInfo.key
+            counter_name = perf_counter_info.nameInfo.key
+            counter_rollup_type = perf_counter_info.rollupType
+            counter_id = perf_counter_info.key
+
+            counter_full_name = (counter_group + ":" + counter_name + ":" +
+                                 counter_rollup_type)
+            self._perf_counter_id_lookup_map[counter_full_name] = counter_id
+
+    def get_perf_counter_id(self, counter_full_name):
+        """Method returns the ID of VC performance counter by its full name.
+
+        A VC performance counter is uniquely identified by the
+        tuple {'Group Name', 'Counter Name', 'Rollup Type'}.
+        It will have an id - counter ID (changes from one VC to another),
+        which is required to query performance stats from that VC.
+        This method returns the ID for a counter,
+        assuming 'CounterFullName' => 'Group Name:CounterName:RollupType'.
+        """
+        if not self._perf_counter_id_lookup_map:
+            self._init_perf_counter_id_lookup_map()
+        return self._perf_counter_id_lookup_map[counter_full_name]
+
+    # TODO(akhils@vmware.com) Move this method to common library
+    # when it gets checked-in
+    def query_vm_property(self, vm_moid, property_name):
+        """Method returns the value of specified property for a VM.
+
+        :param vm_moid: moid of the VM whose property is to be queried
+        :param property_name: path of the property
+        """
+        vm_mobj = vim_util.get_moref(vm_moid, "VirtualMachine")
+        session = self._api_session
+        return session.invoke_api(vim_util, "get_object_property",
+                                  session.vim, vm_mobj, property_name)
+
+    def query_vm_aggregate_stats(self, vm_moid, counter_id):
+        """Method queries the aggregated real-time stat value for a VM.
+
+        This method should be used for aggregate counters.
+
+        :param vm_moid: moid of the VM
+        :param counter_id: id of the perf counter in VC
+        :return: the aggregated stats value for the counter
+        """
+        # For aggregate counters, device_name should be ""
+        stats = self._query_vm_perf_stats(vm_moid, counter_id, "")
+
+        # Performance manager provides the aggregated stats value
+        # with device name -> None
+        return stats.get(None, 0)
+
+    def query_vm_device_stats(self, vm_moid, counter_id):
+        """Method queries the real-time stat values for a VM, for all devices.
+
+        This method should be used for device(non-aggregate) counters.
+
+        :param vm_moid: moid of the VM
+        :param counter_id: id of the perf counter in VC
+        :return: a map containing the stat values keyed by the device ID/name
+        """
+        # For device counters, device_name should be "*" to get stat values
+        # for all devices.
+        stats = self._query_vm_perf_stats(vm_moid, counter_id, "*")
+
+        # For some device counters, in addition to the per device value
+        # the Performance manager also returns the aggregated value.
+        # Just to be consistent, deleting the aggregated value if present.
+        stats.pop(None, None)
+        return stats
+
+    def _query_vm_perf_stats(self, vm_moid, counter_id, device_name):
+        """Method queries the real-time stat values for a VM.
+
+        :param vm_moid: moid of the VM for which stats are needed
+        :param counter_id: id of the perf counter in VC
+        :param device_name: name of the device for which stats are to be
+            queried. For aggregate counters pass empty string ("").
+            For device counters pass "*", if stats are required over all
+            devices.
+        :return: a map containing the stat values keyed by the device ID/name
+        """
+
+        session = self._api_session
+        client_factory = session.vim.client.factory
+
+        # Construct the QuerySpec
+        metric_id = client_factory.create('ns0:PerfMetricId')
+        metric_id.counterId = counter_id
+        metric_id.instance = device_name
+
+        query_spec = client_factory.create('ns0:PerfQuerySpec')
+        query_spec.entity = vim_util.get_moref(vm_moid, "VirtualMachine")
+        query_spec.metricId = [metric_id]
+        query_spec.intervalId = VC_REAL_TIME_SAMPLING_INTERVAL
+        # The following setting ensures that we need only one latest sample
+        query_spec.maxSample = 1
+
+        perf_manager = session.vim.service_content.perfManager
+        perf_stats = session.invoke_api(session.vim, 'QueryPerf', perf_manager,
+                                        querySpec=[query_spec])
+
+        stat_values = {}
+        if perf_stats:
+            entity_metric = perf_stats[0]
+            sample_infos = entity_metric.sampleInfo
+            samples_count = len(sample_infos)
+
+            if samples_count > 0:
+                for metric_series in entity_metric.value:
+                    stat_value = metric_series.value[samples_count - 1]
+                    device_id = metric_series.id.instance
+                    stat_values[device_id] = stat_value
+
+        return stat_values

ceilometer/tests/compute/virt/vmware/test_vsphere_operations.py

@@ -0,0 +1,174 @@
+# Copyright (c) 2014 VMware, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import mock
+from oslo.vmware import api
+
+from ceilometer.compute.virt.vmware import vsphere_operations
+from ceilometer.openstack.common import test
+
+
+class VsphereOperationsTest(test.BaseTestCase):
+
+    def setUp(self):
+        api_session = api.VMwareAPISession("test_server", "test_user",
+                                           "test_password", 0, None,
+                                           create_session=False)
+        api_session._vim = mock.MagicMock()
+        self._vsphere_ops = vsphere_operations.VsphereOperations(api_session,
+                                                                 1000)
+        super(VsphereOperationsTest, self).setUp()
+
+    def test_get_vm_moid(self):
+
+        vm1_moid = "vm-1"
+        vm2_moid = "vm-2"
+        vm1_instance = "0a651a71-142c-4813-aaa6-42e5d5c80d85"
+        vm2_instance = "db1d2533-6bef-4cb2-aef3-920e109f5693"
+
+        def construct_mock_vm_object(vm_moid, vm_instance):
+            vm_object = mock.MagicMock()
+            vm_object.obj.value = vm_moid
+            vm_object.propSet[0].val = vm_instance
+            return vm_object
+
+        def retrieve_props_side_effect(pc, specSet, options):
+            # assert inputs
+            self.assertEqual(self._vsphere_ops._max_objects,
+                             options.maxObjects)
+            self.assertEqual(vsphere_operations.VM_INSTANCE_ID_PROPERTY,
+                             specSet[0].pathSet[0])
+
+            # mock return result
+            vm1 = construct_mock_vm_object(vm1_moid, vm1_instance)
+            vm2 = construct_mock_vm_object(vm2_moid, vm2_instance)
+            result = mock.MagicMock()
+            result.objects.__iter__.return_value = [vm1, vm2]
+            return result
+
+        vim_mock = self._vsphere_ops._api_session._vim
+        vim_mock.RetrievePropertiesEx.side_effect = retrieve_props_side_effect
+        vim_mock.ContinueRetrievePropertiesEx.return_value = None
+
+        vm_moid = self._vsphere_ops.get_vm_moid(vm1_instance)
+        self.assertEqual(vm1_moid, vm_moid)
+
+        vm_moid = self._vsphere_ops.get_vm_moid(vm2_instance)
+        self.assertEqual(vm2_moid, vm_moid)
+
+    def test_query_vm_property(self):
+
+        vm_moid = "vm-21"
+        vm_property_name = "runtime.powerState"
+        vm_property_val = "poweredON"
+
+        def retrieve_props_side_effect(pc, specSet, options):
+            # assert inputs
+            self.assertEqual(vm_moid, specSet[0].obj.value)
+            self.assertEqual(vm_property_name, specSet[0].pathSet[0])
+
+            # mock return result
+            result = mock.MagicMock()
+            result.objects[0].propSet[0].val = vm_property_val
+            return result
+
+        vim_mock = self._vsphere_ops._api_session._vim
+        vim_mock.RetrievePropertiesEx.side_effect = retrieve_props_side_effect
+
+        actual_val = self._vsphere_ops.query_vm_property(vm_moid,
+                                                         vm_property_name)
+        self.assertEqual(vm_property_val, actual_val)
+
+    def test_get_perf_counter_id(self):
+
+        def construct_mock_counter_info(group_name, counter_name, rollup_type,
+                                        counter_id):
+            counter_info = mock.MagicMock()
+            counter_info.groupInfo.key = group_name
+            counter_info.nameInfo.key = counter_name
+            counter_info.rollupType = rollup_type
+            counter_info.key = counter_id
+            return counter_info
+
+        def retrieve_props_side_effect(pc, specSet, options):
+            # assert inputs
+            self.assertEqual(vsphere_operations.PERF_COUNTER_PROPERTY,
+                             specSet[0].pathSet[0])
+
+            # mock return result
+            counter_info1 = construct_mock_counter_info("a", "b", "c", 1)
+            counter_info2 = construct_mock_counter_info("x", "y", "z", 2)
+            result = mock.MagicMock()
+            result.objects[0].propSet[0].val.PerfCounterInfo.__iter__. \
+                return_value = [counter_info1, counter_info2]
+            return result
+
+        vim_mock = self._vsphere_ops._api_session._vim
+        vim_mock.RetrievePropertiesEx.side_effect = retrieve_props_side_effect
+
+        counter_id = self._vsphere_ops.get_perf_counter_id("a:b:c")
+        self.assertEqual(1, counter_id)
+
+        counter_id = self._vsphere_ops.get_perf_counter_id("x:y:z")
+        self.assertEqual(2, counter_id)
+
+    def test_query_vm_stats(self):
+
+        vm_moid = "vm-21"
+        device1 = "device-1"
+        device2 = "device-2"
+        device3 = "device-3"
+        counter_id = 5
+
+        def construct_mock_metric_series(device_name, stat_values):
+            metric_series = mock.MagicMock()
+            metric_series.value = stat_values
+            metric_series.id.instance = device_name
+            return metric_series
+
+        def vim_query_perf_side_effect(perf_manager, querySpec):
+            # assert inputs
+            self.assertEqual(vm_moid, querySpec[0].entity.value)
+            self.assertEqual(counter_id, querySpec[0].metricId[0].counterId)
+            self.assertEqual(vsphere_operations.VC_REAL_TIME_SAMPLING_INTERVAL,
+                             querySpec[0].intervalId)
+
+            # mock return result
+            perf_stats = mock.MagicMock()
+            perf_stats[0].sampleInfo = ["s1", "s2", "s3"]
+            perf_stats[0].value.__iter__.return_value = [
+                construct_mock_metric_series(None, [111, 222, 333]),
+                construct_mock_metric_series(device1, [100, 200, 300]),
+                construct_mock_metric_series(device2, [10, 20, 30]),
+                construct_mock_metric_series(device3, [1, 2, 3])
+            ]
+            return perf_stats
+
+        vim_mock = self._vsphere_ops._api_session._vim
+        vim_mock.QueryPerf.side_effect = vim_query_perf_side_effect
+        ops = self._vsphere_ops
+
+        # test aggregate stat
+        stat_val = ops.query_vm_aggregate_stats(vm_moid, counter_id)
+        self.assertEqual(333, stat_val)
+
+        # test per-device(non-aggregate) stats
+        expected_device_stats = {
+            device1: 300,
+            device2: 30,
+            device3: 3
+        }
+        stats = ops.query_vm_device_stats(vm_moid, counter_id)
+        self.assertEqual(expected_device_stats, stats)

requirements.txt

@@ -13,6 +13,7 @@ lxml>=2.3
 msgpack-python
 MySQL-python
 oslo.config>=1.2.0
+oslo.vmware
 pbr>=0.6,<1.0
 pecan>=0.4.5
 pymongo>=2.4