openstack/aodh
Code Issues Proposed changes

Break up compute pollsters

Browse Source 
Break the compute pollsters up into separate plugins.  This is
submitted as one patch for all of the compute pollsters because
they are moved into a new package to make the files a little easier
to manage.

blueprint one-meter-per-plugin

Change-Id: Ibc6a9c3f73594c94b578f4229e1f9e9c941c7f4f
Signed-off-by: Doug Hellmann <doug.hellmann@dreamhost.com>
This commit is contained in:
Doug Hellmann 2013-07-03 19:33:22 -04:00
parent ced2b691c1
commit a1f3554555
17 changed files with 1240 additions and 799 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

404
ceilometer/compute/pollsters.py
View File

@ -1,404 +0,0 @@
# -*- encoding: utf-8 -*-
#
# Copyright © 2012 eNovance <licensing@enovance.com>
# Copyright © 2012 Red Hat, Inc
#
# Author: Julien Danjou <julien@danjou.info>
# Author: Eoghan Glynn <eglynn@redhat.com>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import collections
import copy
import datetime
from oslo.config import cfg
from ceilometer.compute import plugin
from ceilometer import counter
from ceilometer.openstack.common import log
from ceilometer.openstack.common import timeutils
LOG = log.getLogger(__name__)
INSTANCE_PROPERTIES = [
# Identity properties
'reservation_id',
# Type properties
'architecture',
'OS-EXT-AZ:availability_zone',
'kernel_id',
'os_type',
'ramdisk_id',
# Capacity properties
'disk_gb',
'ephemeral_gb',
'memory_mb',
'root_gb',
'vcpus']
OPTS = [
cfg.ListOpt('reserved_metadata_namespace',
default=['metering.'],
help='list of metadata prefixes reserved for metering use'),
cfg.IntOpt('reserved_metadata_length',
default=256,
help='limit on length of reserved metadata values'),
]
cfg.CONF.register_opts(OPTS)
def _add_reserved_user_metadata(instance, metadata):
limit = cfg.CONF.reserved_metadata_length
user_metadata = {}
for prefix in cfg.CONF.reserved_metadata_namespace:
md = dict(
(k[len(prefix):].replace('.', '_'),
v[:limit] if isinstance(v, basestring) else v)
for k, v in instance.metadata.items()
if (k.startswith(prefix) and
k[len(prefix):].replace('.', '_') not in metadata)
)
user_metadata.update(md)
if user_metadata:
metadata['user_metadata'] = user_metadata
return metadata
def _get_metadata_from_object(instance):
"""Return a metadata dictionary for the instance.
"""
metadata = {
'display_name': instance.name,
'name': getattr(instance, 'OS-EXT-SRV-ATTR:instance_name', u''),
'instance_type': (instance.flavor['id'] if instance.flavor else None),
'host': instance.hostId,
'flavor': instance.flavor,
# Image properties
'image': instance.image,
'image_ref': (instance.image['id'] if instance.image else None),
}
# Images that come through the conductor API in the nova notifier
# plugin will not have links.
if instance.image and instance.image.get('links'):
metadata['image_ref_url'] = instance.image['links'][0]['href']
else:
metadata['image_ref_url'] = None
for name in INSTANCE_PROPERTIES:
metadata[name] = getattr(instance, name, u'')
return _add_reserved_user_metadata(instance, metadata)
def _instance_name(instance):
"""Shortcut to get instance name."""
return getattr(instance, 'OS-EXT-SRV-ATTR:instance_name', None)
def make_counter_from_instance(instance, name, type, unit, volume):
return counter.Counter(
name=name,
type=type,
unit=unit,
volume=volume,
user_id=instance.user_id,
project_id=instance.tenant_id,
resource_id=instance.id,
timestamp=timeutils.isotime(),
resource_metadata=_get_metadata_from_object(instance),
)
class InstancePollster(plugin.ComputePollster):
@staticmethod
def get_counter_names():
# Instance type counter is specific because it includes
# variable. We don't need such format in future
return ['instance', 'instance:*']
def get_counters(self, manager, cache, instance):
yield make_counter_from_instance(instance,
name='instance',
type=counter.TYPE_GAUGE,
unit='instance',
volume=1)
yield make_counter_from_instance(instance,
name='instance:%s' %
instance.flavor['name'],
type=counter.TYPE_GAUGE,
unit='instance',
volume=1)
DiskIOData = collections.namedtuple(
'DiskIOData',
'r_bytes r_requests w_bytes w_requests',
)
class DiskIOPollster(plugin.ComputePollster):
LOG = log.getLogger(__name__ + '.diskio')
DISKIO_USAGE_MESSAGE = ' '.join(["DISKIO USAGE:",
"%s %s:",
"read-requests=%d",
"read-bytes=%d",
"write-requests=%d",
"write-bytes=%d",
"errors=%d",
])
@staticmethod
def get_counter_names():
return ['disk.read.requests',
'disk.read.bytes',
'disk.write.requests',
'disk.write.bytes']
CACHE_KEY_DISK = 'diskio'
def _populate_cache(self, inspector, cache, instance, instance_name):
i_cache = cache.setdefault(self.CACHE_KEY_DISK, {})
if instance_name not in i_cache:
r_bytes = 0
r_requests = 0
w_bytes = 0
w_requests = 0
for disk, info in inspector.inspect_disks(instance_name):
self.LOG.info(self.DISKIO_USAGE_MESSAGE,
instance, disk.device, info.read_requests,
info.read_bytes, info.write_requests,
info.write_bytes, info.errors)
r_bytes += info.read_bytes
r_requests += info.read_requests
w_bytes += info.write_bytes
w_requests += info.write_requests
i_cache[instance_name] = DiskIOData(
r_bytes=r_bytes,
r_requests=r_requests,
w_bytes=w_bytes,
w_requests=w_requests,
)
return i_cache[instance_name]
def get_counters(self, manager, cache, instance):
instance_name = _instance_name(instance)
c_data = self._populate_cache(
manager.inspector,
cache,
instance,
instance_name,
)
try:
yield make_counter_from_instance(instance,
name='disk.read.requests',
type=counter.TYPE_CUMULATIVE,
unit='request',
volume=c_data.r_requests,
)
yield make_counter_from_instance(instance,
name='disk.read.bytes',
type=counter.TYPE_CUMULATIVE,
unit='B',
volume=c_data.r_bytes,
)
yield make_counter_from_instance(instance,
name='disk.write.requests',
type=counter.TYPE_CUMULATIVE,
unit='request',
volume=c_data.w_requests,
)
yield make_counter_from_instance(instance,
name='disk.write.bytes',
type=counter.TYPE_CUMULATIVE,
unit='B',
volume=c_data.w_bytes,
)
except Exception as err:
self.LOG.warning('Ignoring instance %s: %s',
instance_name, err)
self.LOG.exception(err)
class CPUPollster(plugin.ComputePollster):
LOG = log.getLogger(__name__ + '.cpu')
utilization_map = {}
def _get_cpu_util(self, instance, cpu_info):
prev_times = self.utilization_map.get(instance.id)
self.utilization_map[instance.id] = (cpu_info.time,
datetime.datetime.now())
cpu_util = 0.0
if prev_times:
prev_cpu = prev_times[0]
prev_timestamp = prev_times[1]
delta = self.utilization_map[instance.id][1] - prev_timestamp
elapsed = (delta.seconds * (10 ** 6) + delta.microseconds) * 1000
cores_fraction = 1.0 / cpu_info.number
# account for cpu_time being reset when the instance is restarted
time_used = (cpu_info.time - prev_cpu
if prev_cpu <= cpu_info.time else cpu_info.time)
cpu_util = 100 * cores_fraction * time_used / elapsed
return cpu_util
CACHE_KEY_CPU = 'cpu'
def _get_cpu_info(self, inspector, instance_name, cache):
i_cache = cache.setdefault(self.CACHE_KEY_CPU, {})
if instance_name not in i_cache:
i_cache[instance_name] = inspector.inspect_cpus(instance_name)
return i_cache[instance_name]
@staticmethod
def get_counter_names():
return ['cpu', 'cpu_util']
def get_counters(self, manager, cache, instance):
self.LOG.info('checking instance %s', instance.id)
instance_name = _instance_name(instance)
try:
cpu_info = self._get_cpu_info(
manager.inspector,
instance_name,
cache,
)
self.LOG.info("CPUTIME USAGE: %s %d",
instance.__dict__, cpu_info.time)
cpu_util = self._get_cpu_util(instance, cpu_info)
self.LOG.info("CPU UTILIZATION %%: %s %0.2f",
instance.__dict__, cpu_util)
# FIXME(eglynn): once we have a way of configuring which measures
# are published to each sink, we should by default
# disable publishing this derived measure to the
# metering store, only publishing to those sinks
# that specifically need it
yield make_counter_from_instance(instance,
name='cpu_util',
type=counter.TYPE_GAUGE,
unit='%',
volume=cpu_util,
)
yield make_counter_from_instance(instance,
name='cpu',
type=counter.TYPE_CUMULATIVE,
unit='ns',
volume=cpu_info.time,
)
except Exception as err:
self.LOG.error('could not get CPU time for %s: %s',
instance.id, err)
self.LOG.exception(err)
class NetPollster(plugin.ComputePollster):
LOG = log.getLogger(__name__ + '.net')
NET_USAGE_MESSAGE = ' '.join(["NETWORK USAGE:", "%s %s:", "read-bytes=%d",
"write-bytes=%d"])
@staticmethod
def make_vnic_counter(instance, name, type, unit, volume, vnic_data):
metadata = copy.copy(vnic_data)
resource_metadata = dict(zip(metadata._fields, metadata))
resource_metadata['instance_id'] = instance.id
resource_metadata['instance_type'] = \
instance.flavor['id'] if instance.flavor else None
if vnic_data.fref is not None:
rid = vnic_data.fref
else:
instance_name = _instance_name(instance)
rid = "%s-%s-%s" % (instance_name, instance.id, vnic_data.name)
return counter.Counter(
name=name,
type=type,
unit=unit,
volume=volume,
user_id=instance.user_id,
project_id=instance.tenant_id,
resource_id=rid,
timestamp=timeutils.isotime(),
resource_metadata=resource_metadata
)
@staticmethod
def get_counter_names():
return ['network.incoming.bytes',
'network.incoming.packets',
'network.outgoing.bytes',
'network.outgoing.packets']
CACHE_KEY_VNIC = 'vnics'
def _get_vnics_for_instance(self, cache, inspector, instance_name):
i_cache = cache.setdefault(self.CACHE_KEY_VNIC, {})
if instance_name not in i_cache:
i_cache[instance_name] = list(
inspector.inspect_vnics(instance_name)
)
return i_cache[instance_name]
def get_counters(self, manager, cache, instance):
instance_name = _instance_name(instance)
self.LOG.info('checking instance %s', instance.id)
try:
vnics = self._get_vnics_for_instance(
cache,
manager.inspector,
instance_name,
)
for vnic, info in vnics:
self.LOG.info(self.NET_USAGE_MESSAGE, instance_name,
vnic.name, info.rx_bytes, info.tx_bytes)
yield self.make_vnic_counter(instance,
name='network.incoming.bytes',
type=counter.TYPE_CUMULATIVE,
unit='B',
volume=info.rx_bytes,
vnic_data=vnic,
)
yield self.make_vnic_counter(instance,
name='network.outgoing.bytes',
type=counter.TYPE_CUMULATIVE,
unit='B',
volume=info.tx_bytes,
vnic_data=vnic,
)
yield self.make_vnic_counter(instance,
name='network.incoming.packets',
type=counter.TYPE_CUMULATIVE,
unit='packet',
volume=info.rx_packets,
vnic_data=vnic,
)
yield self.make_vnic_counter(instance,
name='network.outgoing.packets',
type=counter.TYPE_CUMULATIVE,
unit='packet',
volume=info.tx_packets,
vnic_data=vnic,
)
except Exception as err:
self.LOG.warning('Ignoring instance %s: %s',
instance_name, err)
self.LOG.exception(err)

15
ceilometer/compute/pollsters/__init__.py Normal file
View File

@ -0,0 +1,15 @@
#
# Copyright 2013 New Dream Network, LLC (DreamHost)
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#

125
ceilometer/compute/pollsters/cpu.py Normal file
View File

@ -0,0 +1,125 @@
# -*- encoding: utf-8 -*-
#
# Copyright © 2012 eNovance <licensing@enovance.com>
# Copyright © 2012 Red Hat, Inc
#
# Author: Julien Danjou <julien@danjou.info>
# Author: Eoghan Glynn <eglynn@redhat.com>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import abc
import datetime
from ceilometer import counter
from ceilometer.compute import plugin
from ceilometer.compute.pollsters import util
from ceilometer.openstack.common import log
LOG = log.getLogger(__name__)
class _Base(plugin.ComputePollster):
CACHE_KEY_CPU = 'cpu'
def _get_cpu_info(self, inspector, instance_name, cache):
i_cache = cache.setdefault(self.CACHE_KEY_CPU, {})
if instance_name not in i_cache:
i_cache[instance_name] = inspector.inspect_cpus(instance_name)
return i_cache[instance_name]
@abc.abstractmethod
def _get_counter(instance, instance_name, cpu_info):
"""Return one Counter."""
def get_counters(self, manager, cache, instance):
LOG.info('checking instance %s', instance.id)
instance_name = util.instance_name(instance)
try:
cpu_info = self._get_cpu_info(
manager.inspector,
instance_name,
cache,
)
yield self._get_counter(
instance,
instance_name,
cpu_info,
)
except Exception as err:
LOG.error('could not get CPU time for %s: %s',
instance.id, err)
LOG.exception(err)
class CPUPollster(_Base):
@staticmethod
def get_counter_names():
return ['cpu']
@staticmethod
def _get_counter(instance, instance_name, cpu_info):
LOG.info("CPUTIME USAGE: %s %d",
instance.__dict__, cpu_info.time)
return util.make_counter_from_instance(
instance,
name='cpu',
type=counter.TYPE_CUMULATIVE,
unit='ns',
volume=cpu_info.time,
)
class CPUUtilPollster(_Base):
# FIXME(eglynn): once we have a way of configuring which measures
# are published to each sink, we should by default
# disable publishing this derived measure to the
# metering store, only publishing to those sinks
# that specifically need it
utilization_map = {}
@staticmethod
def get_counter_names():
return ['cpu_util']
def _get_cpu_util(self, instance, cpu_info):
prev_times = self.utilization_map.get(instance.id)
self.utilization_map[instance.id] = (cpu_info.time,
datetime.datetime.now())
cpu_util = 0.0
if prev_times:
prev_cpu = prev_times[0]
prev_timestamp = prev_times[1]
delta = self.utilization_map[instance.id][1] - prev_timestamp
elapsed = (delta.seconds * (10 ** 6) + delta.microseconds) * 1000
cores_fraction = 1.0 / cpu_info.number
# account for cpu_time being reset when the instance is restarted
time_used = (cpu_info.time - prev_cpu
if prev_cpu <= cpu_info.time else cpu_info.time)
cpu_util = 100 * cores_fraction * time_used / elapsed
return cpu_util
def _get_counter(self, instance, instance_name, cpu_info):
cpu_util = self._get_cpu_util(instance, cpu_info)
LOG.info("CPU UTILIZATION %%: %s %0.2f",
instance.__dict__, cpu_util)
return util.make_counter_from_instance(
instance,
name='cpu_util',
type=counter.TYPE_GAUGE,
unit='%',
volume=cpu_util,
)

160
ceilometer/compute/pollsters/disk.py Normal file
View File

@ -0,0 +1,160 @@
# -*- encoding: utf-8 -*-
#
# Copyright © 2012 eNovance <licensing@enovance.com>
# Copyright © 2012 Red Hat, Inc
#
# Author: Julien Danjou <julien@danjou.info>
# Author: Eoghan Glynn <eglynn@redhat.com>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import abc
import collections
from ceilometer import counter
from ceilometer.compute import plugin
from ceilometer.compute.pollsters import util
from ceilometer.openstack.common import log
LOG = log.getLogger(__name__)
DiskIOData = collections.namedtuple(
'DiskIOData',
'r_bytes r_requests w_bytes w_requests',
)
class _Base(plugin.ComputePollster):
DISKIO_USAGE_MESSAGE = ' '.join(["DISKIO USAGE:",
"%s %s:",
"read-requests=%d",
"read-bytes=%d",
"write-requests=%d",
"write-bytes=%d",
"errors=%d",
])
CACHE_KEY_DISK = 'diskio'
def _populate_cache(self, inspector, cache, instance, instance_name):
i_cache = cache.setdefault(self.CACHE_KEY_DISK, {})
if instance_name not in i_cache:
r_bytes = 0
r_requests = 0
w_bytes = 0
w_requests = 0
for disk, info in inspector.inspect_disks(instance_name):
LOG.info(self.DISKIO_USAGE_MESSAGE,
instance, disk.device, info.read_requests,
info.read_bytes, info.write_requests,
info.write_bytes, info.errors)
r_bytes += info.read_bytes
r_requests += info.read_requests
w_bytes += info.write_bytes
w_requests += info.write_requests
i_cache[instance_name] = DiskIOData(
r_bytes=r_bytes,
r_requests=r_requests,
w_bytes=w_bytes,
w_requests=w_requests,
)
return i_cache[instance_name]
@abc.abstractmethod
def _get_counter(instance, c_data):
"""Return one Counter."""
def get_counters(self, manager, cache, instance):
instance_name = util.instance_name(instance)
c_data = self._populate_cache(
manager.inspector,
cache,
instance,
instance_name,
)
try:
yield self._get_counter(instance, c_data)
except Exception as err:
LOG.warning('Ignoring instance %s: %s',
instance_name, err)
LOG.exception(err)
class ReadRequestsPollster(_Base):
@staticmethod
def get_counter_names():
return ['disk.read.requests']
@staticmethod
def _get_counter(instance, c_data):
return util.make_counter_from_instance(
instance,
name='disk.read.requests',
type=counter.TYPE_CUMULATIVE,
unit='request',
volume=c_data.r_requests,
)
class ReadBytesPollster(_Base):
@staticmethod
def get_counter_names():
return ['disk.read.bytes']
@staticmethod
def _get_counter(instance, c_data):
return util.make_counter_from_instance(
instance,
name='disk.read.bytes',
type=counter.TYPE_CUMULATIVE,
unit='B',
volume=c_data.r_bytes,
)
class WriteRequestsPollster(_Base):
@staticmethod
def get_counter_names():
return ['disk.write.requests']
@staticmethod
def _get_counter(instance, c_data):
return util.make_counter_from_instance(
instance,
name='disk.write.requests',
type=counter.TYPE_CUMULATIVE,
unit='request',
volume=c_data.w_requests,
)
class WriteBytesPollster(_Base):
@staticmethod
def get_counter_names():
return ['disk.write.bytes']
@staticmethod
def _get_counter(instance, c_data):
return util.make_counter_from_instance(
instance,
name='disk.write.bytes',
type=counter.TYPE_CUMULATIVE,
unit='B',
volume=c_data.w_bytes,
)

60
ceilometer/compute/pollsters/instance.py Normal file
View File

@ -0,0 +1,60 @@
# -*- encoding: utf-8 -*-
#
# Copyright © 2012 eNovance <licensing@enovance.com>
# Copyright © 2012 Red Hat, Inc
#
# Author: Julien Danjou <julien@danjou.info>
# Author: Eoghan Glynn <eglynn@redhat.com>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from ceilometer import counter
from ceilometer.compute import plugin
from ceilometer.compute.pollsters import util
class InstancePollster(plugin.ComputePollster):
@staticmethod
def get_counter_names():
return ['instance']
@staticmethod
def get_counters(manager, cache, instance):
yield util.make_counter_from_instance(
instance,
name='instance',
type=counter.TYPE_GAUGE,
unit='instance',
volume=1,
)
class InstanceFlavorPollster(plugin.ComputePollster):
@staticmethod
def get_counter_names():
# Instance type counter is specific because it includes
# variable. We don't need such format in future
return ['instance:*']
@staticmethod
def get_counters(manager, cache, instance):
yield util.make_counter_from_instance(
instance,
name='instance:%s' %
instance.flavor['name'],
type=counter.TYPE_GAUGE,
unit='instance',
volume=1,
)

157
ceilometer/compute/pollsters/net.py Normal file
View File

@ -0,0 +1,157 @@
# -*- encoding: utf-8 -*-
#
# Copyright © 2012 eNovance <licensing@enovance.com>
# Copyright © 2012 Red Hat, Inc
#
# Author: Julien Danjou <julien@danjou.info>
# Author: Eoghan Glynn <eglynn@redhat.com>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import copy
from ceilometer import counter
from ceilometer.compute import plugin
from ceilometer.compute.pollsters import util
from ceilometer.openstack.common import log
from ceilometer.openstack.common import timeutils
LOG = log.getLogger(__name__)
class _Base(plugin.ComputePollster):
NET_USAGE_MESSAGE = ' '.join(["NETWORK USAGE:", "%s %s:", "read-bytes=%d",
"write-bytes=%d"])
@staticmethod
def make_vnic_counter(instance, name, type, unit, volume, vnic_data):
metadata = copy.copy(vnic_data)
resource_metadata = dict(zip(metadata._fields, metadata))
resource_metadata['instance_id'] = instance.id
resource_metadata['instance_type'] = \
instance.flavor['id'] if instance.flavor else None
if vnic_data.fref is not None:
rid = vnic_data.fref
else:
instance_name = util.instance_name(instance)
rid = "%s-%s-%s" % (instance_name, instance.id, vnic_data.name)
return counter.Counter(
name=name,
type=type,
unit=unit,
volume=volume,
user_id=instance.user_id,
project_id=instance.tenant_id,
resource_id=rid,
timestamp=timeutils.isotime(),
resource_metadata=resource_metadata
)
CACHE_KEY_VNIC = 'vnics'
def _get_vnics_for_instance(self, cache, inspector, instance_name):
i_cache = cache.setdefault(self.CACHE_KEY_VNIC, {})
if instance_name not in i_cache:
i_cache[instance_name] = list(
inspector.inspect_vnics(instance_name)
)
return i_cache[instance_name]
def get_counters(self, manager, cache, instance):
instance_name = util.instance_name(instance)
LOG.info('checking instance %s', instance.id)
try:
vnics = self._get_vnics_for_instance(
cache,
manager.inspector,
instance_name,
)
for vnic, info in vnics:
LOG.info(self.NET_USAGE_MESSAGE, instance_name,
vnic.name, info.rx_bytes, info.tx_bytes)
yield self._get_counter(instance, vnic, info)
except Exception as err:
LOG.warning('Ignoring instance %s: %s',
instance_name, err)
LOG.exception(err)
class IncomingBytesPollster(_Base):
@staticmethod
def get_counter_names():
return ['network.incoming.bytes']
def _get_counter(self, instance, vnic, info):
return self.make_vnic_counter(
instance,
name='network.incoming.bytes',
type=counter.TYPE_CUMULATIVE,
unit='B',
volume=info.rx_bytes,
vnic_data=vnic,
)
class IncomingPacketsPollster(_Base):
@staticmethod
def get_counter_names():
return ['network.incoming.packets']
def _get_counter(self, instance, vnic, info):
return self.make_vnic_counter(
instance,
name='network.incoming.packets',
type=counter.TYPE_CUMULATIVE,
unit='packet',
volume=info.rx_packets,
vnic_data=vnic,
)
class OutgoingBytesPollster(_Base):
@staticmethod
def get_counter_names():
return ['network.outgoing.bytes']
def _get_counter(self, instance, vnic, info):
return self.make_vnic_counter(
instance,
name='network.outgoing.bytes',
type=counter.TYPE_CUMULATIVE,
unit='B',
volume=info.tx_bytes,
vnic_data=vnic,
)
class OutgoingPacketsPollster(_Base):
@staticmethod
def get_counter_names():
return ['network.outgoing.packets']
def _get_counter(self, instance, vnic, info):
return self.make_vnic_counter(
instance,
name='network.outgoing.packets',
type=counter.TYPE_CUMULATIVE,
unit='packet',
volume=info.tx_packets,
vnic_data=vnic,
)

115
ceilometer/compute/pollsters/util.py Normal file
View File

@ -0,0 +1,115 @@
# -*- encoding: utf-8 -*-
#
# Copyright © 2012 eNovance <licensing@enovance.com>
# Copyright © 2012 Red Hat, Inc
#
# Author: Julien Danjou <julien@danjou.info>
# Author: Eoghan Glynn <eglynn@redhat.com>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslo.config import cfg
from ceilometer import counter
from ceilometer.openstack.common import timeutils
INSTANCE_PROPERTIES = [
# Identity properties
'reservation_id',
# Type properties
'architecture',
'OS-EXT-AZ:availability_zone',
'kernel_id',
'os_type',
'ramdisk_id',
# Capacity properties
'disk_gb',
'ephemeral_gb',
'memory_mb',
'root_gb',
'vcpus']
OPTS = [
cfg.ListOpt('reserved_metadata_namespace',
default=['metering.'],
help='list of metadata prefixes reserved for metering use'),
cfg.IntOpt('reserved_metadata_length',
default=256,
help='limit on length of reserved metadata values'),
]
cfg.CONF.register_opts(OPTS)
def _add_reserved_user_metadata(instance, metadata):
limit = cfg.CONF.reserved_metadata_length
user_metadata = {}
for prefix in cfg.CONF.reserved_metadata_namespace:
md = dict(
(k[len(prefix):].replace('.', '_'),
v[:limit] if isinstance(v, basestring) else v)
for k, v in instance.metadata.items()
if (k.startswith(prefix) and
k[len(prefix):].replace('.', '_') not in metadata)
)
user_metadata.update(md)
if user_metadata:
metadata['user_metadata'] = user_metadata
return metadata
def _get_metadata_from_object(instance):
"""Return a metadata dictionary for the instance.
"""
metadata = {
'display_name': instance.name,
'name': getattr(instance, 'OS-EXT-SRV-ATTR:instance_name', u''),
'instance_type': (instance.flavor['id'] if instance.flavor else None),
'host': instance.hostId,
'flavor': instance.flavor,
# Image properties
'image': instance.image,
'image_ref': (instance.image['id'] if instance.image else None),
}
# Images that come through the conductor API in the nova notifier
# plugin will not have links.
if instance.image and instance.image.get('links'):
metadata['image_ref_url'] = instance.image['links'][0]['href']
else:
metadata['image_ref_url'] = None
for name in INSTANCE_PROPERTIES:
metadata[name] = getattr(instance, name, u'')
return _add_reserved_user_metadata(instance, metadata)
def make_counter_from_instance(instance, name, type, unit, volume):
return counter.Counter(
name=name,
type=type,
unit=unit,
volume=volume,
user_id=instance.user_id,
project_id=instance.tenant_id,
resource_id=instance.id,
timestamp=timeutils.isotime(),
resource_metadata=_get_metadata_from_object(instance),
)
def instance_name(instance):
"""Shortcut to get instance name."""
return getattr(instance, 'OS-EXT-SRV-ATTR:instance_name', None)

2
etc/ceilometer/ceilometer.conf.sample
View File

@ -37,7 +37,7 @@
#
# Options defined in ceilometer.compute.pollsters
# Options defined in ceilometer.compute.pollsters.util
#
# list of metadata prefixes reserved for metering use (list

16
setup.cfg
View File

@ -51,10 +51,18 @@ ceilometer.collector =
floatingip = ceilometer.network.notifications:FloatingIP
ceilometer.poll.compute =
diskio = ceilometer.compute.pollsters:DiskIOPollster
cpu = ceilometer.compute.pollsters:CPUPollster
net = ceilometer.compute.pollsters:NetPollster
instance = ceilometer.compute.pollsters:InstancePollster
disk.read.requests = ceilometer.compute.pollsters.disk:ReadRequestsPollster
disk.write.requests = ceilometer.compute.pollsters.disk:WriteRequestsPollster
disk.read.bytes = ceilometer.compute.pollsters.disk:ReadBytesPollster
disk.write.bytes = ceilometer.compute.pollsters.disk:WriteBytesPollster
cpu = ceilometer.compute.pollsters.cpu:CPUPollster
cpu_util = ceilometer.compute.pollsters.cpu:CPUUtilPollster
network.incoming.bytes = ceilometer.compute.pollsters.net:IncomingBytesPollster
network.incoming.packets = ceilometer.compute.pollsters.net:IncomingPacketsPollster
network.outgoing.bytes = ceilometer.compute.pollsters.net:OutgoingBytesPollster
network.outgoing.packets = ceilometer.compute.pollsters.net:OutgoingPacketsPollster
instance = ceilometer.compute.pollsters.instance:InstancePollster
instance_flavor = ceilometer.compute.pollsters.instance:InstanceFlavorPollster
ceilometer.poll.central =
network_floatingip = ceilometer.network.floatingip:FloatingIPPollster

0
tests/compute/pollsters/__init__.py Normal file
View File

40
tests/compute/pollsters/base.py Normal file
View File

@ -0,0 +1,40 @@
# -*- encoding: utf-8 -*-
#
# Copyright © 2012 eNovance <licensing@enovance.com>
# Copyright © 2012 Red Hat, Inc
#
# Author: Julien Danjou <julien@danjou.info>
# Author: Eoghan Glynn <eglynn@redhat.com>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import mock
from ceilometer.tests import base as test_base
from ceilometer.compute.virt import inspector as virt_inspector
class TestPollsterBase(test_base.TestCase):
def setUp(self):
super(TestPollsterBase, self).setUp()
self.mox.StubOutWithMock(virt_inspector, 'get_hypervisor_inspector')
self.inspector = self.mox.CreateMock(virt_inspector.Inspector)
virt_inspector.get_hypervisor_inspector().AndReturn(self.inspector)
self.instance = mock.MagicMock()
self.instance.name = 'instance-00000001'
setattr(self.instance, 'OS-EXT-SRV-ATTR:instance_name',
self.instance.name)
self.instance.id = 1
self.instance.flavor = {'name': 'm1.small', 'id': 2, 'vcpus': 1,
'ram': 512, 'disk': 0}

144
tests/compute/pollsters/test_cpu.py Normal file
View File

@ -0,0 +1,144 @@
# -*- encoding: utf-8 -*-
#
# Copyright © 2012 eNovance <licensing@enovance.com>
# Copyright © 2012 Red Hat, Inc
#
# Author: Julien Danjou <julien@danjou.info>
# Author: Eoghan Glynn <eglynn@redhat.com>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import time
import mock
from ceilometer.compute import manager
from ceilometer.compute.pollsters import cpu
from ceilometer.compute.virt import inspector as virt_inspector
from . import base
class TestCPUPollster(base.TestPollsterBase):
def setUp(self):
super(TestCPUPollster, self).setUp()
@mock.patch('ceilometer.pipeline.setup_pipeline', mock.MagicMock())
def test_get_counters(self):
self.inspector.inspect_cpus(self.instance.name).AndReturn(
virt_inspector.CPUStats(time=1 * (10 ** 6), number=2))
self.inspector.inspect_cpus(self.instance.name).AndReturn(
virt_inspector.CPUStats(time=3 * (10 ** 6), number=2))
# cpu_time resets on instance restart
self.inspector.inspect_cpus(self.instance.name).AndReturn(
virt_inspector.CPUStats(time=2 * (10 ** 6), number=2))
self.mox.ReplayAll()
mgr = manager.AgentManager()
pollster = cpu.CPUPollster()
def _verify_cpu_metering(expected_time):
cache = {}
counters = list(pollster.get_counters(mgr, cache, self.instance))
self.assertEquals(len(counters), 1)
self.assertEqual(set([c.name for c in counters]),
set(pollster.get_counter_names()))
assert counters[0].name == 'cpu'
assert counters[0].volume == expected_time
assert pollster.CACHE_KEY_CPU in cache
assert self.instance.name in cache[pollster.CACHE_KEY_CPU]
# ensure elapsed time between polling cycles is non-zero
time.sleep(0.001)
_verify_cpu_metering(1 * (10 ** 6))
_verify_cpu_metering(3 * (10 ** 6))
_verify_cpu_metering(2 * (10 ** 6))
@mock.patch('ceilometer.pipeline.setup_pipeline', mock.MagicMock())
def test_get_counters_cache(self):
self.mox.ReplayAll()
mgr = manager.AgentManager()
pollster = cpu.CPUPollster()
cache = {
pollster.CACHE_KEY_CPU: {
self.instance.name: virt_inspector.CPUStats(
time=10 ** 6,
number=2,
),
},
}
counters = list(pollster.get_counters(mgr, cache, self.instance))
self.assertEquals(len(counters), 1)
self.assertEquals(counters[0].volume, 10 ** 6)
class TestCPUUtilPollster(base.TestPollsterBase):
def setUp(self):
super(TestCPUUtilPollster, self).setUp()
@mock.patch('ceilometer.pipeline.setup_pipeline', mock.MagicMock())
def test_get_counters(self):
self.inspector.inspect_cpus(self.instance.name).AndReturn(
virt_inspector.CPUStats(time=1 * (10 ** 6), number=2))
self.inspector.inspect_cpus(self.instance.name).AndReturn(
virt_inspector.CPUStats(time=3 * (10 ** 6), number=2))
# cpu_time resets on instance restart
self.inspector.inspect_cpus(self.instance.name).AndReturn(
virt_inspector.CPUStats(time=2 * (10 ** 6), number=2))
self.mox.ReplayAll()
mgr = manager.AgentManager()
pollster = cpu.CPUUtilPollster()
pollster.utilization_map = {} # clear the internal cache
def _verify_cpu_metering(zero):
cache = {}
counters = list(pollster.get_counters(mgr, cache, self.instance))
self.assertEquals(len(counters), 1)
self.assertEqual(set([c.name for c in counters]),
set(pollster.get_counter_names()))
assert counters[0].name == 'cpu_util'
assert (counters[0].volume == 0.0 if zero else
counters[0].volume > 0.0)
assert pollster.CACHE_KEY_CPU in cache
assert self.instance.name in cache[pollster.CACHE_KEY_CPU]
# ensure elapsed time between polling cycles is non-zero
time.sleep(0.001)
_verify_cpu_metering(True)
_verify_cpu_metering(False)
_verify_cpu_metering(False)
@mock.patch('ceilometer.pipeline.setup_pipeline', mock.MagicMock())
def test_get_counters_cache(self):
self.mox.ReplayAll()
mgr = manager.AgentManager()
pollster = cpu.CPUUtilPollster()
pollster.utilization_map = {} # clear the internal cache
cache = {
pollster.CACHE_KEY_CPU: {
self.instance.name: virt_inspector.CPUStats(
time=10 ** 6,
number=2,
),
},
}
counters = list(pollster.get_counters(mgr, cache, self.instance))
self.assertEquals(len(counters), 1)
self.assertEquals(counters[0].volume, 0)

77
tests/compute/pollsters/test_diskio.py Normal file
View File

@ -0,0 +1,77 @@
# -*- encoding: utf-8 -*-
#
# Copyright © 2012 eNovance <licensing@enovance.com>
# Copyright © 2012 Red Hat, Inc
#
# Author: Julien Danjou <julien@danjou.info>
# Author: Eoghan Glynn <eglynn@redhat.com>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import mock
from ceilometer.compute import manager
from ceilometer.compute.pollsters import disk
from ceilometer.compute.virt import inspector as virt_inspector
from . import base
class TestDiskPollsters(base.TestPollsterBase):
DISKS = [
(virt_inspector.Disk(device='vda'),
virt_inspector.DiskStats(read_bytes=1L, read_requests=2L,
write_bytes=3L, write_requests=4L,
errors=-1L))
]
def setUp(self):
super(TestDiskPollsters, self).setUp()
self.inspector.inspect_disks(self.instance.name).AndReturn(self.DISKS)
self.mox.ReplayAll()
@mock.patch('ceilometer.pipeline.setup_pipeline', mock.MagicMock())
def _check_get_counters(self, factory, name, expected_volume):
pollster = factory()
mgr = manager.AgentManager()
cache = {}
counters = list(pollster.get_counters(mgr, cache, self.instance))
assert counters
assert pollster.CACHE_KEY_DISK in cache
assert self.instance.name in cache[pollster.CACHE_KEY_DISK]
self.assertEqual(set([c.name for c in counters]),
set(pollster.get_counter_names()))
match = [c for c in counters if c.name == name]
self.assertEquals(len(match), 1, 'missing counter %s' % name)
self.assertEquals(match[0].volume, expected_volume)
self.assertEquals(match[0].type, 'cumulative')
def test_disk_read_requests(self):
self._check_get_counters(disk.ReadRequestsPollster,
'disk.read.requests', 2L)
def test_disk_read_bytes(self):
self._check_get_counters(disk.ReadBytesPollster,
'disk.read.bytes', 1L)
def test_disk_write_requests(self):
self._check_get_counters(disk.WriteRequestsPollster,
'disk.write.requests', 4L)
def test_disk_write_bytes(self):
self._check_get_counters(disk.WriteBytesPollster,
'disk.write.bytes', 3L)

52
tests/compute/pollsters/test_instance.py Normal file
View File

@ -0,0 +1,52 @@
# -*- encoding: utf-8 -*-
#
# Copyright © 2012 eNovance <licensing@enovance.com>
# Copyright © 2012 Red Hat, Inc
#
# Author: Julien Danjou <julien@danjou.info>
# Author: Eoghan Glynn <eglynn@redhat.com>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import mock
from ceilometer.compute import manager
from ceilometer.compute.pollsters import instance as pollsters_instance
from . import base
class TestInstancePollster(base.TestPollsterBase):
def setUp(self):
super(TestInstancePollster, self).setUp()
@mock.patch('ceilometer.pipeline.setup_pipeline', mock.MagicMock())
def test_get_counters_instance(self):
self.mox.ReplayAll()
mgr = manager.AgentManager()
pollster = pollsters_instance.InstancePollster()
counters = list(pollster.get_counters(mgr, {}, self.instance))
self.assertEquals(len(counters), 1)
self.assertEqual(counters[0].name, 'instance')
@mock.patch('ceilometer.pipeline.setup_pipeline', mock.MagicMock())
def test_get_counters_instance_flavor(self):
self.mox.ReplayAll()
mgr = manager.AgentManager()
pollster = pollsters_instance.InstanceFlavorPollster()
counters = list(pollster.get_counters(mgr, {}, self.instance))
self.assertEquals(len(counters), 1)
self.assertEqual(counters[0].name, 'instance:m1.small')

103
tests/compute/pollsters/test_location_metadata.py Normal file
View File

@ -0,0 +1,103 @@
# -*- encoding: utf-8 -*-
#
# Copyright © 2012 eNovance <licensing@enovance.com>
# Copyright © 2012 Red Hat, Inc
#
# Author: Julien Danjou <julien@danjou.info>
# Author: Eoghan Glynn <eglynn@redhat.com>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Tests for the compute pollsters.
"""
import mock
from ceilometer.compute import manager
from ceilometer.compute.pollsters import util
from ceilometer.tests import base as test_base
class FauxInstance(object):
def __init__(self, **kwds):
for name, value in kwds.items():
setattr(self, name, value)
def __getitem__(self, key):
return getattr(self, key)
def get(self, key, default):
try:
return getattr(self, key)
except AttributeError:
return default
class TestLocationMetadata(test_base.TestCase):
@mock.patch('ceilometer.pipeline.setup_pipeline', mock.MagicMock())
def setUp(self):
self.manager = manager.AgentManager()
super(TestLocationMetadata, self).setUp()
# Mimics an instance returned from nova api call
self.INSTANCE_PROPERTIES = {'name': 'display name',
'OS-EXT-SRV-ATTR:instance_name':
'instance-000001',
'OS-EXT-AZ:availability_zone':
'foo-zone',
'reservation_id': 'reservation id',
'architecture': 'x86_64',
'kernel_id': 'kernel id',
'os_type': 'linux',
'ramdisk_id': 'ramdisk id',
'ephemeral_gb': 7,
'root_gb': 3,
'image': {'id': 1,
'links': [{"rel": "bookmark",
'href': 2}]},
'hostId': '1234-5678',
'flavor': {'id': 1,
'disk': 0,
'ram': 512,
'vcpus': 2},
'metadata': {'metering.autoscale.group':
'X' * 512,
'metering.ephemeral_gb': 42}}
self.instance = FauxInstance(**self.INSTANCE_PROPERTIES)
def test_metadata(self):
md = util._get_metadata_from_object(self.instance)
for prop, value in self.INSTANCE_PROPERTIES.iteritems():
if prop not in ("metadata"):
# Special cases
if prop == 'name':
prop = 'display_name'
elif prop == 'hostId':
prop = "host"
elif prop == 'OS-EXT-SRV-ATTR:instance_name':
prop = 'name'
self.assertEqual(md[prop], value)
user_metadata = md['user_metadata']
expected = self.INSTANCE_PROPERTIES[
'metadata']['metering.autoscale.group'][:256]
self.assertEqual(user_metadata['autoscale_group'], expected)
self.assertEqual(len(user_metadata), 1)
def test_metadata_empty_image(self):
self.INSTANCE_PROPERTIES['image'] = ''
self.instance = FauxInstance(**self.INSTANCE_PROPERTIES)
md = util._get_metadata_from_object(self.instance)
self.assertEqual(md['image_ref'], None)
self.assertEqual(md['image_ref_url'], None)

179
tests/compute/pollsters/test_net.py Normal file
View File

@ -0,0 +1,179 @@
# -*- encoding: utf-8 -*-
#
# Copyright © 2012 eNovance <licensing@enovance.com>
# Copyright © 2012 Red Hat, Inc
#
# Author: Julien Danjou <julien@danjou.info>
# Author: Eoghan Glynn <eglynn@redhat.com>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import mock
from ceilometer.compute import manager
from ceilometer.compute.pollsters import net
from ceilometer.compute.virt import inspector as virt_inspector
from . import base
class TestNetPollster(base.TestPollsterBase):
def setUp(self):
super(TestNetPollster, self).setUp()
self.vnic0 = virt_inspector.Interface(
name='vnet0',
fref='fa163e71ec6e',
mac='fa:16:3e:71:ec:6d',
parameters=dict(ip='10.0.0.2',
projmask='255.255.255.0',
projnet='proj1',
dhcp_server='10.0.0.1'))
stats0 = virt_inspector.InterfaceStats(rx_bytes=1L, rx_packets=2L,
tx_bytes=3L, tx_packets=4L)
self.vnic1 = virt_inspector.Interface(
name='vnet1',
fref='fa163e71ec6f',
mac='fa:16:3e:71:ec:6e',
parameters=dict(ip='192.168.0.3',
projmask='255.255.255.0',
projnet='proj2',
dhcp_server='10.0.0.2'))
stats1 = virt_inspector.InterfaceStats(rx_bytes=5L, rx_packets=6L,
tx_bytes=7L, tx_packets=8L)
self.vnic2 = virt_inspector.Interface(
name='vnet2',
fref=None,
mac='fa:18:4e:72:fc:7e',
parameters=dict(ip='192.168.0.4',
projmask='255.255.255.0',
projnet='proj3',
dhcp_server='10.0.0.3'))
stats2 = virt_inspector.InterfaceStats(rx_bytes=9L, rx_packets=10L,
tx_bytes=11L, tx_packets=12L)
vnics = [
(self.vnic0, stats0),
(self.vnic1, stats1),
(self.vnic2, stats2),
]
self.inspector.inspect_vnics(self.instance.name).AndReturn(vnics)
self.mox.ReplayAll()
@mock.patch('ceilometer.pipeline.setup_pipeline', mock.MagicMock())
def _check_get_counters(self, factory, expected):
mgr = manager.AgentManager()
pollster = factory()
counters = list(pollster.get_counters(mgr, {}, self.instance))
self.assertEqual(len(counters), 3) # one for each nic
self.assertEqual(set([c.name for c in counters]),
set(pollster.get_counter_names()))
def _verify_vnic_metering(ip, expected_volume, expected_rid):
match = [c for c in counters
if c.resource_metadata['parameters']['ip'] == ip
]
self.assertEquals(len(match), 1, 'missing ip %s' % ip)
self.assertEquals(match[0].volume, expected_volume)
self.assertEquals(match[0].type, 'cumulative')
self.assertEquals(match[0].resource_id, expected_rid)
for ip, volume, rid in expected:
_verify_vnic_metering(ip, volume, rid)
def test_incoming_bytes(self):
instance_name_id = "%s-%s" % (self.instance.name, self.instance.id)
self._check_get_counters(
net.IncomingBytesPollster,
[('10.0.0.2', 1L, self.vnic0.fref),
('192.168.0.3', 5L, self.vnic1.fref),
('192.168.0.4', 9L,
"%s-%s" % (instance_name_id, self.vnic2.name)),
],
)
def test_outgoing_bytes(self):
instance_name_id = "%s-%s" % (self.instance.name, self.instance.id)
self._check_get_counters(
net.OutgoingBytesPollster,
[('10.0.0.2', 3L, self.vnic0.fref),
('192.168.0.3', 7L, self.vnic1.fref),
('192.168.0.4', 11L,
"%s-%s" % (instance_name_id, self.vnic2.name)),
],
)
def test_incoming_packets(self):
instance_name_id = "%s-%s" % (self.instance.name, self.instance.id)
self._check_get_counters(
net.IncomingPacketsPollster,
[('10.0.0.2', 2L, self.vnic0.fref),
('192.168.0.3', 6L, self.vnic1.fref),
('192.168.0.4', 10L,
"%s-%s" % (instance_name_id, self.vnic2.name)),
],
)
def test_outgoing_packets(self):
instance_name_id = "%s-%s" % (self.instance.name, self.instance.id)
self._check_get_counters(
net.OutgoingPacketsPollster,
[('10.0.0.2', 4L, self.vnic0.fref),
('192.168.0.3', 8L, self.vnic1.fref),
('192.168.0.4', 12L,
"%s-%s" % (instance_name_id, self.vnic2.name)),
],
)
class TestNetPollsterCache(base.TestPollsterBase):
def setUp(self):
super(TestNetPollsterCache, self).setUp()
self.mox.ReplayAll()
@mock.patch('ceilometer.pipeline.setup_pipeline', mock.MagicMock())
def _check_get_counters_cache(self, factory):
vnic0 = virt_inspector.Interface(
name='vnet0',
fref='fa163e71ec6e',
mac='fa:16:3e:71:ec:6d',
parameters=dict(ip='10.0.0.2',
projmask='255.255.255.0',
projnet='proj1',
dhcp_server='10.0.0.1'))
stats0 = virt_inspector.InterfaceStats(rx_bytes=1L, rx_packets=2L,
tx_bytes=3L, tx_packets=4L)
vnics = [(vnic0, stats0)]
mgr = manager.AgentManager()
pollster = factory()
cache = {
pollster.CACHE_KEY_VNIC: {
self.instance.name: vnics,
},
}
counters = list(pollster.get_counters(mgr, cache, self.instance))
self.assertEqual(len(counters), 1)
def test_incoming_bytes(self):
self._check_get_counters_cache(net.IncomingBytesPollster)
def test_outgoing_bytes(self):
self._check_get_counters_cache(net.OutgoingBytesPollster)
def test_incoming_packets(self):
self._check_get_counters_cache(net.IncomingPacketsPollster)
def test_outgoing_packets(self):
self._check_get_counters_cache(net.OutgoingPacketsPollster)

390
tests/compute/test_pollsters.py
View File

@ -1,390 +0,0 @@
# -*- encoding: utf-8 -*-
#
# Copyright © 2012 eNovance <licensing@enovance.com>
# Copyright © 2012 Red Hat, Inc
#
# Author: Julien Danjou <julien@danjou.info>
# Author: Eoghan Glynn <eglynn@redhat.com>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Tests for the compute pollsters.
"""
import mock
import time
from ceilometer.compute import manager
from ceilometer.compute import pollsters
from ceilometer.compute.virt import inspector as virt_inspector
from ceilometer.tests import base as test_base
class FauxInstance(object):
def __init__(self, **kwds):
for name, value in kwds.items():
setattr(self, name, value)
def __getitem__(self, key):
return getattr(self, key)
def get(self, key, default):
try:
return getattr(self, key)
except AttributeError:
return default
class TestLocationMetadata(test_base.TestCase):
@mock.patch('ceilometer.pipeline.setup_pipeline', mock.MagicMock())
def setUp(self):
self.manager = manager.AgentManager()
super(TestLocationMetadata, self).setUp()
# Mimics an instance returned from nova api call
self.INSTANCE_PROPERTIES = {'name': 'display name',
'OS-EXT-SRV-ATTR:instance_name':
'instance-000001',
'OS-EXT-AZ:availability_zone':
'foo-zone',
'reservation_id': 'reservation id',
'architecture': 'x86_64',
'kernel_id': 'kernel id',
'os_type': 'linux',
'ramdisk_id': 'ramdisk id',
'ephemeral_gb': 7,
'root_gb': 3,
'image': {'id': 1,
'links': [{"rel": "bookmark",
'href': 2}]},
'hostId': '1234-5678',
'flavor': {'id': 1,
'disk': 0,
'ram': 512,
'vcpus': 2},
'metadata': {'metering.autoscale.group':
'X' * 512,
'metering.ephemeral_gb': 42}}
self.instance = FauxInstance(**self.INSTANCE_PROPERTIES)
def test_metadata(self):
md = pollsters._get_metadata_from_object(self.instance)
for prop, value in self.INSTANCE_PROPERTIES.iteritems():
if prop not in ("metadata"):
# Special cases
if prop == 'name':
prop = 'display_name'
elif prop == 'hostId':
prop = "host"
elif prop == 'OS-EXT-SRV-ATTR:instance_name':
prop = 'name'
self.assertEqual(md[prop], value)
user_metadata = md['user_metadata']
expected = self.INSTANCE_PROPERTIES[
'metadata']['metering.autoscale.group'][:256]
self.assertEqual(user_metadata['autoscale_group'], expected)
self.assertEqual(len(user_metadata), 1)
def test_metadata_empty_image(self):
self.INSTANCE_PROPERTIES['image'] = ''
self.instance = FauxInstance(**self.INSTANCE_PROPERTIES)
md = pollsters._get_metadata_from_object(self.instance)
self.assertEqual(md['image_ref'], None)
self.assertEqual(md['image_ref_url'], None)
class TestPollsterBase(test_base.TestCase):
def setUp(self):
super(TestPollsterBase, self).setUp()
self.mox.StubOutWithMock(virt_inspector, 'get_hypervisor_inspector')
self.inspector = self.mox.CreateMock(virt_inspector.Inspector)
virt_inspector.get_hypervisor_inspector().AndReturn(self.inspector)
self.instance = mock.MagicMock()
self.instance.name = 'instance-00000001'
setattr(self.instance, 'OS-EXT-SRV-ATTR:instance_name',
self.instance.name)
self.instance.id = 1
self.instance.flavor = {'name': 'm1.small', 'id': 2, 'vcpus': 1,
'ram': 512, 'disk': 0}
class TestInstancePollster(TestPollsterBase):
def setUp(self):
super(TestInstancePollster, self).setUp()
@mock.patch('ceilometer.pipeline.setup_pipeline', mock.MagicMock())
def test_get_counters(self):
self.mox.ReplayAll()
mgr = manager.AgentManager()
pollster = pollsters.InstancePollster()
counters = list(pollster.get_counters(mgr, {}, self.instance))
self.assertEquals(len(counters), 2)
self.assertEqual(counters[0].name, 'instance')
self.assertEqual(counters[1].name, 'instance:m1.small')
class TestDiskIOPollster(TestPollsterBase):
def setUp(self):
super(TestDiskIOPollster, self).setUp()
@mock.patch('ceilometer.pipeline.setup_pipeline', mock.MagicMock())
def test_get_counters(self):
disks = [
(virt_inspector.Disk(device='vda'),
virt_inspector.DiskStats(read_bytes=1L, read_requests=2L,
write_bytes=3L, write_requests=4L,
errors=-1L))
]
self.inspector.inspect_disks(self.instance.name).AndReturn(disks)
self.mox.ReplayAll()
mgr = manager.AgentManager()
pollster = pollsters.DiskIOPollster()
cache = {}
counters = list(pollster.get_counters(mgr, cache, self.instance))
assert counters
assert pollster.CACHE_KEY_DISK in cache
assert self.instance.name in cache[pollster.CACHE_KEY_DISK]
self.assertEqual(set([c.name for c in counters]),
set(pollster.get_counter_names()))
def _verify_disk_metering(name, expected_volume):
match = [c for c in counters if c.name == name]
self.assertEquals(len(match), 1, 'missing counter %s' % name)
self.assertEquals(match[0].volume, expected_volume)
self.assertEquals(match[0].type, 'cumulative')
_verify_disk_metering('disk.read.requests', 2L)
_verify_disk_metering('disk.read.bytes', 1L)
_verify_disk_metering('disk.write.requests', 4L)
_verify_disk_metering('disk.write.bytes', 3L)
@mock.patch('ceilometer.pipeline.setup_pipeline', mock.MagicMock())
def test_get_counters_cache(self):
self.mox.ReplayAll()
mgr = manager.AgentManager()
pollster = pollsters.DiskIOPollster()
cache = {
pollster.CACHE_KEY_DISK: {
self.instance.name: pollsters.DiskIOData(
r_bytes=-1,
r_requests=-2,
w_bytes=-3,
w_requests=-4,
),
},
}
counters = list(pollster.get_counters(mgr, cache, self.instance))
assert counters
self.assertEqual(set([c.name for c in counters]),
set(pollster.get_counter_names()))
def _verify_disk_metering(name, expected_volume):
match = [c for c in counters if c.name == name]
self.assertEquals(len(match), 1, 'missing counter %s' % name)
self.assertEquals(match[0].volume, expected_volume)
self.assertEquals(match[0].type, 'cumulative')
_verify_disk_metering('disk.read.requests', -2L)
_verify_disk_metering('disk.read.bytes', -1L)
_verify_disk_metering('disk.write.requests', -4L)
_verify_disk_metering('disk.write.bytes', -3L)
class TestNetPollster(TestPollsterBase):
def setUp(self):
super(TestNetPollster, self).setUp()
@mock.patch('ceilometer.pipeline.setup_pipeline', mock.MagicMock())
def test_get_counters(self):
vnic0 = virt_inspector.Interface(
name='vnet0',
fref='fa163e71ec6e',
mac='fa:16:3e:71:ec:6d',
parameters=dict(ip='10.0.0.2',
projmask='255.255.255.0',
projnet='proj1',
dhcp_server='10.0.0.1'))
stats0 = virt_inspector.InterfaceStats(rx_bytes=1L, rx_packets=2L,
tx_bytes=3L, tx_packets=4L)
vnic1 = virt_inspector.Interface(
name='vnet1',
fref='fa163e71ec6f',
mac='fa:16:3e:71:ec:6e',
parameters=dict(ip='192.168.0.3',
projmask='255.255.255.0',
projnet='proj2',
dhcp_server='10.0.0.2'))
stats1 = virt_inspector.InterfaceStats(rx_bytes=5L, rx_packets=6L,
tx_bytes=7L, tx_packets=8L)
vnic2 = virt_inspector.Interface(
name='vnet2',
fref=None,
mac='fa:18:4e:72:fc:7e',
parameters=dict(ip='192.168.0.4',
projmask='255.255.255.0',
projnet='proj3',
dhcp_server='10.0.0.3'))
stats2 = virt_inspector.InterfaceStats(rx_bytes=9L, rx_packets=10L,
tx_bytes=11L, tx_packets=12L)
vnics = [(vnic0, stats0), (vnic1, stats1), (vnic2, stats2)]
self.inspector.inspect_vnics(self.instance.name).AndReturn(vnics)
self.mox.ReplayAll()
mgr = manager.AgentManager()
pollster = pollsters.NetPollster()
counters = list(pollster.get_counters(mgr, {}, self.instance))
assert counters
self.assertEqual(set([c.name for c in counters]),
set(pollster.get_counter_names()))
def _verify_vnic_metering(name, ip, expected_volume, expected_rid):
match = [c for c in counters if c.name == name and
c.resource_metadata['parameters']['ip'] == ip]
self.assertEquals(len(match), 1, 'missing counter %s' % name)
self.assertEquals(match[0].volume, expected_volume)
self.assertEquals(match[0].type, 'cumulative')
self.assertEquals(match[0].resource_id, expected_rid)
instance_name_id = "%s-%s" % (self.instance.name, self.instance.id)
_verify_vnic_metering('network.incoming.bytes', '10.0.0.2', 1L,
vnic0.fref)
_verify_vnic_metering('network.incoming.bytes', '192.168.0.3', 5L,
vnic1.fref)
_verify_vnic_metering('network.incoming.bytes', '192.168.0.4', 9L,
"%s-%s" % (instance_name_id, vnic2.name))
_verify_vnic_metering('network.outgoing.bytes', '10.0.0.2', 3L,
vnic0.fref)
_verify_vnic_metering('network.outgoing.bytes', '192.168.0.3', 7L,
vnic1.fref)
_verify_vnic_metering('network.outgoing.bytes', '192.168.0.4', 11L,
"%s-%s" % (instance_name_id, vnic2.name))
_verify_vnic_metering('network.incoming.packets', '10.0.0.2', 2L,
vnic0.fref)
_verify_vnic_metering('network.incoming.packets', '192.168.0.3', 6L,
vnic1.fref)
_verify_vnic_metering('network.incoming.packets', '192.168.0.4', 10L,
"%s-%s" % (instance_name_id, vnic2.name))
_verify_vnic_metering('network.outgoing.packets', '10.0.0.2', 4L,
vnic0.fref)
_verify_vnic_metering('network.outgoing.packets', '192.168.0.3', 8L,
vnic1.fref)
_verify_vnic_metering('network.outgoing.packets', '192.168.0.4', 12L,
"%s-%s" % (instance_name_id, vnic2.name))
@mock.patch('ceilometer.pipeline.setup_pipeline', mock.MagicMock())
def test_get_counters_cache(self):
vnic0 = virt_inspector.Interface(
name='vnet0',
fref='fa163e71ec6e',
mac='fa:16:3e:71:ec:6d',
parameters=dict(ip='10.0.0.2',
projmask='255.255.255.0',
projnet='proj1',
dhcp_server='10.0.0.1'))
stats0 = virt_inspector.InterfaceStats(rx_bytes=1L, rx_packets=2L,
tx_bytes=3L, tx_packets=4L)
vnics = [(vnic0, stats0)]
self.mox.ReplayAll()
mgr = manager.AgentManager()
pollster = pollsters.NetPollster()
cache = {
pollster.CACHE_KEY_VNIC: {
self.instance.name: vnics,
},
}
counters = list(pollster.get_counters(mgr, cache, self.instance))
assert counters
# We should have one of each counter for one vnic
self.assertEqual(len(counters), 4)
class TestCPUPollster(TestPollsterBase):
def setUp(self):
super(TestCPUPollster, self).setUp()
@mock.patch('ceilometer.pipeline.setup_pipeline', mock.MagicMock())
def test_get_counters(self):
self.inspector.inspect_cpus(self.instance.name).AndReturn(
virt_inspector.CPUStats(time=1 * (10 ** 6), number=2))
self.inspector.inspect_cpus(self.instance.name).AndReturn(
virt_inspector.CPUStats(time=3 * (10 ** 6), number=2))
# cpu_time resets on instance restart
self.inspector.inspect_cpus(self.instance.name).AndReturn(
virt_inspector.CPUStats(time=2 * (10 ** 6), number=2))
self.mox.ReplayAll()
mgr = manager.AgentManager()
pollster = pollsters.CPUPollster()
def _verify_cpu_metering(zero, expected_time):
cache = {}
counters = list(pollster.get_counters(mgr, cache, self.instance))
self.assertEquals(len(counters), 2)
self.assertEqual(set([c.name for c in counters]),
set(pollster.get_counter_names()))
assert counters[0].name == 'cpu_util'
assert (counters[0].volume == 0.0 if zero else
counters[0].volume > 0.0)
assert counters[1].name == 'cpu'
assert counters[1].volume == expected_time
assert pollster.CACHE_KEY_CPU in cache
assert self.instance.name in cache[pollster.CACHE_KEY_CPU]
# ensure elapsed time between polling cycles is non-zero
time.sleep(0.001)
_verify_cpu_metering(True, 1 * (10 ** 6))
_verify_cpu_metering(False, 3 * (10 ** 6))
_verify_cpu_metering(False, 2 * (10 ** 6))
@mock.patch('ceilometer.pipeline.setup_pipeline', mock.MagicMock())
def test_get_counters_cache(self):
# self.inspector.inspect_cpus(self.instance.name).AndReturn(
# virt_inspector.CPUStats(time=1 * (10 ** 6), number=2))
# self.inspector.inspect_cpus(self.instance.name).AndReturn(
# virt_inspector.CPUStats(time=3 * (10 ** 6), number=2))
# # cpu_time resets on instance restart
# self.inspector.inspect_cpus(self.instance.name).AndReturn(
# virt_inspector.CPUStats(time=2 * (10 ** 6), number=2))
self.mox.ReplayAll()
mgr = manager.AgentManager()
pollster = pollsters.CPUPollster()
cache = {
pollster.CACHE_KEY_CPU: {
self.instance.name: virt_inspector.CPUStats(
time=10 ** 6,
number=2,
),
},
}
counters = list(pollster.get_counters(mgr, cache, self.instance))
self.assertEquals(len(counters), 2)
self.assertEquals(counters[1].volume, 10 ** 6)