Add python iostat collectd plugin

* Deploys collectd-iostat-python
* Exposes vars to determine disk and/or iostat plugin
* Dashboard for iostat metrics and on general dashboards
* Add inodes percent usage collection with df plugin
* Graph inodes % usage in dashboards
* Separate df from disk in dashboards
* Severity in collectd.log

iostat provides 18 metrics across disks and lvm devices
collectd disk plugin provides 10 metrics across disks

Provides more IO metrics than Collectd disk plugin

Change-Id: I9de36c53ebe62de6e50edf894ca74ce34f13e362

ansible/install/collectd-openstack.yml

@@ -22,6 +22,7 @@
   roles:
     - { role: common, when: collectd_undercloud }
     - { role: epel, when: collectd_undercloud }
+    - { role: repo }
     - { role: collectd-openstack, when: collectd_undercloud }
   tasks:
     - name: Collectd off if not collectd_undercloud
@@ -38,7 +39,7 @@
   roles:
     - { role: common, when: collectd_controller }
     - { role: epel, when: collectd_controller }
-    - { role: repo, when: keystone_overcloud_collectd_plugin }
+    - { role: repo }
     - { role: collectd-openstack, when: collectd_controller }
   tasks:
     - name: Collectd off if not collectd_controller
@@ -55,6 +56,7 @@
   roles:
     - { role: common, when: collectd_blockstorage }
     - { role: epel, when: collectd_blockstorage }
+    - { role: repo }
     - { role: collectd-openstack, when: collectd_blockstorage }
   tasks:
     - name: Collectd off if not collectd_blockstorage
@@ -71,6 +73,7 @@
   roles:
     - { role: common, when: collectd_objectstorage }
     - { role: epel, when: collectd_objectstorage }
+    - { role: repo }
     - { role: collectd-openstack, when: collectd_objectstorage }
   tasks:
     - name: Collectd off if not collectd_objectstorage
@@ -87,6 +90,7 @@
   roles:
     - { role: common, when: collectd_cephstorage }
     - { role: epel, when: collectd_cephstorage }
+    - { role: repo }
     - { role: collectd-openstack, when: collectd_cephstorage }
   tasks:
     - name: Collectd off if not collectd_cephstorage
@@ -103,6 +107,7 @@
   roles:
     - { role: common, when: collectd_compute }
     - { role: epel, when: collectd_compute }
+    - { role: repo }
     - { role: collectd-openstack, when: collectd_compute }
   tasks:
     - name: Collectd off if not collectd_compute

ansible/install/grafana-dashboards.yml

@@ -22,6 +22,7 @@
       - cloud_rabbitmq_monitoring
       - cloud_system_performance_comparsion
       - cloud_total_memory_usage
+      - iostat
       - three_node_performance_food_groups
       - openstack_ironic_metrics
     cloud_specific_dashboards:

ansible/install/group_vars/all.yml

@@ -137,6 +137,30 @@ gnocchi_status_undercloud_collectd_interval: 10
 gnocchi_status_controller_collectd_plugin: false
 gnocchi_status_controller_collectd_interval: 10
 
+########################
+# Disk/IOStat plugin
+########################
+# Disk plugin metrics are opt-out, IOStat metrics are opt-in
+disk_undercloud_collectd_plugin: true
+disk_controller_collectd_plugin: true
+disk_cephstorage_collectd_plugin: true
+disk_compute_collectd_plugin: true
+disk_blockstorage_collectd_plugin: true
+disk_objectstorage_collectd_plugin: true
+# Enable these for more comprehensive IOStat metrics
+iostat_undercloud_collectd_plugin: false
+iostat_undercloud_collectd_interval: 10
+iostat_controller_collectd_plugin: false
+iostat_controller_collectd_interval: 10
+iostat_cephstorage_collectd_plugin: false
+iostat_cephstorage_collectd_interval: 10
+iostat_compute_collectd_plugin: false
+iostat_compute_collectd_interval: 10
+iostat_blockstorage_collectd_plugin: false
+iostat_blockstorage_collectd_interval: 10
+iostat_objectstorage_collectd_plugin: false
+iostat_objectstorage_collectd_interval: 10
+
 ########################
 # Keystone token count
 ########################

ansible/install/roles/collectd-openstack/defaults/main.yml

@@ -47,6 +47,30 @@ gnocchi_status_undercloud_collectd_interval: 10
 gnocchi_status_controller_collectd_plugin: false
 gnocchi_status_controller_collectd_interval: 10
 
+########################
+# Disk/IOStat plugin
+########################
+# Disk plugin metrics are opt-out, IOStat metrics are opt-in
+disk_undercloud_collectd_plugin: true
+disk_controller_collectd_plugin: true
+disk_cephstorage_collectd_plugin: true
+disk_compute_collectd_plugin: true
+disk_blockstorage_collectd_plugin: true
+disk_objectstorage_collectd_plugin: true
+# Enable these for more comprehensive IOStat metrics
+iostat_undercloud_collectd_plugin: false
+iostat_undercloud_collectd_interval: 10
+iostat_controller_collectd_plugin: false
+iostat_controller_collectd_interval: 10
+iostat_cephstorage_collectd_plugin: false
+iostat_cephstorage_collectd_interval: 10
+iostat_compute_collectd_plugin: false
+iostat_compute_collectd_interval: 10
+iostat_blockstorage_collectd_plugin: false
+iostat_blockstorage_collectd_interval: 10
+iostat_objectstorage_collectd_plugin: false
+iostat_objectstorage_collectd_interval: 10
+
 ########################
 # Keystone token count
 ########################

ansible/install/roles/collectd-openstack/files/collectd_iostat_python.py

@@ -0,0 +1,403 @@
+#!/usr/bin/env python
+# coding=utf-8
+# The MIT License (MIT)
+#
+# Copyright (c) 2014-2016 Denis Zhdanov
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+#
+# collectd-iostat-python
+# ======================
+#
+# Collectd-iostat-python is an iostat plugin for collectd that allows you to
+# graph Linux iostat metrics in Graphite or other output formats that are
+# supported by collectd.
+#
+# https://github.com/powdahound/redis-collectd-plugin
+#   - was used as template
+# https://github.com/keirans/collectd-iostat/
+#   - was used as inspiration and contains some code from
+# https://bitbucket.org/jakamkon/python-iostat
+#   - by Kuba Kończyk <jakamkon at users.sourceforge.net>
+#
+
+import signal
+import string
+import subprocess
+import sys
+import re
+try:
+    import pyudev
+    pyudev_available = True
+except ImportError:
+    pyudev_available = False
+
+# Original Version/Author
+__version__ = '0.0.5'
+__author__ = 'denis.zhdanov@gmail.com'
+
+
+class IOStatError(Exception):
+    pass
+
+
+class CmdError(IOStatError):
+    pass
+
+
+class ParseError(IOStatError):
+    pass
+
+
+class IOStat(object):
+    def __init__(self, path='/usr/bin/iostat', interval=2, count=2, disks=[], no_dm_name=False):
+        self.path = path
+        self.interval = interval
+        self.count = count
+        self.disks = disks
+        self.no_dm_name = no_dm_name
+
+    def parse_diskstats(self, input):
+        """
+        Parse iostat -d and -dx output.If there are more
+        than one series of statistics, get the last one.
+        By default parse statistics for all avaliable block devices.
+
+        @type input: C{string}
+        @param input: iostat output
+
+        @type disks: list of C{string}s
+        @param input: lists of block devices that
+        statistics are taken for.
+
+        @return: C{dictionary} contains per block device statistics.
+        Statistics are in form of C{dictonary}.
+        Main statistics:
+          tps  Blk_read/s  Blk_wrtn/s  Blk_read  Blk_wrtn
+        Extended staistics (available with post 2.5 kernels):
+          rrqm/s  wrqm/s  r/s  w/s  rsec/s  wsec/s  rkB/s  wkB/s  avgrq-sz \
+          avgqu-sz  await  svctm  %util
+        See I{man iostat} for more details.
+        """
+        dstats = {}
+        dsi = input.rfind('Device:')
+        if dsi == -1:
+            raise ParseError('Unknown input format: %r' % input)
+
+        ds = input[dsi:].splitlines()
+        hdr = ds.pop(0).split()[1:]
+
+        for d in ds:
+            if d:
+                d = d.split()
+                d = [re.sub(r',','.',element) for element in d]
+                dev = d.pop(0)
+                if (dev in self.disks) or not self.disks:
+                    dstats[dev] = dict([(k, float(v)) for k, v in zip(hdr, d)])
+
+        return dstats
+
+    def sum_dstats(self, stats, smetrics):
+        """
+        Compute the summary statistics for chosen metrics.
+        """
+        avg = {}
+
+        for disk, metrics in stats.iteritems():
+            for mname, metric in metrics.iteritems():
+                if mname not in smetrics:
+                    continue
+                if mname in avg:
+                    avg[mname] += metric
+                else:
+                    avg[mname] = metric
+
+        return avg
+
+    def _run(self, options=None):
+        """
+        Run iostat command.
+        """
+        close_fds = 'posix' in sys.builtin_module_names
+        args = '%s %s %s %s %s' % (
+            self.path,
+            ''.join(options),
+            self.interval,
+            self.count,
+            ' '.join(self.disks))
+
+        return subprocess.Popen(
+            args,
+            bufsize=1,
+            shell=True,
+            stdout=subprocess.PIPE,
+            close_fds=close_fds)
+
+    @staticmethod
+    def _get_childs_data(child):
+        """
+        Return child's data when avaliable.
+        """
+        (stdout, stderr) = child.communicate()
+        ecode = child.poll()
+
+        if ecode != 0:
+            raise CmdError('Command %r returned %d' % (child.cmd, ecode))
+
+        return stdout
+
+    def get_diskstats(self):
+        """
+        Get all avaliable disks statistics that we can get.
+        iostat -kNd
+         tps    kB_read/s    kB_wrtn/s    kB_read    kB_wrtn
+        iostat -kNdx
+         rrqm/s   wrqm/s     r/s     w/s    rkB/s    wkB/s avgrq-sz
+         avgqu-sz   await r_await w_await  svctm  %util
+        """
+        options=['-','k','N','d']
+        extdoptions=['-','k','N','d','x']
+        if self.no_dm_name:
+            options.remove('N')
+            extdoptions.remove('N')
+        dstats = self._run(options)
+        extdstats = self._run(extdoptions)
+        dsd = self._get_childs_data(dstats)
+        edd = self._get_childs_data(extdstats)
+        ds = self.parse_diskstats(dsd)
+        eds = self.parse_diskstats(edd)
+
+        for dk, dv in ds.iteritems():
+            if dk in eds:
+                ds[dk].update(eds[dk])
+
+        return ds
+
+
+class IOMon(object):
+    def __init__(self):
+        self.plugin_name = 'collectd-iostat-python'
+        self.iostat_path = '/usr/bin/iostat'
+        self.interval = 60.0
+        self.iostat_interval = 2
+        self.iostat_count = 2
+        self.iostat_disks = []
+        self.iostat_nice_names = False
+        self.iostat_disks_regex = ''
+        self.iostat_udevnameattr = ''
+        self.skip_multipath = False
+        self.verbose_logging = False
+        self.iostat_no_dm_name = False
+        self.names = {
+            'tps': {'t': 'transfers_per_second'},
+            'Blk_read/s': {'t': 'blocks_per_second', 'ti': 'read'},
+            'kB_read/s': {'t': 'bytes_per_second', 'ti': 'read', 'm': 1024},
+            'MB_read/s': {'t': 'bytes_per_second', 'ti': 'read', 'm': 1048576},
+            'Blk_wrtn/s': {'t': 'blocks_per_second', 'ti': 'write'},
+            'kB_wrtn/s': {'t': 'bytes_per_second', 'ti': 'write', 'm': 1024},
+            'MB_wrtn/s': {'t': 'bytes_per_second', 'ti': 'write', 'm': 1048576},
+            'Blk_read': {'t': 'blocks', 'ti': 'read'},
+            'kB_read': {'t': 'bytes', 'ti': 'read', 'm': 1024},
+            'MB_read': {'t': 'bytes', 'ti': 'read', 'm': 1048576},
+            'Blk_wrtn': {'t': 'blocks', 'ti': 'write'},
+            'kB_wrtn': {'t': 'bytes', 'ti': 'write', 'm': 1024},
+            'MB_wrtn': {'t': 'bytes', 'ti': 'write', 'm': 1048576},
+            'rrqm/s': {'t': 'requests_merged_per_second', 'ti': 'read'},
+            'wrqm/s': {'t': 'requests_merged_per_second', 'ti': 'write'},
+            'r/s': {'t': 'per_second', 'ti': 'read'},
+            'w/s': {'t': 'per_second', 'ti': 'write'},
+            'rsec/s': {'t': 'sectors_per_second', 'ti': 'read'},
+            'rkB/s': {'t': 'bytes_per_second', 'ti': 'read', 'm': 1024},
+            'rMB/s': {'t': 'bytes_per_second', 'ti': 'read', 'm': 1048576},
+            'wsec/s': {'t': 'sectors_per_second', 'ti': 'write'},
+            'wkB/s': {'t': 'bytes_per_second', 'ti': 'write', 'm': 1024},
+            'wMB/s': {'t': 'bytes_per_second', 'ti': 'write', 'm': 1048576},
+            'avgrq-sz': {'t': 'avg_request_size'},
+            'avgqu-sz': {'t': 'avg_request_queue'},
+            'await': {'t': 'avg_wait_time'},
+            'r_await': {'t': 'avg_wait_time', 'ti': 'read'},
+            'w_await': {'t': 'avg_wait_time', 'ti': 'write'},
+            'svctm': {'t': 'avg_service_time'},
+            '%util': {'t': 'percent', 'ti': 'util'}
+        }
+
+    def log_verbose(self, msg):
+        if not self.verbose_logging:
+            return
+        collectd.info('%s plugin [verbose]: %s' % (self.plugin_name, msg))
+
+    def configure_callback(self, conf):
+        """
+        Receive configuration block
+        """
+        for node in conf.children:
+            val = str(node.values[0])
+
+            if node.key == 'Path':
+                self.iostat_path = val
+            elif node.key == 'Interval':
+                self.interval = float(val)
+            elif node.key == 'IostatInterval':
+                self.iostat_interval = int(float(val))
+            elif node.key == 'Count':
+                self.iostat_count = int(float(val))
+            elif node.key == 'Disks':
+                self.iostat_disks = val.split(',')
+            elif node.key == 'NiceNames':
+                self.iostat_nice_names = val in ['True', 'true']
+            elif node.key == 'DisksRegex':
+                self.iostat_disks_regex = val
+            elif node.key == 'UdevNameAttr':
+                self.iostat_udevnameattr = val
+            elif node.key == 'PluginName':
+                self.plugin_name = val
+            elif node.key == 'Verbose':
+                self.verbose_logging = val in ['True', 'true']
+            elif node.key == 'SkipPhysicalMultipath':
+                self.skip_multipath = val in [ 'True', 'true' ]
+            elif node.key == 'NoDisplayDMName':
+                self.iostat_no_dm_name = val in [ 'True', 'true' ]
+            else:
+                collectd.warning(
+                    '%s plugin: Unknown config key: %s.' % (
+                        self.plugin_name,
+                        node.key))
+
+        self.log_verbose(
+            'Configured with iostat=%s, interval=%s, count=%s, disks=%s, '
+            'disks_regex=%s udevnameattr=%s skip_multipath=%s no_dm_name=%s' % (
+                self.iostat_path,
+                self.iostat_interval,
+                self.iostat_count,
+                self.iostat_disks,
+                self.iostat_disks_regex,
+                self.iostat_udevnameattr,
+                self.skip_multipath,
+                self.iostat_no_dm_name))
+
+        collectd.register_read(self.read_callback, self.interval)
+
+    def dispatch_value(self, plugin_instance, val_type, type_instance, value):
+        """
+        Dispatch a value to collectd
+        """
+        self.log_verbose(
+            'Sending value: %s-%s.%s=%s' % (
+                self.plugin_name,
+                plugin_instance,
+                '-'.join([val_type, type_instance]),
+                value))
+
+        val = collectd.Values()
+        val.plugin = self.plugin_name
+        val.plugin_instance = plugin_instance
+        val.type = val_type
+        if len(type_instance):
+            val.type_instance = type_instance
+        val.values = [value, ]
+        val.meta={'0': True}
+        val.dispatch()
+
+    def read_callback(self):
+        """
+        Collectd read callback
+        """
+        self.log_verbose('Read callback called')
+        iostat = IOStat(
+            path=self.iostat_path,
+            interval=self.iostat_interval,
+            count=self.iostat_count,
+            disks=self.iostat_disks,
+            no_dm_name=self.iostat_no_dm_name)
+        ds = iostat.get_diskstats()
+
+        if not ds:
+            self.log_verbose('%s plugin: No info received.' % self.plugin_name)
+            return
+
+        if self.iostat_udevnameattr and pyudev_available:
+            context = pyudev.Context()
+
+        for disk in ds:
+            if not re.match(self.iostat_disks_regex, disk):
+                continue
+            if self.iostat_udevnameattr and pyudev_available:
+                device = pyudev.Device.from_device_file(context, "/dev/" + disk)
+                if self.skip_multipath:
+                    mp_managed = device.get('DM_MULTIPATH_DEVICE_PATH')
+                    if mp_managed and mp_managed == '1':
+                        self.log_verbose('Skipping physical multipath disk %s' % disk)
+                        continue
+                if self.iostat_udevnameattr:
+                    persistent_name = device.get(self.iostat_udevnameattr)
+                    if not persistent_name:
+                        self.log_verbose('Unable to determine disk name based on UdevNameAttr: %s' % self.iostat_udevnameattr)
+                        persistent_name = disk
+            else:
+                persistent_name = disk
+
+            for name in ds[disk]:
+                if self.iostat_nice_names and name in self.names:
+                    val_type = self.names[name]['t']
+
+                    if 'ti' in self.names[name]:
+                        type_instance = self.names[name]['ti']
+                    else:
+                        type_instance = ''
+
+                    value = ds[disk][name]
+                    if 'm' in self.names[name]:
+                        value *= self.names[name]['m']
+                else:
+                    val_type = 'gauge'
+                    tbl = string.maketrans('/-%', '___')
+                    type_instance = name.translate(tbl)
+                    value = ds[disk][name]
+                self.dispatch_value(
+                    persistent_name, val_type, type_instance, value)
+
+def restore_sigchld():
+    """
+    Restore SIGCHLD handler for python <= v2.6
+    It will BREAK exec plugin!!!
+    See https://github.com/deniszh/collectd-iostat-python/issues/2 for details
+    """
+    if sys.version_info[0] == 2 and sys.version_info[1] <= 6:
+        signal.signal(signal.SIGCHLD, signal.SIG_DFL)
+
+
+if __name__ == '__main__':
+    iostat = IOStat()
+    ds = iostat.get_diskstats()
+
+    for disk in ds:
+        for metric in ds[disk]:
+            tbl = string.maketrans('/-%', '___')
+            metric_name = metric.translate(tbl)
+            print("%s.%s:%s" % (disk, metric_name, ds[disk][metric]))
+
+    sys.exit(0)
+else:
+    import collectd
+
+    iomon = IOMon()
+
+    # Register callbacks
+    collectd.register_init(restore_sigchld)
+    collectd.register_config(iomon.configure_callback)

ansible/install/roles/collectd-openstack/tasks/main.yml

@@ -29,6 +29,14 @@
     - collectd-mysql
     - collectd-turbostat
 
+# Iostat plugin requires sysstat since shelling iostat for stats, Also it is
+# handy to have sysstat.
+- name: (Iostat python plugin) Install sysstat
+  yum:
+    name: sysstat
+    state: present
+  become: true
+
 - name: (Keystone Token Count) Install libdbi mysql driver
   yum:
     name: "{{item}}"
@@ -148,15 +156,21 @@
   become: true
   when: "((gnocchi_status_controller_collectd_plugin == true and inventory_hostname == groups['controller'][0]) or ('undercloud' in group_names and {{gnocchi_status_undercloud_collectd_plugin}} == true))"
 
-- name: Copy collectd_gnocchi_status.py
+- name: Copy python plugins
   copy:
-    src: collectd_gnocchi_status.py
-    dest: /usr/local/bin/collectd_gnocchi_status.py
+    src: "{{item.src}}"
+    dest: "{{item.dest}}"
     owner: root
     group: root
     mode: 0755
   become: true
-  when: "((gnocchi_status_controller_collectd_plugin == true and inventory_hostname == groups['controller'][0]) or ('undercloud' in group_names and {{gnocchi_status_undercloud_collectd_plugin}} == true))"
+  with_items:
+    - src: collectd_gnocchi_status.py
+      dest: /usr/local/bin/collectd_gnocchi_status.py
+    - src: collectd_rabbitmq_monitoring.py
+      dest: /usr/local/bin/collectd_rabbitmq_monitoring.py
+    - src: collectd_iostat_python.py
+      dest: /usr/local/bin/collectd_iostat_python.py
 
 # Rabbitmq monitoring
 - name: Install pyrabbit
@@ -199,16 +213,6 @@
   register: controller0_rabbitmq_password
   become: true
   when: "('controller' in group_names and {{rabbitmq_controller_collectd_plugin}} == true and '{{inventory_hostname}}' == groups['controller'][0])"
-
-- name: Copy collectd_rabbitmq_monitoring.py
-  copy:
-    src: collectd_rabbitmq_monitoring.py
-    dest: /usr/local/bin/collectd_rabbitmq_monitoring.py
-    owner: root
-    group: root
-    mode: 0755
-  become: true
-  when:  "('undercloud' in group_names or '{{inventory_hostname}}' == groups['controller'][0])"
 # End Rabbitmq monitoring
 
 # CephStorage OSD monitoring

ansible/install/roles/collectd-openstack/templates/blockstorage.collectd.conf.j2

@@ -10,8 +10,9 @@ Hostname "{{inventory_hostname}}"
 # Loaded Plugins:
 LoadPlugin "logfile"
 <Plugin "logfile">
-  LogLevel "info"
   File "/var/log/collectd.log"
+  LogLevel "info"
+  PrintSeverity true
   Timestamp true
 </Plugin>
 
@@ -28,6 +29,11 @@ LoadPlugin match_regex
 LoadPlugin memory
 LoadPlugin numa
 LoadPlugin processes
+{%if iostat_blockstorage_collectd_plugin %}
+<LoadPlugin python>
+  Globals true
+</LoadPlugin>
+{% endif %}
 LoadPlugin swap
 LoadPlugin tail
 LoadPlugin turbostat
@@ -76,14 +82,62 @@ PreCacheChain "PreCache"
 </Plugin>
 
 <Plugin df>
+  FSType anon_inodefs
+  FSType bdev
+  FSType cgroup
+  FSType cpuset
+  FSType debugfs
+  FSType devpts
+  FSType devtmpfs
+  FSType ecryptfs
+  FSType fuse
+  FSType fusectl
+  FSType hugetlbfs
+  FSType mqueue
+  FSType nfs
+  FSType nfs4
+  FSType nfsd
+  FSType pipefs
+  FSType proc
+  FSType pstore
+  FSType ramfs
+  #FSType rootfs
+  FSType rpc_pipefs
+  FSType securityfs
+  FSType sockfs
+  FSType sysfs
+  FSType tmpfs
+  FSType vboxsf
+  IgnoreSelected true
   ValuesPercentage true
+  ReportInodes true
 </Plugin>
 
+{%if disk_blockstorage_collectd_plugin %}
 <Plugin disk>
   Disk "/^[hsv]d[a-z]+[0-9]?$/"
+  Disk "/^nvm/"
   IgnoreSelected false
 </Plugin>
 
+{% endif %}
+{%if iostat_blockstorage_collectd_plugin %}
+<Plugin python>
+  ModulePath "/usr/local/bin/"
+  Import "collectd_iostat_python"
+
+  <Module collectd_iostat_python>
+    Path "/usr/bin/iostat"
+    Interval {{iostat_blockstorage_collectd_interval}}
+    IostatInterval 2
+    Count 2
+    Verbose false
+    NiceNames false
+    PluginName collectd_iostat_python
+  </Module>
+</Plugin>
+
+{% endif %}
 # (akrzos) Including the version of OpenStack that the process was verified as running after
 # OpenStack Installation with a comment at the end of each Process/ProcessMatch statement.
 # A Minus before the version means the process was not found in that version. (Ex -9)

ansible/install/roles/collectd-openstack/templates/cephstorage.collectd.conf.j2

@@ -10,8 +10,9 @@ Hostname "{{inventory_hostname}}"
 # Loaded Plugins:
 LoadPlugin "logfile"
 <Plugin "logfile">
-  LogLevel "info"
   File "/var/log/collectd.log"
+  LogLevel "info"
+  PrintSeverity true
   Timestamp true
 </Plugin>
 
@@ -31,6 +32,11 @@ LoadPlugin match_regex
 LoadPlugin memory
 LoadPlugin numa
 LoadPlugin processes
+{%if iostat_cephstorage_collectd_plugin %}
+<LoadPlugin python>
+  Globals true
+</LoadPlugin>
+{% endif %}
 LoadPlugin swap
 LoadPlugin tail
 LoadPlugin turbostat
@@ -89,14 +95,62 @@ PreCacheChain "PreCache"
 
 {% endif %}
 <Plugin df>
+  FSType anon_inodefs
+  FSType bdev
+  FSType cgroup
+  FSType cpuset
+  FSType debugfs
+  FSType devpts
+  FSType devtmpfs
+  FSType ecryptfs
+  FSType fuse
+  FSType fusectl
+  FSType hugetlbfs
+  FSType mqueue
+  FSType nfs
+  FSType nfs4
+  FSType nfsd
+  FSType pipefs
+  FSType proc
+  FSType pstore
+  FSType ramfs
+  #FSType rootfs
+  FSType rpc_pipefs
+  FSType securityfs
+  FSType sockfs
+  FSType sysfs
+  FSType tmpfs
+  FSType vboxsf
+  IgnoreSelected true
   ValuesPercentage true
+  ReportInodes true
 </Plugin>
 
+{%if disk_cephstorage_collectd_plugin %}
 <Plugin disk>
   Disk "/^[hsv]d[a-z]+[0-9]?|nvme.+$/"
+  Disk "/^nvm/"
   IgnoreSelected false
 </Plugin>
 
+{% endif %}
+{%if iostat_cephstorage_collectd_plugin %}
+<Plugin python>
+  ModulePath "/usr/local/bin/"
+  Import "collectd_iostat_python"
+
+  <Module collectd_iostat_python>
+    Path "/usr/bin/iostat"
+    Interval {{iostat_cephstorage_collectd_interval}}
+    IostatInterval 2
+    Count 2
+    Verbose false
+    NiceNames false
+    PluginName collectd_iostat_python
+  </Module>
+</Plugin>
+
+{% endif %}
 # (akrzos) Including the version of OpenStack that the process was verified as running after
 # OpenStack Installation with a comment at the end of each Process/ProcessMatch statement.
 # A Minus before the version means the process was not found in that version. (Ex -9)

ansible/install/roles/collectd-openstack/templates/compute.collectd.conf.j2

@@ -10,8 +10,9 @@ Hostname "{{inventory_hostname}}"
 # Loaded Plugins:
 LoadPlugin "logfile"
 <Plugin "logfile">
-  LogLevel "info"
   File "/var/log/collectd.log"
+  LogLevel "info"
+  PrintSeverity true
   Timestamp true
 </Plugin>
 
@@ -28,6 +29,11 @@ LoadPlugin match_regex
 LoadPlugin memory
 LoadPlugin numa
 LoadPlugin processes
+{%if iostat_compute_collectd_plugin %}
+<LoadPlugin python>
+  Globals true
+</LoadPlugin>
+{% endif %}
 LoadPlugin swap
 LoadPlugin tail
 LoadPlugin turbostat
@@ -76,14 +82,62 @@ PreCacheChain "PreCache"
 </Plugin>
 
 <Plugin df>
+  FSType anon_inodefs
+  FSType bdev
+  FSType cgroup
+  FSType cpuset
+  FSType debugfs
+  FSType devpts
+  FSType devtmpfs
+  FSType ecryptfs
+  FSType fuse
+  FSType fusectl
+  FSType hugetlbfs
+  FSType mqueue
+  FSType nfs
+  FSType nfs4
+  FSType nfsd
+  FSType pipefs
+  FSType proc
+  FSType pstore
+  FSType ramfs
+  #FSType rootfs
+  FSType rpc_pipefs
+  FSType securityfs
+  FSType sockfs
+  FSType sysfs
+  FSType tmpfs
+  FSType vboxsf
+  IgnoreSelected true
   ValuesPercentage true
+  ReportInodes true
 </Plugin>
 
+{%if disk_compute_collectd_plugin %}
 <Plugin disk>
   Disk "/^[hsv]d[a-z]+[0-9]?$/"
+  Disk "/^nvm/"
   IgnoreSelected false
 </Plugin>
 
+{% endif %}
+{%if iostat_compute_collectd_plugin %}
+<Plugin python>
+  ModulePath "/usr/local/bin/"
+  Import "collectd_iostat_python"
+
+  <Module collectd_iostat_python>
+    Path "/usr/bin/iostat"
+    Interval {{iostat_compute_collectd_interval}}
+    IostatInterval 2
+    Count 2
+    Verbose false
+    NiceNames false
+    PluginName collectd_iostat_python
+  </Module>
+</Plugin>
+
+{% endif %}
 # (akrzos) Including the version of OpenStack that the process was verified as running after
 # OpenStack Installation with a comment at the end of each Process/ProcessMatch statement.
 # A Minus before the version means the process was not found in that version. (Ex -10)

ansible/install/roles/collectd-openstack/templates/controller.collectd.conf.j2

@@ -10,8 +10,9 @@ Hostname "{{inventory_hostname}}"
 # Loaded Plugins:
 LoadPlugin "logfile"
 <Plugin "logfile">
-  LogLevel "info"
   File "/var/log/collectd.log"
+  LogLevel "info"
+  PrintSeverity true
   Timestamp true
 </Plugin>
 
@@ -42,7 +43,7 @@ LoadPlugin memory
 LoadPlugin mysql
 LoadPlugin numa
 LoadPlugin processes
-{%if gnocchi_status_controller_collectd_plugin or rabbitmq_controller_collectd_plugin %}
+{%if gnocchi_status_controller_collectd_plugin or iostat_controller_collectd_plugin or rabbitmq_controller_collectd_plugin %}
 <LoadPlugin python>
   Globals true
 </LoadPlugin>
@@ -142,14 +143,62 @@ PreCacheChain "PreCache"
 {% endif %}
 
 <Plugin df>
+  FSType anon_inodefs
+  FSType bdev
+  FSType cgroup
+  FSType cpuset
+  FSType debugfs
+  FSType devpts
+  FSType devtmpfs
+  FSType ecryptfs
+  FSType fuse
+  FSType fusectl
+  FSType hugetlbfs
+  FSType mqueue
+  FSType nfs
+  FSType nfs4
+  FSType nfsd
+  FSType pipefs
+  FSType proc
+  FSType pstore
+  FSType ramfs
+  #FSType rootfs
+  FSType rpc_pipefs
+  FSType securityfs
+  FSType sockfs
+  FSType sysfs
+  FSType tmpfs
+  FSType vboxsf
+  IgnoreSelected true
   ValuesPercentage true
+  ReportInodes true
 </Plugin>
 
+{%if disk_controller_collectd_plugin %}
 <Plugin disk>
   Disk "/^[hsv]d[a-z]+[0-9]?$/"
+  Disk "/^nvm/"
   IgnoreSelected false
 </Plugin>
 
+{% endif %}
+{%if iostat_controller_collectd_plugin %}
+<Plugin python>
+  ModulePath "/usr/local/bin/"
+  Import "collectd_iostat_python"
+
+  <Module collectd_iostat_python>
+    Path "/usr/bin/iostat"
+    Interval {{iostat_controller_collectd_interval}}
+    IostatInterval 2
+    Count 2
+    Verbose false
+    NiceNames false
+    PluginName collectd_iostat_python
+  </Module>
+</Plugin>
+
+{% endif %}
 {%if gnocchi_status_controller_collectd_plugin %}
 {%if inventory_hostname == groups['controller'][0] %}
 <Plugin python>

ansible/install/roles/collectd-openstack/templates/objectstorage.collectd.conf.j2

@@ -10,8 +10,9 @@ Hostname "{{inventory_hostname}}"
 # Loaded Plugins:
 LoadPlugin "logfile"
 <Plugin "logfile">
-  LogLevel "info"
   File "/var/log/collectd.log"
+  LogLevel "info"
+  PrintSeverity true
   Timestamp true
 </Plugin>
 
@@ -28,6 +29,11 @@ LoadPlugin match_regex
 LoadPlugin memory
 LoadPlugin numa
 LoadPlugin processes
+{%if iostat_objectstorage_collectd_plugin %}
+<LoadPlugin python>
+  Globals true
+</LoadPlugin>
+{% endif %}
 LoadPlugin swap
 LoadPlugin tail
 LoadPlugin turbostat
@@ -76,14 +82,62 @@ PreCacheChain "PreCache"
 </Plugin>
 
 <Plugin df>
+  FSType anon_inodefs
+  FSType bdev
+  FSType cgroup
+  FSType cpuset
+  FSType debugfs
+  FSType devpts
+  FSType devtmpfs
+  FSType ecryptfs
+  FSType fuse
+  FSType fusectl
+  FSType hugetlbfs
+  FSType mqueue
+  FSType nfs
+  FSType nfs4
+  FSType nfsd
+  FSType pipefs
+  FSType proc
+  FSType pstore
+  FSType ramfs
+  #FSType rootfs
+  FSType rpc_pipefs
+  FSType securityfs
+  FSType sockfs
+  FSType sysfs
+  FSType tmpfs
+  FSType vboxsf
+  IgnoreSelected true
   ValuesPercentage true
+  ReportInodes true
 </Plugin>
 
+{%if disk_objectstorage_collectd_plugin %}
 <Plugin disk>
   Disk "/^[hsv]d[a-z]+[0-9]?$/"
+  Disk "/^nvm/"
   IgnoreSelected false
 </Plugin>
 
+{% endif %}
+{%if iostat_objectstorage_collectd_plugin %}
+<Plugin python>
+  ModulePath "/usr/local/bin/"
+  Import "collectd_iostat_python"
+
+  <Module collectd_iostat_python>
+    Path "/usr/bin/iostat"
+    Interval {{iostat_objectstorage_collectd_interval}}
+    IostatInterval 2
+    Count 2
+    Verbose false
+    NiceNames false
+    PluginName collectd_iostat_python
+  </Module>
+</Plugin>
+
+{% endif %}
 # (akrzos) Including the version of OpenStack that the process was verified as running after
 # OpenStack Installation with a comment at the end of each Process/ProcessMatch statement.
 # A Minus before the version means the process was not found in that version. (Ex -9)

ansible/install/roles/collectd-openstack/templates/undercloud.collectd.conf.j2

@@ -10,8 +10,9 @@ Hostname "{{inventory_hostname}}"
 # Loaded Plugins:
 LoadPlugin "logfile"
 <Plugin "logfile">
-  LogLevel "info"
   File "/var/log/collectd.log"
+  LogLevel "info"
+  PrintSeverity true
   Timestamp true
 </Plugin>
 
@@ -35,7 +36,7 @@ LoadPlugin memory
 LoadPlugin mysql
 LoadPlugin numa
 LoadPlugin processes
-{%if gnocchi_status_undercloud_collectd_plugin or rabbitmq_undercloud_collectd_plugin %}
+{%if gnocchi_status_undercloud_collectd_plugin or iostat_undercloud_collectd_plugin or rabbitmq_undercloud_collectd_plugin %}
 <LoadPlugin python>
   Globals true
 </LoadPlugin>
@@ -70,6 +71,15 @@ PreCacheChain "PreCache"
     </Match>
     Target "stop"
   </Rule>
+  {%if iostat_undercloud_collectd_plugin %}
+  <Rule "ignore_docker_iostat">
+    <Match "regex">
+      Plugin "^collectd_iostat_python$"
+      PluginInstance "^docker.*$"
+    </Match>
+    Target "stop"
+  </Rule>
+  {% endif %}
   Target "return"
 </Chain>
 
@@ -116,14 +126,62 @@ PreCacheChain "PreCache"
 {% endif %}
 
 <Plugin df>
+  FSType anon_inodefs
+  FSType bdev
+  FSType cgroup
+  FSType cpuset
+  FSType debugfs
+  FSType devpts
+  FSType devtmpfs
+  FSType ecryptfs
+  FSType fuse
+  FSType fusectl
+  FSType hugetlbfs
+  FSType mqueue
+  FSType nfs
+  FSType nfs4
+  FSType nfsd
+  FSType pipefs
+  FSType proc
+  FSType pstore
+  FSType ramfs
+  #FSType rootfs
+  FSType rpc_pipefs
+  FSType securityfs
+  FSType sockfs
+  FSType sysfs
+  FSType tmpfs
+  FSType vboxsf
+  IgnoreSelected true
   ValuesPercentage true
+  ReportInodes true
 </Plugin>
 
+{%if disk_undercloud_collectd_plugin %}
 <Plugin disk>
   Disk "/^[hsv]d[a-z]+[0-9]?$/"
+  Disk "/^nvm/"
   IgnoreSelected false
 </Plugin>
 
+{% endif %}
+{%if iostat_undercloud_collectd_plugin %}
+<Plugin python>
+  ModulePath "/usr/local/bin/"
+  Import "collectd_iostat_python"
+
+  <Module collectd_iostat_python>
+    Path "/usr/bin/iostat"
+    Interval {{iostat_undercloud_collectd_interval}}
+    IostatInterval 2
+    Count 2
+    Verbose false
+    NiceNames false
+    PluginName collectd_iostat_python
+  </Module>
+</Plugin>
+
+{% endif %}
 {%if gnocchi_status_undercloud_collectd_plugin %}
 <Plugin python>
   ModulePath "/usr/local/bin/"