Add ovs flows monitoring

This commit

1. Adds a custom plugin to get number of ovs flows on br-int
2. Adds a row/panel to the OpenStack General System Performance
   dashboard to visualize this
3. Adds support for containerized/non-containerized collectd
4. In case of containerized, ovs is installed on container and
   /etc/openvswitch and /var/run/openvswitch from host are mounted
   into container
5. The custom collectd exec plugin cannot be run as root, hence stack
   user is created within container and the plugin is run as stack user.
   In the case of non-containerized collectd, heat-admin user existing on
   overcloud nodes is used
6. Controllers and Computes are monitored

Change-Id: I4687bd6aa0168c0b4bab72cf8eaca5f92c42b635

ansible/install/group_vars/all.yml

@@ -275,6 +275,7 @@ controller_monitored_queues:
 ovsagent_compute_monitor: false
 ovsagent_controller_monitor: false
 ovsagent_networker_monitor: false
+ovs_flows_monitoring: false
 
 controller_monitored_ints:
   - "tap"

ansible/install/roles/collectd-openstack/files/ovs_flows.sh

@@ -0,0 +1,9 @@
+#!/bin/sh
+HOSTNAME="${COLLECTD_HOSTNAME:-`hostname -f`}"
+INTERVAL="${COLLECTD_INTERVAL:-15}"
+
+while sleep "$INTERVAL"; do
+  VALUE=$(sudo ovs-ofctl dump-flows br-int | wc -l)
+  echo "PUTVAL \"$HOSTNAME/ovs-flows/gauge-ovs_flows\" interval=$INTERVAL N:$VALUE"
+done
+

ansible/install/roles/collectd-openstack/tasks/main.yml

@@ -245,6 +245,19 @@
       dest: /usr/local/bin/collectd_ovsagent.py
   when: "('controller' in group_names ) or ('compute' in group_names) or ('networker' in group_names)"
 
+- name: Copy exec plugins
+  copy:
+    src: "{{item.src}}"
+    dest: "{{item.dest}}"
+    owner: root
+    group: root
+    mode: 0755
+  become: true
+  with_items:
+    - src: ovs_flows.sh
+      dest: /usr/local/bin/ovs_flows.sh
+  when: "('controller' in group_names ) or ('compute' in group_names)"
+
 # Rabbitmq monitoring
 - name: Install pyrabbit
   easy_install:

ansible/install/roles/collectd-openstack/templates/compute.collectd.conf.j2

@@ -123,6 +123,13 @@ PreCacheChain "PreCache"
 </Plugin>
 
 {% endif %}
+
+{% if ovs_flows_monitoring %}
+<Plugin exec>
+  Exec "heat-admin:heat-admin" "/usr/local/bin/ovs_flows.sh"
+</Plugin>
+{% endif %}
+
 {%if iostat_compute_collectd_plugin %}
 <Plugin python>
   ModulePath "/usr/local/bin/"

ansible/install/roles/collectd-openstack/templates/controller.collectd.conf.j2

@@ -106,6 +106,12 @@ PreCacheChain "PreCache"
 {% if ceph_controller_collectd_plugin %}
 {% if inventory_hostname == groups['controller'][0] %}
 
+{% if ovs_flows_monitoring %}
+<Plugin exec>
+  Exec "heat-admin:heat-admin" "/usr/local/bin/ovs_flows.sh"
+</Plugin>
+{% endif %}
+
 <Plugin ceph>
   LongRunAvgLatency false
   ConvertSpecialMetricTypes true

ansible/install/roles/collectd/tasks/main.yml

@@ -87,5 +87,10 @@
           {% if config_type == 'controller' or config_type == 'undercloud' %}
           -v /var/lib/mysql/mysql.sock:/var/lib/mysql/mysql.sock \
           {% endif %}
+          {% if config_type == 'controller' or config_type == 'compute' %}
+          {% if ovs_flows_monitoring %}
+          -v /etc/openvswitch/:/etc/openvswitch/ -v /var/run/openvswitch/:/var/run/openvswitch/ \
+          {% endif %}
+          {% endif %}
           collectd-openstack
       become: yes

ansible/install/roles/collectd/templates/compute.collectd.conf.j2

@@ -113,6 +113,13 @@ PreCacheChain "PreCache"
 </Plugin>
 
 {% endif %}
+
+{% if ovs_flows_monitoring %}
+<Plugin exec>
+  Exec stack "/usr/local/bin/ovs_flows.sh"
+</Plugin>
+{% endif %}
+
 {%if iostat_compute_collectd_plugin %}
 <Plugin python>
   ModulePath "/usr/local/bin/"

ansible/install/roles/collectd/templates/controller.collectd.conf.j2

@@ -200,6 +200,13 @@ PreCacheChain "PreCache"
 </Plugin>
 
 {% endif %}
+
+{% if ovs_flows_monitoring %}
+<Plugin exec>
+  Exec stack "/usr/local/bin/ovs_flows.sh"
+</Plugin>
+{% endif %}
+
 {%if iostat_controller_collectd_plugin %}
 <Plugin python>
   ModulePath "/usr/local/bin/"

ansible/install/roles/grafana-dashboards/templates/openstack_general_system_performance.yaml.j2

@@ -5,6 +5,7 @@
 {% set ironic_groups = ['undercloud'] %}
 {% set mariadb_groups = ['undercloud', 'controller', '*'] %}
 {% set odl_groups = ['controller', '*'] %}
+{% set ovsflows_groups = ['controller', 'compute', '*'] %}
 {% set ovsagent_groups = ['controller', 'compute', '*'] %}
 {% set rabbitmq_groups = ['undercloud', 'controller', '*'] %}
 {% set swift_stat_groups = ['controller', '*'] %}
@@ -150,6 +151,10 @@ dashboard:
   {% include 'partials/ironic_metrics.yaml' %}
 {% endif %}
 
+{% if item.template_node_type in ovsflows_groups %}
+  {% include 'partials/ovs_flows.yaml' %}
+{% endif %}
+
 {% if item.template_node_type in ovsagent_groups %}
   {% include 'partials/neutron_resources.yaml' %}
 {% endif %}

ansible/install/roles/grafana-dashboards/templates/partials/ovs_flows.yaml

@@ -0,0 +1,23 @@
+    - title: OVS Flows
+      collapse: true
+      height: 200px
+      showTitle: true
+      panels:
+        - title: OVS Flows
+          type: graph
+          legend:
+            alignAsTable: true
+            avg: true
+            current: true
+            max: true
+            min: true
+            rightSide: true
+            show: true
+            total: false
+            values: true
+          nullPointMode: 'null'
+          targets:
+            - target: alias($Cloud.$Node.ovs-flows.gauge-ovs_flows, 'br-int flows')
+          yaxes:
+            - format: short
+            - format: short

browbeat-containers/collectd-openstack/Dockerfile

@@ -2,6 +2,7 @@ FROM centos:7
 
 RUN yum update -y && \
     yum clean all && \
+    yum groupinstall -y development tools && \
     yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm && \
     yum install -y centos-release-opstools && \
     yum install -y collectd collectd-turbostat collectd-disk collectd-apache collectd-ceph \
@@ -9,13 +10,22 @@ RUN yum update -y && \
     yum install -y sysstat && \
     yum install -y python2-pip python2-devel && \
     pip install pyrabbit && \
-    yum install -y libdbi-dbd-mysql collectd-dbi
+    yum install -y libdbi-dbd-mysql collectd-dbi && \
+    yum install -y https://cbs.centos.org/kojifiles/packages/openvswitch/2.11.0/4.el7/x86_64/openvswitch-2.11.0-4.el7.x86_64.rpm && \
+    yum install -y libibverbs && \
+    yum install -y sudo
+
+RUN useradd stack
+RUN echo stack | passwd stack --stdin
+RUN echo "stack ALL=(root) NOPASSWD:ALL" | tee -a /etc/sudoers.d/stack
+RUN chmod 0440 /etc/sudoers.d/stack
 
 
 ADD files/collectd_ceph_storage.py /usr/local/bin/collectd_ceph_storage.py
 ADD files/collectd_gnocchi_status.py /usr/local/bin/collectd_gnocchi_status.py
 ADD files/collectd_rabbitmq_monitoring.py /usr/local/bin/collectd_rabbitmq_monitoring.py
 ADD files/collectd_swift_stat.py /usr/local/bin/collectd_swift_stat.py
+ADD files/ovs_flows.sh /usr/local/bin/ovs_flows.sh
 
 ADD config/collectd.conf /etc/collectd.conf
 

browbeat-containers/collectd-openstack/files/ovs_flows.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+HOSTNAME="${COLLECTD_HOSTNAME:-`hostname -f`}"
+INTERVAL="${COLLECTD_INTERVAL:-15}"
+
+while sleep "$INTERVAL"; do
+  VALUE=$(sudo ovs-ofctl dump-flows br-int | wc -l)
+  echo "PUTVAL \"$HOSTNAME/ovs-flows/gauge-ovs_flows\" interval=$INTERVAL N:$VALUE"
+done