Add support for SQLAlchemy collectd plugin

The purpose of sqlalchemy-collectd is to monitor database usage from the perspective of the applications that are actually attaching to it, which is mostly monitoring of connection pool use.

Change-Id: I59e7e4bf930a05689c0a62d943f40b2199b5a5e5

ansible/install/group_vars/all.yml

@@ -231,6 +231,11 @@ ceph_controller_collectd_pool_interval: 10
 ceph_controller_collectd_plugin: false
 ceph_storage_collectd_plugin: false
 
+############################
+# sqlalchemy plugin
+###########################
+sqlalchemy_collectd_plugin: false
+
 ########################
 # Gnocchi Status plugin
 ########################

ansible/install/roles/collectd/tasks/main.yml

@@ -60,6 +60,49 @@
     version: master
   when: not (directory_exists.stat.isdir is defined and directory_exists.stat.isdir)
 
+- name: Get mysql string
+  shell: "crudini --get {{item.value.path}} database connection"
+  become: true
+  register: db_connections
+  with_dict: "{{mysql_svcs}}"
+  when: ('Controller' in group_names and sqlalchemy_collectd_plugin)
+  ignore_errors: true
+
+- name: Append to mysql string
+  ini_file:
+    backup: true
+    dest: "{{ item.item.value.path }}"
+    section: database
+    option: connection
+    value: "{{ item.stdout }}&plugin=collectd&collectd_program_name={{ item.item.key }}"
+  when: (('Controller' in group_names) and ('plugin=collectd' not in item.stdout) and sqlalchemy_collectd_plugin)
+  with_items: "{{ db_connections.results }}"
+  become: true
+
+- name: restart containers
+  shell: podman restart {{ item }}
+  become: true
+  with_items:
+    - cinder_api
+    - cinder_api_cron
+    - cinder_scheduler
+    - nova_conductor
+    - nova_api_cron
+    - nova_scheduler
+    - nova_vnc_proxy
+    - nova_api
+    - nova_metadata
+    - neutron_api
+    - gnocchi_api
+    - keystone
+    - heat_api
+    - heat_engine
+    - heat_api_cron
+    - heat_api_cfn
+  ignore_errors: yes
+  when: ('Controller' in group_names and sqlalchemy_collectd_plugin)
+
+
 # CephStorage OSD monitoring
 # Only Monitors a single OSD on each CephStorage Node
 - name: Get 1st OSD socket

ansible/install/roles/collectd/templates/controller.collectd.conf.j2

@@ -539,6 +539,31 @@ PreCacheChain "PreCache"
 
 {% endif %}
 
+{% if sqlalchemy_collectd_plugin %}
+LoadPlugin python
+<Plugin python>
+   LogTraces true
+   Import "sqlalchemy_collectd.server.plugin"
+
+   <Module "sqlalchemy_collectd.server.plugin">
+      # ipv4 only for the moment
+      listen "localhost" 25827
+
+      # set to "debug" to show messages received
+      loglevel "info"
+
+   </Module>
+
+   Import "sqlalchemy_collectd.connmon.plugin"
+   <Module "sqlalchemy_collectd.connmon.plugin">
+      monitor "localhost" 25828
+
+      # set to "debug" to show messaging
+      #loglevel "debug"
+   </Module>
+</Plugin>
+{% endif %}
+
 {%if pacemaker_monitoring %}
 <Plugin python>
   ModulePath "/usr/local/bin/"

ansible/install/roles/collectd/vars/16.yml

@@ -161,3 +161,22 @@ collectd_logs:
           instance: nova-compute
         - log_path: /var/log/containers/openvswitch/ovn-controller.log
           instance: ovn-controller
+mysql_svcs:
+  cinder:
+    path: "/var/lib/config-data/puppet-generated/cinder/etc/cinder/cinder.conf"
+  glance:
+    path: "/var/lib/config-data/puppet-generated/glance_api/etc/glance/glance-api.conf"
+  heat:
+    path: "/var/lib/config-data/puppet-generated/heat/etc/heat/heat.conf"
+  heat_api:
+    path: "/var/lib/config-data/puppet-generated/heat_api/etc/heat/heat.conf"
+  heat_api_cnf:
+    path: "/var/lib/config-data/puppet-generated/heat_api_cfn/etc/heat/heat.conf"
+  keystone:
+    path: "/var/lib/config-data/puppet-generated/keystone/etc/keystone/keystone.conf"
+  neutron:
+    path: "/var/lib/config-data/puppet-generated/neutron/etc/neutron/neutron.conf"
+  nova:
+    path: "/var/lib/config-data/puppet-generated/nova/etc/nova/nova.conf"
+  nova_metadata:
+    path: "/var/lib/config-data/puppet-generated/nova_metadata/etc/nova/nova.conf"

ansible/install/roles/grafana-dashboards/templates/openstack_general_system_performance.yaml.j2

@@ -9,6 +9,7 @@
 {% set ovsagent_groups = ['controller', 'compute', '*'] %}
 {% set rabbitmq_groups = ['undercloud', 'controller', '*'] %}
 {% set swift_stat_groups = ['controller', '*'] %}
+{% set sqlalchemy_groups = ['controller','*'] %}
 {% set controller_groups = ['controller', '*'] %}
 ---
 dashboard:
@@ -173,3 +174,7 @@ dashboard:
 {% include 'partials/ovn_metrics.yaml' %}
 
 {% include 'partials/tail.yaml' %}
+
+{% if item.template_node_type in sqlalchemy_groups %}
+  {% include 'partials/sqlalchemy.yaml' %}
+{% endif %}

ansible/install/roles/grafana-dashboards/templates/partials/sqlalchemy.yaml

@@ -0,0 +1,165 @@
+    - title: SQLAlchemy Metrics
+      collapse: true
+      height: 200px
+      showTitle: true
+      panels:
+        - title: SQLAlchemy Checkedin
+          type: graph
+          fill: 0
+          legend:
+            alignAsTable: true
+            avg: false
+            current: true
+            max: true
+            min: true
+            rightSide: true
+            show: true
+            total: false
+            values: true
+          nullPointMode: 'null'
+          targets:
+            - target: aliasByNode($Cloud.$Node.sqlalchemy-*.count-checkedin, 2, 3)
+        - title: SQLAlchemy Count Checkouts
+          type: graph
+          fill: 0
+          legend:
+            alignAsTable: true
+            avg: false
+            current: true
+            max: true
+            min: true
+            rightSide: true
+            show: true
+            total: false
+            values: true
+          nullPointMode: 'null'
+          targets:
+            - target: aliasByNode($Cloud.$Node.sqlalchemy-*.count-checkedout, 2, 3)
+        - title: SQLAlchemy Connections
+          type: graph
+          fill: 0
+          legend:
+            alignAsTable: true
+            avg: false
+            current: true
+            max: true
+            min: true
+            rightSide: true
+            show: true
+            total: false
+            values: true
+          nullPointMode: 'null'
+          targets:
+            - target: aliasByNode($Cloud.$Node.sqlalchemy-*.count-connections, 2, 3)
+        - title: SQLAlchemy Count Detached
+          type: graph
+          fill: 0
+          legend:
+            alignAsTable: true
+            avg: false
+            current: true
+            max: true
+            min: true
+            rightSide: true
+            show: true
+            total: false
+            values: true
+          nullPointMode: 'null'
+          targets:
+            - target: aliasByNode($Cloud.$Node.sqlalchemy-*.count-detached, 2, 3)
+        - title: SQLAlchemy Count NumPools
+          type: graph
+          fill: 0
+          legend:
+            alignAsTable: true
+            avg: false
+            current: true
+            max: true
+            min: true
+            rightSide: true
+            show: true
+            total: false
+            values: true
+          nullPointMode: 'null'
+          targets:
+            - target: aliasByNode($Cloud.$Node.sqlalchemy-*.count-numpools, 2, 3)
+        - title: SQLAlchemy Count NumProcs
+          type: graph
+          fill: 0
+          legend:
+            alignAsTable: true
+            avg: false
+            current: true
+            max: true
+            min: true
+            rightSide: true
+            show: true
+            total: false
+            values: true
+          nullPointMode: 'null'
+          targets:
+            - target: aliasByNode($Cloud.$Node.sqlalchemy-*.count-numprocs, 2, 3)
+        - title: SQLAlchemy Derive Checkouts
+          type: graph
+          fill: 0
+          legend:
+            alignAsTable: true
+            avg: false
+            current: true
+            max: true
+            min: true
+            rightSide: true
+            show: true
+            total: false
+            values: true
+          nullPointMode: 'null'
+          targets:
+            - target: aliasByNode($Cloud.$Node.sqlalchemy-*.derive-checkouts, 2, 3)
+        - title: SQLAlchemy Derive Connects
+          type: graph
+          fill: 0
+          legend:
+            alignAsTable: true
+            avg: false
+            current: true
+            max: true
+            min: true
+            rightSide: true
+            show: true
+            total: false
+            values: true
+          nullPointMode: 'null'
+          targets:
+            - target: aliasByNode($Cloud.$Node.sqlalchemy-*.derive-connects, 2, 3)
+        - title: SQLAlchemy Derive Disconnects
+          type: graph
+          fill: 0
+          legend:
+            alignAsTable: true
+            avg: false
+            current: true
+            max: true
+            min: true
+            rightSide: true
+            show: true
+            total: false
+            values: true
+          nullPointMode: 'null'
+          targets:
+            - target: aliasByNode($Cloud.$Node.sqlalchemy-*.derive-disconnects, 2, 3)
+        - title: SQLAlchemy Derive Invalidated
+          type: graph
+          fill: 0
+          legend:
+            alignAsTable: true
+            avg: false
+            current: true
+            max: true
+            min: true
+            rightSide: true
+            show: true
+            total: false
+            values: true
+          nullPointMode: 'null'
+          targets:
+            - target: aliasByNode($Cloud.$Node.sqlalchemy-*.derive-invalidated, 2, 3)

browbeat-containers/collectd-openstack/Dockerfile

@@ -5,7 +5,7 @@ RUN dnf clean all && \
     dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && \
     dnf install -y centos-release-opstools && \
     dnf install -y collectd collectd-turbostat collectd-disk collectd-apache collectd-ceph \
-     collectd-mysql collectd-python collectd-ping && \
+     collectd-mysql collectd-python collectd-ping python3-sqlalchemy-collectd && \
     dnf install -y sysstat && \
     dnf install -y python3-pip python3-devel && \
     pip3 install --upgrade pip && \