OSP Resource Response Time Metrics

This patch adds the following changes. 1. A collectd tail plugin has been added to record average, minimum and maximum response times of different nova and neutron OSP resources through the controller logs. 2. Grafana panels have been added to visualize the data collected by collectd for OSP resource response times. Change-Id: I69582974fab25287ab68f529153ddbde08d95b66
2021-09-21 14:03:00 +05:30 · 2021-09-21 14:03:00 +05:30 · aaa9db0ec6
commit aaa9db0ec6
parent 9fb65e9b8a
5 changed files with 1207 additions and 0 deletions
--- a/ansible/install/group_vars/all.yml
+++ b/ansible/install/group_vars/all.yml
@ -356,6 +356,12 @@ ping_interval: 1
 # Monitors OSP resources (networks, instances ..) created in DB.
 osp_resources_collectd_plugin: true

+#######################################################################
+# OSP individual resources collectd response time plugin on controllers
+#######################################################################
+# Monitors response time for individual OSP resources (networks, subnets, routers ..) created in DB.
+osp_individual_resources_response_time: false
+
 ########################################################
 # OVN compute Plugin for OVN monitoring on computes
 ########################################################
--- a/ansible/install/roles/collectd-openstack/templates/controller.collectd.conf.j2
+++ b/ansible/install/roles/collectd-openstack/templates/controller.collectd.conf.j2
@ -823,6 +823,540 @@ PreCacheChain "PreCache"
      Instance "delete"
    </Match>
  </File>
+
+  <File "/var/log/containers/neutron/server.log">
+  Instance "neutron_avg_response_time"
+    <Match>
+      Regex "POST /v2.0.*time: ([0-9.]+)"
+      DSType "GaugeAverage"
+      Type "response_time"
+      Instance "post"
+    </Match>
+    <Match>
+      Regex "PUT /v2.0.*time: ([0-9.]+)"
+      DSType "GaugeAverage"
+      Type "response_time"
+      Instance "put"
+    </Match>
+    <Match>
+      Regex "GET /v2.0.*time: ([0-9.]+)"
+      DSType "GaugeAverage"
+      Type "response_time"
+      Instance "get"
+    </Match>
+    <Match>
+      Regex "DELETE /v2.0.*time: ([0-9.]+)"
+      DSType "GaugeAverage"
+      Type "response_time"
+      Instance "delete"
+    </Match>
+  {% if osp_individual_resources_response_time %}
+    <Match>
+      Regex "POST /v2.0/networks.*time: ([0-9.]+)"
+      DSType "GaugeAverage"
+      Type "response_time"
+      Instance "post_networks"
+    </Match>
+    <Match>
+      Regex "PUT /v2.0/networks.*time: ([0-9.]+)"
+      DSType "GaugeAverage"
+      Type "response_time"
+      Instance "put_networks"
+    </Match>
+    <Match>
+      Regex "GET /v2.0/networks.*time: ([0-9.]+)"
+      DSType "GaugeAverage"
+      Type "response_time"
+      Instance "get_networks"
+    </Match>
+    <Match>
+      Regex "DELETE /v2.0/networks.*time: ([0-9.]+)"
+      DSType "GaugeAverage"
+      Type "response_time"
+      Instance "delete_networks"
+    </Match>
+    <Match>
+      Regex "POST /v2.0/routers.*time: ([0-9.]+)"
+      DSType "GaugeAverage"
+      Type "response_time"
+      Instance "post_routers"
+    </Match>
+    <Match>
+      Regex "PUT /v2.0/routers.*time: ([0-9.]+)"
+      DSType "GaugeAverage"
+      Type "response_time"
+      Instance "put_routers"
+    </Match>
+    <Match>
+      Regex "GET /v2.0/routers.*time: ([0-9.]+)"
+      DSType "GaugeAverage"
+      Type "response_time"
+      Instance "get_routers"
+    </Match>
+    <Match>
+      Regex "DELETE /v2.0/routers.*time: ([0-9.]+)"
+      DSType "GaugeAverage"
+      Type "response_time"
+      Instance "delete_routers"
+    </Match>
+    <Match>
+      Regex "POST /v2.0/subnets.*time: ([0-9.]+)"
+      DSType "GaugeAverage"
+      Type "response_time"
+      Instance "post_subnets"
+    </Match>
+    <Match>
+      Regex "PUT /v2.0/subnets.*time: ([0-9.]+)"
+      DSType "GaugeAverage"
+      Type "response_time"
+      Instance "put_subnets"
+    </Match>
+    <Match>
+      Regex "GET /v2.0/subnets.*time: ([0-9.]+)"
+      DSType "GaugeAverage"
+      Type "response_time"
+      Instance "get_subnets"
+    </Match>
+    <Match>
+      Regex "DELETE /v2.0/subnets.*time: ([0-9.]+)"
+      DSType "GaugeAverage"
+      Type "response_time"
+      Instance "delete_subnets"
+    </Match>
+    <Match>
+      Regex "POST /v2.0/ports.*time: ([0-9.]+)"
+      DSType "GaugeAverage"
+      Type "response_time"
+      Instance "post_ports"
+    </Match>
+    <Match>
+      Regex "PUT /v2.0/ports.*time: ([0-9.]+)"
+      DSType "GaugeAverage"
+      Type "response_time"
+      Instance "put_ports"
+    </Match>
+    <Match>
+      Regex "GET /v2.0/ports.*time: ([0-9.]+)"
+      DSType "GaugeAverage"
+      Type "response_time"
+      Instance "get_ports"
+    </Match>
+    <Match>
+      Regex "DELETE /v2.0/ports.*time: ([0-9.]+)"
+      DSType "GaugeAverage"
+      Type "response_time"
+      Instance "delete_ports"
+    </Match>
+    <Match>
+      Regex "POST /v2.0/trunks.*time: ([0-9.]+)"
+      DSType "GaugeAverage"
+      Type "response_time"
+      Instance "post_trunks"
+    </Match>
+    <Match>
+      Regex "PUT /v2.0/trunks.*time: ([0-9.]+)"
+      DSType "GaugeAverage"
+      Type "response_time"
+      Instance "put_trunks"
+    </Match>
+    <Match>
+      Regex "GET /v2.0/trunks.*time: ([0-9.]+)"
+      DSType "GaugeAverage"
+      Type "response_time"
+      Instance "get_trunks"
+    </Match>
+    <Match>
+      Regex "DELETE /v2.0/trunks.*time: ([0-9.]+)"
+      DSType "GaugeAverage"
+      Type "response_time"
+      Instance "delete_trunks"
+    </Match>
+  {% endif %}
+  </File>
+
+  <File "/var/log/containers/neutron/server.log">
+  Instance "neutron_min_response_time"
+    <Match>
+      Regex "POST /v2.0.*time: ([0-9.]+)"
+      DSType "GaugeMin"
+      Type "response_time"
+      Instance "post"
+    </Match>
+    <Match>
+      Regex "PUT /v2.0.*time: ([0-9.]+)"
+      DSType "GaugeMin"
+      Type "response_time"
+      Instance "put"
+    </Match>
+    <Match>
+      Regex "GET /v2.0.*time: ([0-9.]+)"
+      DSType "GaugeMin"
+      Type "response_time"
+      Instance "get"
+    </Match>
+    <Match>
+      Regex "DELETE /v2.0.*time: ([0-9.]+)"
+      DSType "GaugeMin"
+      Type "response_time"
+      Instance "delete"
+    </Match>
+  {% if osp_individual_resources_response_time %}
+    <Match>
+      Regex "POST /v2.0/networks.*time: ([0-9.]+)"
+      DSType "GaugeMin"
+      Type "response_time"
+      Instance "post_networks"
+    </Match>
+    <Match>
+      Regex "PUT /v2.0/networks.*time: ([0-9.]+)"
+      DSType "GaugeMin"
+      Type "response_time"
+      Instance "put_networks"
+    </Match>
+    <Match>
+      Regex "GET /v2.0/networks.*time: ([0-9.]+)"
+      DSType "GaugeMin"
+      Type "response_time"
+      Instance "get_networks"
+    </Match>
+    <Match>
+      Regex "DELETE /v2.0/networks.*time: ([0-9.]+)"
+      DSType "GaugeMin"
+      Type "response_time"
+      Instance "delete_networks"
+    </Match>
+    <Match>
+      Regex "POST /v2.0/routers.*time: ([0-9.]+)"
+      DSType "GaugeMin"
+      Type "response_time"
+      Instance "post_routers"
+    </Match>
+    <Match>
+      Regex "PUT /v2.0/routers.*time: ([0-9.]+)"
+      DSType "GaugeMin"
+      Type "response_time"
+      Instance "put_routers"
+    </Match>
+    <Match>
+      Regex "GET /v2.0/routers.*time: ([0-9.]+)"
+      DSType "GaugeMin"
+      Type "response_time"
+      Instance "get_routers"
+    </Match>
+    <Match>
+      Regex "DELETE /v2.0/routers.*time: ([0-9.]+)"
+      DSType "GaugeMin"
+      Type "response_time"
+      Instance "delete_routers"
+    </Match>
+    <Match>
+      Regex "POST /v2.0/subnets.*time: ([0-9.]+)"
+      DSType "GaugeMin"
+      Type "response_time"
+      Instance "post_subnets"
+    </Match>
+    <Match>
+      Regex "PUT /v2.0/subnets.*time: ([0-9.]+)"
+      DSType "GaugeMin"
+      Type "response_time"
+      Instance "put_subnets"
+    </Match>
+    <Match>
+      Regex "GET /v2.0/subnets.*time: ([0-9.]+)"
+      DSType "GaugeMin"
+      Type "response_time"
+      Instance "get_subnets"
+    </Match>
+    <Match>
+      Regex "DELETE /v2.0/subnets.*time: ([0-9.]+)"
+      DSType "GaugeMin"
+      Type "response_time"
+      Instance "delete_subnets"
+    </Match>
+    <Match>
+      Regex "POST /v2.0/ports.*time: ([0-9.]+)"
+      DSType "GaugeMin"
+      Type "response_time"
+      Instance "post_ports"
+    </Match>
+    <Match>
+      Regex "PUT /v2.0/ports.*time: ([0-9.]+)"
+      DSType "GaugeMin"
+      Type "response_time"
+      Instance "put_ports"
+    </Match>
+    <Match>
+      Regex "GET /v2.0/ports.*time: ([0-9.]+)"
+      DSType "GaugeMin"
+      Type "response_time"
+      Instance "get_ports"
+    </Match>
+    <Match>
+      Regex "DELETE /v2.0/ports.*time: ([0-9.]+)"
+      DSType "GaugeMin"
+      Type "response_time"
+      Instance "delete_ports"
+    </Match>
+    <Match>
+      Regex "POST /v2.0/trunks.*time: ([0-9.]+)"
+      DSType "GaugeMin"
+      Type "response_time"
+      Instance "post_trunks"
+    </Match>
+    <Match>
+      Regex "PUT /v2.0/trunks.*time: ([0-9.]+)"
+      DSType "GaugeMin"
+      Type "response_time"
+      Instance "put_trunks"
+    </Match>
+    <Match>
+      Regex "GET /v2.0/trunks.*time: ([0-9.]+)"
+      DSType "GaugeMin"
+      Type "response_time"
+      Instance "get_trunks"
+    </Match>
+    <Match>
+      Regex "DELETE /v2.0/ports.*time: ([0-9.]+)"
+      DSType "GaugeMin"
+      Type "response_time"
+      Instance "delete_trunks"
+    </Match>
+  {% endif %}
+  </File>
+
+  <File "/var/log/containers/neutron/server.log">
+  Instance "neutron_max_response_time"
+    <Match>
+      Regex "POST /v2.0.*time: ([0-9.]+)"
+      DSType "GaugeMax"
+      Type "response_time"
+      Instance "post"
+    </Match>
+    <Match>
+      Regex "PUT /v2.0.*time: ([0-9.]+)"
+      DSType "GaugeMax"
+      Type "response_time"
+      Instance "put"
+    </Match>
+    <Match>
+      Regex "GET /v2.0.*time: ([0-9.]+)"
+      DSType "GaugeMax"
+      Type "response_time"
+      Instance "get"
+    </Match>
+    <Match>
+      Regex "DELETE /v2.0.*time: ([0-9.]+)"
+      DSType "GaugeMax"
+      Type "response_time"
+      Instance "delete"
+    </Match>
+    {% if osp_individual_resources_response_time %}
+    <Match>
+      Regex "POST /v2.0/networks.*time: ([0-9.]+)"
+      DSType "GaugeMax"
+      Type "response_time"
+      Instance "post_networks"
+    </Match>
+    <Match>
+      Regex "PUT /v2.0/networks.*time: ([0-9.]+)"
+      DSType "GaugeMax"
+      Type "response_time"
+      Instance "put_networks"
+    </Match>
+    <Match>
+      Regex "GET /v2.0/networks.*time: ([0-9.]+)"
+      DSType "GaugeMax"
+      Type "response_time"
+      Instance "get_networks"
+    </Match>
+    <Match>
+      Regex "DELETE /v2.0/networks.*time: ([0-9.]+)"
+      DSType "GaugeMax"
+      Type "response_time"
+      Instance "delete_networks"
+    </Match>
+    <Match>
+      Regex "POST /v2.0/routers.*time: ([0-9.]+)"
+      DSType "GaugeMax"
+      Type "response_time"
+      Instance "post_routers"
+    </Match>
+    <Match>
+      Regex "PUT /v2.0/routers.*time: ([0-9.]+)"
+      DSType "GaugeMax"
+      Type "response_time"
+      Instance "put_routers"
+    </Match>
+    <Match>
+      Regex "GET /v2.0/routers.*time: ([0-9.]+)"
+      DSType "GaugeMax"
+      Type "response_time"
+      Instance "get_routers"
+    </Match>
+    <Match>
+      Regex "DELETE /v2.0/routers.*time: ([0-9.]+)"
+      DSType "GaugeMax"
+      Type "response_time"
+      Instance "delete_routers"
+    </Match>
+    <Match>
+      Regex "POST /v2.0/subnets.*time: ([0-9.]+)"
+      DSType "GaugeMax"
+      Type "response_time"
+      Instance "post_subnets"
+    </Match>
+    <Match>
+      Regex "PUT /v2.0/subnets.*time: ([0-9.]+)"
+      DSType "GaugeMax"
+      Type "response_time"
+      Instance "put_subnets"
+    </Match>
+    <Match>
+      Regex "GET /v2.0/subnets.*time: ([0-9.]+)"
+      DSType "GaugeMax"
+      Type "response_time"
+      Instance "get_subnets"
+    </Match>
+    <Match>
+      Regex "DELETE /v2.0/subnets.*time: ([0-9.]+)"
+      DSType "GaugeMax"
+      Type "response_time"
+      Instance "delete_subnets"
+    </Match>
+    <Match>
+      Regex "POST /v2.0/ports.*time: ([0-9.]+)"
+      DSType "GaugeMax"
+      Type "response_time"
+      Instance "post_ports"
+    </Match>
+    <Match>
+      Regex "PUT /v2.0/ports.*time: ([0-9.]+)"
+      DSType "GaugeMax"
+      Type "response_time"
+      Instance "put_ports"
+    </Match>
+    <Match>
+      Regex "GET /v2.0/ports.*time: ([0-9.]+)"
+      DSType "GaugeMax"
+      Type "response_time"
+      Instance "get_ports"
+    </Match>
+    <Match>
+      Regex "DELETE /v2.0/ports.*time: ([0-9.]+)"
+      DSType "GaugeMax"
+      Type "response_time"
+      Instance "delete_ports"
+    </Match>
+    <Match>
+      Regex "POST /v2.0/trunks.*time: ([0-9.]+)"
+      DSType "GaugeMax"
+      Type "response_time"
+      Instance "post_trunks"
+    </Match>
+    <Match>
+      Regex "PUT /v2.0/trunks.*time: ([0-9.]+)"
+      DSType "GaugeMax"
+      Type "response_time"
+      Instance "put_trunks"
+    </Match>
+    <Match>
+      Regex "GET /v2.0/trunks.*time: ([0-9.]+)"
+      DSType "GaugeMax"
+      Type "response_time"
+      Instance "get_trunks"
+    </Match>
+    <Match>
+      Regex "DELETE /v2.0/ports.*time: ([0-9.]+)"
+      DSType "GaugeMax"
+      Type "response_time"
+      Instance "delete_trunks"
+    </Match>
+  {% endif %}
+  </File>
+
+  <File "/var/log/containers/nova/nova-api.log">
+  Instance "nova_avg_response_time"
+    <Match>
+      Regex "POST /v2.1.*time: ([0-9.]+)"
+      DSType "GaugeAverage"
+      Type "response_time"
+      Instance "post"
+    </Match>
+    <Match>
+      Regex "PUT /v2.1.*time: ([0-9.]+)"
+      DSType "GaugeAverage"
+      Type "response_time"
+      Instance "put"
+    </Match>
+    <Match>
+      Regex "GET /v2.1.*time: ([0-9.]+)"
+      DSType "GaugeAverage"
+      Type "response_time"
+      Instance "get"
+    </Match>
+    <Match>
+      Regex "DELETE /v2.1.*time: ([0-9.]+)"
+      DSType "GaugeAverage"
+      Type "response_time"
+      Instance "delete"
+    </Match>
+  </File>
+
+  <File "/var/log/containers/nova/nova-api.log">
+  Instance "nova_min_response_time"
+    <Match>
+      Regex "POST /v2.1.*time: ([0-9.]+)"
+      DSType "GaugeMin"
+      Type "response_time"
+      Instance "post"
+    </Match>
+    <Match>
+      Regex "PUT /v2.1.*time: ([0-9.]+)"
+      DSType "GaugeMin"
+      Type "response_time"
+      Instance "put"
+    </Match>
+    <Match>
+      Regex "GET /v2.1.*time: ([0-9.]+)"
+      DSType "GaugeMin"
+      Type "response_time"
+      Instance "get"
+    </Match>
+    <Match>
+      Regex "DELETE /v2.1.*time: ([0-9.]+)"
+      DSType "GaugeMin"
+      Type "response_time"
+      Instance "delete"
+    </Match>
+  </File>
+
+  <File "/var/log/containers/nova/nova-api.log">
+  Instance "nova_max_response_time"
+    <Match>
+      Regex "POST /v2.1.*time: ([0-9.]+)"
+      DSType "GaugeMax"
+      Type "response_time"
+      Instance "post"
+    </Match>
+    <Match>
+      Regex "PUT /v2.1.*time: ([0-9.]+)"
+      DSType "GaugeMax"
+      Type "response_time"
+      Instance "put"
+    </Match>
+    <Match>
+      Regex "GET /v2.1.*time: ([0-9.]+)"
+      DSType "GaugeMax"
+      Type "response_time"
+      Instance "get"
+    </Match>
+    <Match>
+      Regex "DELETE /v2.1.*time: ([0-9.]+)"
+      DSType "GaugeMax"
+      Type "response_time"
+      Instance "delete"
+    </Match>
+  </File>
 {% endif %}

 # Tail httpd request time
--- a/ansible/install/roles/collectd/templates/controller.collectd.conf.j2
+++ b/ansible/install/roles/collectd/templates/controller.collectd.conf.j2
@ -818,6 +818,540 @@ PreCacheChain "PreCache"
      Instance "delete"
    </Match>
  </File>
+
+  <File "/var/log/containers/neutron/server.log">
+  Instance "neutron_avg_response_time"
+    <Match>
+      Regex "POST /v2.0.*time: ([0-9.]+)"
+      DSType "GaugeAverage"
+      Type "response_time"
+      Instance "post"
+    </Match>
+    <Match>
+      Regex "PUT /v2.0.*time: ([0-9.]+)"
+      DSType "GaugeAverage"
+      Type "response_time"
+      Instance "put"
+    </Match>
+    <Match>
+      Regex "GET /v2.0.*time: ([0-9.]+)"
+      DSType "GaugeAverage"
+      Type "response_time"
+      Instance "get"
+    </Match>
+    <Match>
+      Regex "DELETE /v2.0.*time: ([0-9.]+)"
+      DSType "GaugeAverage"
+      Type "response_time"
+      Instance "delete"
+    </Match>
+  {% if osp_individual_resources_response_time %}
+    <Match>
+      Regex "POST /v2.0/networks.*time: ([0-9.]+)"
+      DSType "GaugeAverage"
+      Type "response_time"
+      Instance "post_networks"
+    </Match>
+    <Match>
+      Regex "PUT /v2.0/networks.*time: ([0-9.]+)"
+      DSType "GaugeAverage"
+      Type "response_time"
+      Instance "put_networks"
+    </Match>
+    <Match>
+      Regex "GET /v2.0/networks.*time: ([0-9.]+)"
+      DSType "GaugeAverage"
+      Type "response_time"
+      Instance "get_networks"
+    </Match>
+    <Match>
+      Regex "DELETE /v2.0/networks.*time: ([0-9.]+)"
+      DSType "GaugeAverage"
+      Type "response_time"
+      Instance "delete_networks"
+    </Match>
+    <Match>
+      Regex "POST /v2.0/routers.*time: ([0-9.]+)"
+      DSType "GaugeAverage"
+      Type "response_time"
+      Instance "post_routers"
+    </Match>
+    <Match>
+      Regex "PUT /v2.0/routers.*time: ([0-9.]+)"
+      DSType "GaugeAverage"
+      Type "response_time"
+      Instance "put_routers"
+    </Match>
+    <Match>
+      Regex "GET /v2.0/routers.*time: ([0-9.]+)"
+      DSType "GaugeAverage"
+      Type "response_time"
+      Instance "get_routers"
+    </Match>
+    <Match>
+      Regex "DELETE /v2.0/routers.*time: ([0-9.]+)"
+      DSType "GaugeAverage"
+      Type "response_time"
+      Instance "delete_routers"
+    </Match>
+    <Match>
+      Regex "POST /v2.0/subnets.*time: ([0-9.]+)"
+      DSType "GaugeAverage"
+      Type "response_time"
+      Instance "post_subnets"
+    </Match>
+    <Match>
+      Regex "PUT /v2.0/subnets.*time: ([0-9.]+)"
+      DSType "GaugeAverage"
+      Type "response_time"
+      Instance "put_subnets"
+    </Match>
+    <Match>
+      Regex "GET /v2.0/subnets.*time: ([0-9.]+)"
+      DSType "GaugeAverage"
+      Type "response_time"
+      Instance "get_subnets"
+    </Match>
+    <Match>
+      Regex "DELETE /v2.0/subnets.*time: ([0-9.]+)"
+      DSType "GaugeAverage"
+      Type "response_time"
+      Instance "delete_subnets"
+    </Match>
+    <Match>
+      Regex "POST /v2.0/ports.*time: ([0-9.]+)"
+      DSType "GaugeAverage"
+      Type "response_time"
+      Instance "post_ports"
+    </Match>
+    <Match>
+      Regex "PUT /v2.0/ports.*time: ([0-9.]+)"
+      DSType "GaugeAverage"
+      Type "response_time"
+      Instance "put_ports"
+    </Match>
+    <Match>
+      Regex "GET /v2.0/ports.*time: ([0-9.]+)"
+      DSType "GaugeAverage"
+      Type "response_time"
+      Instance "get_ports"
+    </Match>
+    <Match>
+      Regex "DELETE /v2.0/ports.*time: ([0-9.]+)"
+      DSType "GaugeAverage"
+      Type "response_time"
+      Instance "delete_ports"
+    </Match>
+    <Match>
+      Regex "POST /v2.0/trunks.*time: ([0-9.]+)"
+      DSType "GaugeAverage"
+      Type "response_time"
+      Instance "post_trunks"
+    </Match>
+    <Match>
+      Regex "PUT /v2.0/trunks.*time: ([0-9.]+)"
+      DSType "GaugeAverage"
+      Type "response_time"
+      Instance "put_trunks"
+    </Match>
+    <Match>
+      Regex "GET /v2.0/trunks.*time: ([0-9.]+)"
+      DSType "GaugeAverage"
+      Type "response_time"
+      Instance "get_trunks"
+    </Match>
+    <Match>
+      Regex "DELETE /v2.0/trunks.*time: ([0-9.]+)"
+      DSType "GaugeAverage"
+      Type "response_time"
+      Instance "delete_trunks"
+    </Match>
+  {% endif %}
+  </File>
+
+  <File "/var/log/containers/neutron/server.log">
+  Instance "neutron_min_response_time"
+    <Match>
+      Regex "POST /v2.0.*time: ([0-9.]+)"
+      DSType "GaugeMin"
+      Type "response_time"
+      Instance "post"
+    </Match>
+    <Match>
+      Regex "PUT /v2.0.*time: ([0-9.]+)"
+      DSType "GaugeMin"
+      Type "response_time"
+      Instance "put"
+    </Match>
+    <Match>
+      Regex "GET /v2.0.*time: ([0-9.]+)"
+      DSType "GaugeMin"
+      Type "response_time"
+      Instance "get"
+    </Match>
+    <Match>
+      Regex "DELETE /v2.0.*time: ([0-9.]+)"
+      DSType "GaugeMin"
+      Type "response_time"
+      Instance "delete"
+    </Match>
+  {% if osp_individual_resources_response_time %}
+    <Match>
+      Regex "POST /v2.0/networks.*time: ([0-9.]+)"
+      DSType "GaugeMin"
+      Type "response_time"
+      Instance "post_networks"
+    </Match>
+    <Match>
+      Regex "PUT /v2.0/networks.*time: ([0-9.]+)"
+      DSType "GaugeMin"
+      Type "response_time"
+      Instance "put_networks"
+    </Match>
+    <Match>
+      Regex "GET /v2.0/networks.*time: ([0-9.]+)"
+      DSType "GaugeMin"
+      Type "response_time"
+      Instance "get_networks"
+    </Match>
+    <Match>
+      Regex "DELETE /v2.0/networks.*time: ([0-9.]+)"
+      DSType "GaugeMin"
+      Type "response_time"
+      Instance "delete_networks"
+    </Match>
+    <Match>
+      Regex "POST /v2.0/routers.*time: ([0-9.]+)"
+      DSType "GaugeMin"
+      Type "response_time"
+      Instance "post_routers"
+    </Match>
+    <Match>
+      Regex "PUT /v2.0/routers.*time: ([0-9.]+)"
+      DSType "GaugeMin"
+      Type "response_time"
+      Instance "put_routers"
+    </Match>
+    <Match>
+      Regex "GET /v2.0/routers.*time: ([0-9.]+)"
+      DSType "GaugeMin"
+      Type "response_time"
+      Instance "get_routers"
+    </Match>
+    <Match>
+      Regex "DELETE /v2.0/routers.*time: ([0-9.]+)"
+      DSType "GaugeMin"
+      Type "response_time"
+      Instance "delete_routers"
+    </Match>
+    <Match>
+      Regex "POST /v2.0/subnets.*time: ([0-9.]+)"
+      DSType "GaugeMin"
+      Type "response_time"
+      Instance "post_subnets"
+    </Match>
+    <Match>
+      Regex "PUT /v2.0/subnets.*time: ([0-9.]+)"
+      DSType "GaugeMin"
+      Type "response_time"
+      Instance "put_subnets"
+    </Match>
+    <Match>
+      Regex "GET /v2.0/subnets.*time: ([0-9.]+)"
+      DSType "GaugeMin"
+      Type "response_time"
+      Instance "get_subnets"
+    </Match>
+    <Match>
+      Regex "DELETE /v2.0/subnets.*time: ([0-9.]+)"
+      DSType "GaugeMin"
+      Type "response_time"
+      Instance "delete_subnets"
+    </Match>
+    <Match>
+      Regex "POST /v2.0/ports.*time: ([0-9.]+)"
+      DSType "GaugeMin"
+      Type "response_time"
+      Instance "post_ports"
+    </Match>
+    <Match>
+      Regex "PUT /v2.0/ports.*time: ([0-9.]+)"
+      DSType "GaugeMin"
+      Type "response_time"
+      Instance "put_ports"
+    </Match>
+    <Match>
+      Regex "GET /v2.0/ports.*time: ([0-9.]+)"
+      DSType "GaugeMin"
+      Type "response_time"
+      Instance "get_ports"
+    </Match>
+    <Match>
+      Regex "DELETE /v2.0/ports.*time: ([0-9.]+)"
+      DSType "GaugeMin"
+      Type "response_time"
+      Instance "delete_ports"
+    </Match>
+    <Match>
+      Regex "POST /v2.0/trunks.*time: ([0-9.]+)"
+      DSType "GaugeMin"
+      Type "response_time"
+      Instance "post_trunks"
+    </Match>
+    <Match>
+      Regex "PUT /v2.0/trunks.*time: ([0-9.]+)"
+      DSType "GaugeMin"
+      Type "response_time"
+      Instance "put_trunks"
+    </Match>
+    <Match>
+      Regex "GET /v2.0/trunks.*time: ([0-9.]+)"
+      DSType "GaugeMin"
+      Type "response_time"
+      Instance "get_trunks"
+    </Match>
+    <Match>
+      Regex "DELETE /v2.0/ports.*time: ([0-9.]+)"
+      DSType "GaugeMin"
+      Type "response_time"
+      Instance "delete_trunks"
+    </Match>
+  {% endif %}
+  </File>
+
+  <File "/var/log/containers/neutron/server.log">
+  Instance "neutron_max_response_time"
+    <Match>
+      Regex "POST /v2.0.*time: ([0-9.]+)"
+      DSType "GaugeMax"
+      Type "response_time"
+      Instance "post"
+    </Match>
+    <Match>
+      Regex "PUT /v2.0.*time: ([0-9.]+)"
+      DSType "GaugeMax"
+      Type "response_time"
+      Instance "put"
+    </Match>
+    <Match>
+      Regex "GET /v2.0.*time: ([0-9.]+)"
+      DSType "GaugeMax"
+      Type "response_time"
+      Instance "get"
+    </Match>
+    <Match>
+      Regex "DELETE /v2.0.*time: ([0-9.]+)"
+      DSType "GaugeMax"
+      Type "response_time"
+      Instance "delete"
+    </Match>
+    {% if osp_individual_resources_response_time %}
+    <Match>
+      Regex "POST /v2.0/networks.*time: ([0-9.]+)"
+      DSType "GaugeMax"
+      Type "response_time"
+      Instance "post_networks"
+    </Match>
+    <Match>
+      Regex "PUT /v2.0/networks.*time: ([0-9.]+)"
+      DSType "GaugeMax"
+      Type "response_time"
+      Instance "put_networks"
+    </Match>
+    <Match>
+      Regex "GET /v2.0/networks.*time: ([0-9.]+)"
+      DSType "GaugeMax"
+      Type "response_time"
+      Instance "get_networks"
+    </Match>
+    <Match>
+      Regex "DELETE /v2.0/networks.*time: ([0-9.]+)"
+      DSType "GaugeMax"
+      Type "response_time"
+      Instance "delete_networks"
+    </Match>
+    <Match>
+      Regex "POST /v2.0/routers.*time: ([0-9.]+)"
+      DSType "GaugeMax"
+      Type "response_time"
+      Instance "post_routers"
+    </Match>
+    <Match>
+      Regex "PUT /v2.0/routers.*time: ([0-9.]+)"
+      DSType "GaugeMax"
+      Type "response_time"
+      Instance "put_routers"
+    </Match>
+    <Match>
+      Regex "GET /v2.0/routers.*time: ([0-9.]+)"
+      DSType "GaugeMax"
+      Type "response_time"
+      Instance "get_routers"
+    </Match>
+    <Match>
+      Regex "DELETE /v2.0/routers.*time: ([0-9.]+)"
+      DSType "GaugeMax"
+      Type "response_time"
+      Instance "delete_routers"
+    </Match>
+    <Match>
+      Regex "POST /v2.0/subnets.*time: ([0-9.]+)"
+      DSType "GaugeMax"
+      Type "response_time"
+      Instance "post_subnets"
+    </Match>
+    <Match>
+      Regex "PUT /v2.0/subnets.*time: ([0-9.]+)"
+      DSType "GaugeMax"
+      Type "response_time"
+      Instance "put_subnets"
+    </Match>
+    <Match>
+      Regex "GET /v2.0/subnets.*time: ([0-9.]+)"
+      DSType "GaugeMax"
+      Type "response_time"
+      Instance "get_subnets"
+    </Match>
+    <Match>
+      Regex "DELETE /v2.0/subnets.*time: ([0-9.]+)"
+      DSType "GaugeMax"
+      Type "response_time"
+      Instance "delete_subnets"
+    </Match>
+    <Match>
+      Regex "POST /v2.0/ports.*time: ([0-9.]+)"
+      DSType "GaugeMax"
+      Type "response_time"
+      Instance "post_ports"
+    </Match>
+    <Match>
+      Regex "PUT /v2.0/ports.*time: ([0-9.]+)"
+      DSType "GaugeMax"
+      Type "response_time"
+      Instance "put_ports"
+    </Match>
+    <Match>
+      Regex "GET /v2.0/ports.*time: ([0-9.]+)"
+      DSType "GaugeMax"
+      Type "response_time"
+      Instance "get_ports"
+    </Match>
+    <Match>
+      Regex "DELETE /v2.0/ports.*time: ([0-9.]+)"
+      DSType "GaugeMax"
+      Type "response_time"
+      Instance "delete_ports"
+    </Match>
+    <Match>
+      Regex "POST /v2.0/trunks.*time: ([0-9.]+)"
+      DSType "GaugeMax"
+      Type "response_time"
+      Instance "post_trunks"
+    </Match>
+    <Match>
+      Regex "PUT /v2.0/trunks.*time: ([0-9.]+)"
+      DSType "GaugeMax"
+      Type "response_time"
+      Instance "put_trunks"
+    </Match>
+    <Match>
+      Regex "GET /v2.0/trunks.*time: ([0-9.]+)"
+      DSType "GaugeMax"
+      Type "response_time"
+      Instance "get_trunks"
+    </Match>
+    <Match>
+      Regex "DELETE /v2.0/ports.*time: ([0-9.]+)"
+      DSType "GaugeMax"
+      Type "response_time"
+      Instance "delete_trunks"
+    </Match>
+  {% endif %}
+  </File>
+
+  <File "/var/log/containers/nova/nova-api.log">
+  Instance "nova_avg_response_time"
+    <Match>
+      Regex "POST /v2.1.*time: ([0-9.]+)"
+      DSType "GaugeAverage"
+      Type "response_time"
+      Instance "post"
+    </Match>
+    <Match>
+      Regex "PUT /v2.1.*time: ([0-9.]+)"
+      DSType "GaugeAverage"
+      Type "response_time"
+      Instance "put"
+    </Match>
+    <Match>
+      Regex "GET /v2.1.*time: ([0-9.]+)"
+      DSType "GaugeAverage"
+      Type "response_time"
+      Instance "get"
+    </Match>
+    <Match>
+      Regex "DELETE /v2.1.*time: ([0-9.]+)"
+      DSType "GaugeAverage"
+      Type "response_time"
+      Instance "delete"
+    </Match>
+  </File>
+
+  <File "/var/log/containers/nova/nova-api.log">
+  Instance "nova_min_response_time"
+    <Match>
+      Regex "POST /v2.1.*time: ([0-9.]+)"
+      DSType "GaugeMin"
+      Type "response_time"
+      Instance "post"
+    </Match>
+    <Match>
+      Regex "PUT /v2.1.*time: ([0-9.]+)"
+      DSType "GaugeMin"
+      Type "response_time"
+      Instance "put"
+    </Match>
+    <Match>
+      Regex "GET /v2.1.*time: ([0-9.]+)"
+      DSType "GaugeMin"
+      Type "response_time"
+      Instance "get"
+    </Match>
+    <Match>
+      Regex "DELETE /v2.1.*time: ([0-9.]+)"
+      DSType "GaugeMin"
+      Type "response_time"
+      Instance "delete"
+    </Match>
+  </File>
+
+  <File "/var/log/containers/nova/nova-api.log">
+  Instance "nova_max_response_time"
+    <Match>
+      Regex "POST /v2.1.*time: ([0-9.]+)"
+      DSType "GaugeMax"
+      Type "response_time"
+      Instance "post"
+    </Match>
+    <Match>
+      Regex "PUT /v2.1.*time: ([0-9.]+)"
+      DSType "GaugeMax"
+      Type "response_time"
+      Instance "put"
+    </Match>
+    <Match>
+      Regex "GET /v2.1.*time: ([0-9.]+)"
+      DSType "GaugeMax"
+      Type "response_time"
+      Instance "get"
+    </Match>
+    <Match>
+      Regex "DELETE /v2.1.*time: ([0-9.]+)"
+      DSType "GaugeMax"
+      Type "response_time"
+      Instance "delete"
+    </Match>
+  </File>
 {% endif %}


--- a/ansible/install/roles/grafana-dashboards/templates/openstack_general_system_performance.yaml.j2
+++ b/ansible/install/roles/grafana-dashboards/templates/openstack_general_system_performance.yaml.j2
@ -162,6 +162,8 @@ dashboard:

 {% if item.template_node_type in controller_groups %}
  {% include 'partials/osp_resources.yaml' %}
+
+  {% include 'partials/osp_response_times.yaml' %}
 {% endif %}

 {% include 'partials/ovn_metrics.yaml' %}
--- a/ansible/install/roles/grafana-dashboards/templates/partials/osp_response_times.yaml
+++ b/ansible/install/roles/grafana-dashboards/templates/partials/osp_response_times.yaml
@ -0,0 +1,131 @@
+    - title: OSP Resource Response Times
+      collapse: true
+      height: 200px
+      showTitle: true
+      panels:
+        - title: $Cloud - $Node - OSP REST API Average Response Times
+          type: graph
+          legend:
+            alignAsTable: true
+            avg: false
+            current: true
+            max: true
+            min: true
+            rightSide: true
+            show: true
+            total: false
+            values: true
+          nullPointMode: 'null'
+          targets:
+            - target: alias($Cloud.$Node.tail-nova_avg_response_time.response_time-get, 'nova_get')
+            - target: alias($Cloud.$Node.tail-nova_avg_response_time.response_time-post, 'nova_post')
+            - target: alias($Cloud.$Node.tail-nova_avg_response_time.response_time-put, 'nova_put')
+            - target: alias($Cloud.$Node.tail-nova_avg_response_time.response_time-delete, 'nova_delete')
+            - target: alias($Cloud.$Node.tail-neutron_avg_response_time.response_time-get, 'neutron_get')
+            - target: alias($Cloud.$Node.tail-neutron_avg_response_time.response_time-post, 'neutron_post')
+            - target: alias($Cloud.$Node.tail-neutron_avg_response_time.response_time-put, 'neutron_put')
+            - target: alias($Cloud.$Node.tail-neutron_avg_response_time.response_time-delete, 'neutron_delete')
+            - target: alias($Cloud.$Node.tail-neutron_avg_response_time.response_time-get_networks, 'neutron_get_networks')
+            - target: alias($Cloud.$Node.tail-neutron_avg_response_time.response_time-post_networks, 'neutron_post_networks')
+            - target: alias($Cloud.$Node.tail-neutron_avg_response_time.response_time-put_networks, 'neutron_put_networks')
+            - target: alias($Cloud.$Node.tail-neutron_avg_response_time.response_time-delete_networks, 'neutron_delete_networks')
+            - target: alias($Cloud.$Node.tail-neutron_avg_response_time.response_time-get_routers, 'neutron_get_routers')
+            - target: alias($Cloud.$Node.tail-neutron_avg_response_time.response_time-post_routers, 'neutron_post_routers')
+            - target: alias($Cloud.$Node.tail-neutron_avg_response_time.response_time-put_routers, 'neutron_put_routers')
+            - target: alias($Cloud.$Node.tail-neutron_avg_response_time.response_time-delete_routers, 'neutron_delete_routers')
+            - target: alias($Cloud.$Node.tail-neutron_avg_response_time.response_time-get_subnets, 'neutron_get_subnets')
+            - target: alias($Cloud.$Node.tail-neutron_avg_response_time.response_time-post_subnets, 'neutron_post_subnets')
+            - target: alias($Cloud.$Node.tail-neutron_avg_response_time.response_time-put_subnets, 'neutron_put_subnets')
+            - target: alias($Cloud.$Node.tail-neutron_avg_response_time.response_time-delete_subnets, 'neutron_delete_subnets')
+            - target: alias($Cloud.$Node.tail-neutron_avg_response_time.response_time-get_ports, 'neutron_get_ports')
+            - target: alias($Cloud.$Node.tail-neutron_avg_response_time.response_time-post_ports, 'neutron_post_ports')
+            - target: alias($Cloud.$Node.tail-neutron_avg_response_time.response_time-put_ports, 'neutron_put_ports')
+            - target: alias($Cloud.$Node.tail-neutron_avg_response_time.response_time-delete_ports, 'neutron_delete_ports')
+            - target: alias($Cloud.$Node.tail-neutron_avg_response_time.response_time-get_trunks, 'neutron_get_trunks')
+            - target: alias($Cloud.$Node.tail-neutron_avg_response_time.response_time-post_trunks, 'neutron_post_trunks')
+            - target: alias($Cloud.$Node.tail-neutron_avg_response_time.response_time-put_trunks, 'neutron_put_trunks')
+            - target: alias($Cloud.$Node.tail-neutron_avg_response_time.response_time-delete_trunks, 'neutron_delete_trunks')
+        - title: $Cloud - $Node - OSP REST API Minimum Response Times
+          type: graph
+          legend:
+            alignAsTable: true
+            avg: false
+            current: true
+            max: true
+            min: true
+            rightSide: true
+            show: true
+            total: false
+            values: true
+          nullPointMode: 'null'
+          targets:
+            - target: alias($Cloud.$Node.tail-nova_min_response_time.response_time-get, 'nova_get')
+            - target: alias($Cloud.$Node.tail-nova_min_response_time.response_time-post, 'nova_post')
+            - target: alias($Cloud.$Node.tail-nova_min_response_time.response_time-put, 'nova_put')
+            - target: alias($Cloud.$Node.tail-nova_min_response_time.response_time-delete, 'nova_delete')
+            - target: alias($Cloud.$Node.tail-neutron_min_response_time.response_time-get, 'neutron_get')
+            - target: alias($Cloud.$Node.tail-neutron_min_response_time.response_time-post, 'neutron_post')
+            - target: alias($Cloud.$Node.tail-neutron_min_response_time.response_time-put, 'neutron_put')
+            - target: alias($Cloud.$Node.tail-neutron_min_response_time.response_time-delete, 'neutron_delete')
+            - target: alias($Cloud.$Node.tail-neutron_min_response_time.response_time-get_networks, 'neutron_get_networks')
+            - target: alias($Cloud.$Node.tail-neutron_min_response_time.response_time-post_networks, 'neutron_post_networks')
+            - target: alias($Cloud.$Node.tail-neutron_min_response_time.response_time-put_networks, 'neutron_put_networks')
+            - target: alias($Cloud.$Node.tail-neutron_min_response_time.response_time-delete_networks, 'neutron_delete_networks')
+            - target: alias($Cloud.$Node.tail-neutron_min_response_time.response_time-get_routers, 'neutron_get_routers')
+            - target: alias($Cloud.$Node.tail-neutron_min_response_time.response_time-post_routers, 'neutron_post_routers')
+            - target: alias($Cloud.$Node.tail-neutron_min_response_time.response_time-put_routers, 'neutron_put_routers')
+            - target: alias($Cloud.$Node.tail-neutron_min_response_time.response_time-delete_routers, 'neutron_delete_routers')
+            - target: alias($Cloud.$Node.tail-neutron_min_response_time.response_time-get_subnets, 'neutron_get_subnets')
+            - target: alias($Cloud.$Node.tail-neutron_min_response_time.response_time-post_subnets, 'neutron_post_subnets')
+            - target: alias($Cloud.$Node.tail-neutron_min_response_time.response_time-put_subnets, 'neutron_put_subnets')
+            - target: alias($Cloud.$Node.tail-neutron_min_response_time.response_time-delete_subnets, 'neutron_delete_subnets')
+            - target: alias($Cloud.$Node.tail-neutron_min_response_time.response_time-get_ports, 'neutron_get_ports')
+            - target: alias($Cloud.$Node.tail-neutron_min_response_time.response_time-post_ports, 'neutron_post_ports')
+            - target: alias($Cloud.$Node.tail-neutron_min_response_time.response_time-put_ports, 'neutron_put_ports')
+            - target: alias($Cloud.$Node.tail-neutron_min_response_time.response_time-delete_ports, 'neutron_delete_ports')
+            - target: alias($Cloud.$Node.tail-neutron_min_response_time.response_time-get_trunks, 'neutron_get_trunks')
+            - target: alias($Cloud.$Node.tail-neutron_min_response_time.response_time-post_trunks, 'neutron_post_trunks')
+            - target: alias($Cloud.$Node.tail-neutron_min_response_time.response_time-put_trunks, 'neutron_put_trunks')
+            - target: alias($Cloud.$Node.tail-neutron_min_response_time.response_time-delete_trunks, 'neutron_delete_trunks')
+        - title: $Cloud - $Node - OSP REST API Maximum Response Times
+          type: graph
+          legend:
+            alignAsTable: true
+            avg: false
+            current: true
+            max: true
+            min: true
+            rightSide: true
+            show: true
+            total: false
+            values: true
+          nullPointMode: 'null'
+          targets:
+            - target: alias($Cloud.$Node.tail-nova_max_response_time.response_time-get, 'nova_get')
+            - target: alias($Cloud.$Node.tail-nova_max_response_time.response_time-post, 'nova_post')
+            - target: alias($Cloud.$Node.tail-nova_max_response_time.response_time-put, 'nova_put')
+            - target: alias($Cloud.$Node.tail-nova_max_response_time.response_time-delete, 'nova_delete')
+            - target: alias($Cloud.$Node.tail-neutron_max_response_time.response_time-get, 'neutron_get')
+            - target: alias($Cloud.$Node.tail-neutron_max_response_time.response_time-post, 'neutron_post')
+            - target: alias($Cloud.$Node.tail-neutron_max_response_time.response_time-put, 'neutron_put')
+            - target: alias($Cloud.$Node.tail-neutron_max_response_time.response_time-delete, 'neutron_delete')
+            - target: alias($Cloud.$Node.tail-neutron_max_response_time.response_time-get_networks, 'neutron_get_networks')
+            - target: alias($Cloud.$Node.tail-neutron_max_response_time.response_time-post_networks, 'neutron_post_networks')
+            - target: alias($Cloud.$Node.tail-neutron_max_response_time.response_time-put_networks, 'neutron_put_networks')
+            - target: alias($Cloud.$Node.tail-neutron_max_response_time.response_time-delete_networks, 'neutron_delete_networks')
+            - target: alias($Cloud.$Node.tail-neutron_max_response_time.response_time-get_routers, 'neutron_get_routers')
+            - target: alias($Cloud.$Node.tail-neutron_max_response_time.response_time-post_routers, 'neutron_post_routers')
+            - target: alias($Cloud.$Node.tail-neutron_max_response_time.response_time-put_routers, 'neutron_put_routers')
+            - target: alias($Cloud.$Node.tail-neutron_max_response_time.response_time-delete_routers, 'neutron_delete_routers')
+            - target: alias($Cloud.$Node.tail-neutron_max_response_time.response_time-get_subnets, 'neutron_get_subnets')
+            - target: alias($Cloud.$Node.tail-neutron_max_response_time.response_time-post_subnets, 'neutron_post_subnets')
+            - target: alias($Cloud.$Node.tail-neutron_max_response_time.response_time-put_subnets, 'neutron_put_subnets')
+            - target: alias($Cloud.$Node.tail-neutron_max_response_time.response_time-delete_subnets, 'neutron_delete_subnets')
+            - target: alias($Cloud.$Node.tail-neutron_max_response_time.response_time-get_ports, 'neutron_get_ports')
+            - target: alias($Cloud.$Node.tail-neutron_max_response_time.response_time-post_ports, 'neutron_post_ports')
+            - target: alias($Cloud.$Node.tail-neutron_max_response_time.response_time-put_ports, 'neutron_put_ports')
+            - target: alias($Cloud.$Node.tail-neutron_max_response_time.response_time-delete_ports, 'neutron_delete_ports')
+            - target: alias($Cloud.$Node.tail-neutron_max_response_time.response_time-get_trunks, 'neutron_get_trunks')
+            - target: alias($Cloud.$Node.tail-neutron_max_response_time.response_time-post_trunks, 'neutron_post_trunks')
+            - target: alias($Cloud.$Node.tail-neutron_max_response_time.response_time-put_trunks, 'neutron_put_trunks')
+            - target: alias($Cloud.$Node.tail-neutron_max_response_time.response_time-delete_trunks, 'neutron_delete_trunks')