From 033db44f1c4e1a009625811f64d03a2b1ceec28c Mon Sep 17 00:00:00 2001
From: Will Szumski <will@stackhpc.com>
Date: Fri, 11 Feb 2022 18:26:41 +0000
Subject: [PATCH] Adds prometheus_scrape_interval

Grafana requires the scrape interval to be set to be able to compute
$__rate_interval. The default is 15s which does not match the kolla
default of 60s. The symptom of not setting this is that you will see
"no data" when zooming graphs that use rate queries. This occurs as the
interval will be set to a period shorter than the scrape interval.
The recommendation is that you use a common scrape interval for all
jobs. See:

- https://grafana.com/blog/2020/09/28/new-in-grafana-7.2-__rate_interval-for-prometheus-rate-queries-that-just-work/
- https://stackoverflow.com/questions/66369969/set-scrape-interval-in-provisioned-prometheus-data-source-in-grafana

Change-Id: I7e5c1e20c7b66b64cbd333f669ef8d8da60daaa8
---
 ansible/group_vars/all.yml                         |  5 +++--
 ansible/roles/grafana/templates/prometheus.yaml.j2 |  3 +++
 .../roles/prometheus/templates/prometheus.yml.j2   |  2 +-
 .../logging-and-monitoring/prometheus-guide.rst    | 14 ++++++++++++++
 ...rometheus-scrape-interval-89b791f93413d070.yaml | 12 ++++++++++++
 5 files changed, 33 insertions(+), 3 deletions(-)
 create mode 100644 releasenotes/notes/adds-prometheus-scrape-interval-89b791f93413d070.yaml

diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml
index 2bc744b2ee..ffb1fe8e58 100644
--- a/ansible/group_vars/all.yml
+++ b/ansible/group_vars/all.yml
@@ -1108,9 +1108,10 @@ enable_prometheus_blackbox_exporter: "{{ enable_prometheus | bool }}"
 enable_prometheus_rabbitmq_exporter: "{{ enable_prometheus | bool and enable_rabbitmq | bool }}"
 
 prometheus_alertmanager_user: "admin"
-prometheus_openstack_exporter_interval: "60s"
+prometheus_scrape_interval: "60s"
+prometheus_openstack_exporter_interval: "{{ prometheus_scrape_interval }}"
 prometheus_openstack_exporter_timeout: "10s"
-prometheus_elasticsearch_exporter_interval: "60s"
+prometheus_elasticsearch_exporter_interval: "{{ prometheus_scrape_interval }}"
 prometheus_cmdline_extras:
 prometheus_ceph_mgr_exporter_endpoints: []
 prometheus_openstack_exporter_endpoint_type: "internal"
diff --git a/ansible/roles/grafana/templates/prometheus.yaml.j2 b/ansible/roles/grafana/templates/prometheus.yaml.j2
index 870da4b7f9..3021a791be 100644
--- a/ansible/roles/grafana/templates/prometheus.yaml.j2
+++ b/ansible/roles/grafana/templates/prometheus.yaml.j2
@@ -7,3 +7,6 @@ datasources:
   orgId: 1
   url: {{ grafana_prometheus_url }}
   version: 1
+  jsonData:
+    timeInterval: {{ prometheus_scrape_interval }}
+
diff --git a/ansible/roles/prometheus/templates/prometheus.yml.j2 b/ansible/roles/prometheus/templates/prometheus.yml.j2
index ac11aa3869..f6153b9a36 100644
--- a/ansible/roles/prometheus/templates/prometheus.yml.j2
+++ b/ansible/roles/prometheus/templates/prometheus.yml.j2
@@ -1,5 +1,5 @@
 global:
-  scrape_interval: 60s
+  scrape_interval: {{ prometheus_scrape_interval }}
   scrape_timeout: 10s
   evaluation_interval: 15s
   external_labels:
diff --git a/doc/source/reference/logging-and-monitoring/prometheus-guide.rst b/doc/source/reference/logging-and-monitoring/prometheus-guide.rst
index aa15b52014..3d67bb92b1 100644
--- a/doc/source/reference/logging-and-monitoring/prometheus-guide.rst
+++ b/doc/source/reference/logging-and-monitoring/prometheus-guide.rst
@@ -45,6 +45,20 @@ and data retention size to 30 gigabytes:
 
    prometheus_cmdline_extras: "--query.timeout=1m --storage.tsdb.retention.size=30GB"
 
+Configuration options
+~~~~~~~~~~~~~~~~~~~~~
+
+.. list-table:: Configuration options
+   :widths: 25 25 75
+   :header-rows: 1
+
+   * - Option
+     - Default
+     - Description
+   * - prometheus_scrape_interval
+     - 60s
+     - Default scape interval for all jobs
+
 Extending prometheus.cfg
 ~~~~~~~~~~~~~~~~~~~~~~~~
 
diff --git a/releasenotes/notes/adds-prometheus-scrape-interval-89b791f93413d070.yaml b/releasenotes/notes/adds-prometheus-scrape-interval-89b791f93413d070.yaml
new file mode 100644
index 0000000000..741ff6a8d5
--- /dev/null
+++ b/releasenotes/notes/adds-prometheus-scrape-interval-89b791f93413d070.yaml
@@ -0,0 +1,12 @@
+---
+features:
+  - |
+    Adds the ``prometheus_scrape_interval`` configuration option. The default
+    is set to ``60s``. This configures the default scrape interval for all
+    jobs.
+fixes:
+  - |
+    The scrape interval for the prometheus data source in grafana is now to set
+    to ``prometheus_scrape_interval``. This fixes issues with dashboards that
+    use the ``$__rate_interval`` grafana variable as the default scrape
+    interval of 60s does not match the grafana default of 15s.