Merge "Use friendly prometheus instance labels"

ansible/group_vars/all.yml

@@ -500,6 +500,9 @@ prometheus_elasticsearch_exporter_port: "9108"
 # Prometheus blackbox-exporter ports
 prometheus_blackbox_exporter_port: "9115"
 
+# Prometheus instance label to use for metrics
+prometheus_instance_label:
+
 proxysql_admin_port: "6032"
 
 rabbitmq_port: "{{ '5671' if rabbitmq_enable_tls | bool else '5672' }}"

ansible/roles/prometheus/templates/prometheus.yml.j2

@@ -19,71 +19,103 @@ rule_files:
 scrape_configs:
   - job_name: prometheus
     static_configs:
-      - targets:
 {% for host in groups['prometheus'] %}
+      - targets:
         - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ prometheus_port }}'
+{% if hostvars[host].prometheus_instance_label | default(false, true) %}
+        labels:
+          instance: "{{ hostvars[host].prometheus_instance_label }}"
+{% endif %}
 {% endfor %}
 
 {% if enable_prometheus_node_exporter | bool %}
   - job_name: node
     static_configs:
-      - targets:
 {% for host in groups['prometheus-node-exporter'] %}
+      - targets:
         - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_node_exporter_port'] }}'
+{% if hostvars[host].prometheus_instance_label | default(false, true) %}
+        labels:
+          instance: "{{ hostvars[host].prometheus_instance_label }}"
+{% endif %}
 {% endfor %}
 {% endif %}
 
 {% if enable_prometheus_mysqld_exporter | bool %}
   - job_name: mysqld
     static_configs:
-      - targets:
 {% for host in groups['prometheus-mysqld-exporter'] %}
+      - targets:
         - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_mysqld_exporter_port'] }}'
+{% if hostvars[host].prometheus_instance_label | default(false, true) %}
+        labels:
+          instance: "{{ hostvars[host].prometheus_instance_label }}"
+{% endif %}
 {% endfor %}
 {% endif %}
 
 {% if enable_prometheus_haproxy_exporter | bool %}
   - job_name: haproxy
     static_configs:
-      - targets:
 {% for host in groups['prometheus-haproxy-exporter'] %}
+      - targets:
         - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_haproxy_exporter_port'] }}'
+{% if hostvars[host].prometheus_instance_label | default(false, true) %}
+        labels:
+          instance: "{{ hostvars[host].prometheus_instance_label }}"
+{% endif %}
 {% endfor %}
 {% endif %}
 
 {% if enable_prometheus_rabbitmq_exporter | bool %}
   - job_name: rabbitmq
     static_configs:
-      - targets:
 {% for host in groups['rabbitmq'] %}
+      - targets:
         - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_rabbitmq_exporter_port'] }}'
+{% if hostvars[host].prometheus_instance_label | default(false, true) %}
+        labels:
+          instance: "{{ hostvars[host].prometheus_instance_label }}"
+{% endif %}
 {% endfor %}
 {% endif %}
 
 {% if enable_prometheus_memcached_exporter | bool %}
   - job_name: memcached
     static_configs:
-      - targets:
 {% for host in groups['prometheus-memcached-exporter'] %}
+      - targets:
         - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_memcached_exporter_port'] }}'
+{% if hostvars[host].prometheus_instance_label | default(false, true) %}
+        labels:
+          instance: "{{ hostvars[host].prometheus_instance_label }}"
+{% endif %}
 {% endfor %}
 {% endif %}
 
 {% if enable_prometheus_cadvisor | bool %}
   - job_name: cadvisor
     static_configs:
-      - targets:
 {% for host in groups["prometheus-cadvisor"] %}
+      - targets:
         - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_cadvisor_port'] }}'
+{% if hostvars[host].prometheus_instance_label | default(false, true) %}
+        labels:
+          instance: "{{ hostvars[host].prometheus_instance_label }}"
+{% endif %}
 {% endfor %}
 {% endif %}
 
 {% if enable_prometheus_fluentd_integration | bool %}
   - job_name: fluentd
     static_configs:
-      - targets:
 {% for host in groups['fluentd'] %}
+      - targets:
         - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_fluentd_integration_port'] }}'
+{% if hostvars[host].prometheus_instance_label | default(false, true) %}
+        labels:
+          instance: "{{ hostvars[host].prometheus_instance_label }}"
+{% endif %}
 {% endfor %}
 {% endif %}
 
@@ -114,9 +146,13 @@ scrape_configs:
   - job_name: elasticsearch_exporter
     scrape_interval: {{ prometheus_elasticsearch_exporter_interval }}
     static_configs:
-      - targets:
 {% for host in groups["prometheus-elasticsearch-exporter"] %}
+      - targets:
         - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_elasticsearch_exporter_port'] }}'
+{% if hostvars[host].prometheus_instance_label | default(false, true) %}
+        labels:
+          instance: "{{ hostvars[host].prometheus_instance_label }}"
+{% endif %}
 {% endfor %}
 {% endif %}
 
@@ -155,27 +191,39 @@ scrape_configs:
     scrape_interval: {{ prometheus_libvirt_exporter_interval }}
     honor_labels: true
     static_configs:
-      - targets:
 {% for host in groups["prometheus-libvirt-exporter"] %}
+      - targets:
         - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_libvirt_exporter_port'] }}'
+{% if hostvars[host].prometheus_instance_label | default(false, true) %}
+        labels:
+          instance: "{{ hostvars[host].prometheus_instance_label }}"
+{% endif %}
 {% endfor %}
 {% endif %}
 
 {% if enable_prometheus_etcd_integration | bool %}
   - job_name: etcd
     static_configs:
-      - targets:
 {% for host in groups["etcd"] %}
+      - targets:
         - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_etcd_integration_port'] }}'
+{% if hostvars[host].prometheus_instance_label | default(false, true) %}
+        labels:
+          instance: "{{ hostvars[host].prometheus_instance_label }}"
+{% endif %}
 {% endfor %}
 {% endif %}
 
 {% if enable_prometheus_alertmanager | bool %}
   - job_name: alertmanager
     static_configs:
-      - targets:
 {% for host in groups['prometheus-alertmanager'] %}
+      - targets:
         - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_alertmanager_port'] }}'
+{% if hostvars[host].prometheus_instance_label | default(false, true) %}
+        labels:
+          instance: "{{ hostvars[host].prometheus_instance_label }}"
+{% endif %}
 {% endfor %}
 
 alerting:

doc/source/reference/logging-and-monitoring/prometheus-guide.rst

@@ -156,3 +156,40 @@ files:
         - 192.168.1.1
       labels:
           job: ipmi_exporter
+
+Metric Instance labels
+~~~~~~~~~~~~~~~~~~~~~~
+
+Previously, Prometheus metrics used to label instances based on their IP
+addresses. This behaviour can now be changed such that instances can be
+labelled based on their inventory hostname instead. The IP address remains as
+the target address, therefore, even if the hostname is unresolvable, it doesn't
+pose an issue.
+
+The default behavior still labels instances with their IP addresses. However,
+this can be adjusted by changing the ``prometheus_instance_label`` variable.
+This variable accepts the following values:
+
+* ``None``: Instance labels will be IP addresses (default)
+* ``{{ ansible_facts.hostname }}``: Instance labels will be hostnames
+* ``{{ ansible_facts.nodename }}``: Instance labels will FQDNs
+
+To implement this feature, modify the configuration file
+``/etc/kolla/globals.yml`` and update the ``prometheus_instance_label``
+variable accordingly. Remember, changing this variable will cause Prometheus to
+scrape metrics with new names for a short period. This will result in duplicate
+metrics until all metrics are replaced with their new labels.
+
+.. code-block:: yaml
+
+   prometheus_instance_label: "{{ ansible_facts.hostname }}"
+
+This metric labeling feature may become the default setting in future releases.
+Therefore, if you wish to retain the current default (IP address labels), make
+sure to set the ``prometheus_instance_label`` variable to ``None``.
+
+.. note::
+
+   This feature may generate duplicate metrics temporarily while Prometheus
+   updates the metric labels. Please be aware of this while analyzing metrics
+   during the transition period.

releasenotes/notes/friendly-hostnames-1bb1254b2b434a11.yaml

@@ -0,0 +1,10 @@
+---
+features:
+  - |
+    Adds the ability for the instance label on prometheus metrics to be
+    replaced with the inventory hostname as opposed to using the ip address as
+    the metric label. The ip address is still used as the target address which
+    means that there is no issue of the hostname being unresolvable.
+
+    More information on how to use this feature can be found in the
+    reference documentation for logging and monitoring.