Add Prometheus 2.x deployment

It is now possible to deploy either 1.x or 2.x version of Prometheus.
The new 2.x version introduces breaking changes in terms of storage
format and command line options.

Change-Id: I80cc6f1947f3740ef04b29839bfa655b14fae146
Co-Authored-By: Radosław Piliszek <radoslaw.piliszek@gmail.com>

ansible/roles/prometheus/defaults/main.yml

@@ -113,13 +113,20 @@ prometheus_mysql_exporter_database_user: "{% if use_preconfigured_databases | bo
 # For a list of modules see the alertmanager config.
 prometheus_blackbox_exporter_endpoints: []
 
+####################
+# Compatibility
+####################
+
+# Whether to keep using Prometheus server v1 (due to no data-preserving migration path to v2)
+prometheus_use_v1: no
+
 ####################
 # Docker
 ####################
 prometheus_install_type: "{{ kolla_install_type }}"
 prometheus_tag: "{{ openstack_tag }}"
 
-prometheus_server_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ prometheus_install_type }}-prometheus-server"
+prometheus_server_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ prometheus_install_type }}-{% if not prometheus_use_v1 | bool %}prometheus-v2-server{% else %}prometheus-server{% endif %}"
 prometheus_server_tag: "{{ prometheus_tag }}"
 prometheus_server_image_full: "{{ prometheus_server_image }}:{{ prometheus_server_tag }}"
 
@@ -175,7 +182,7 @@ prometheus_server_default_volumes:
   - "{{ node_config_directory }}/prometheus-server/:{{ container_config_directory }}/:ro"
   - "/etc/localtime:/etc/localtime:ro"
   - "{{ '/etc/timezone:/etc/timezone:ro' if ansible_os_family == 'Debian' else '' }}"
-  - "prometheus:/var/lib/prometheus"
+  - "{{ 'prometheus_v2' if not prometheus_use_v1 | bool else 'prometheus' }}:/var/lib/prometheus"
   - "kolla_logs:/var/log/kolla/"
 prometheus_haproxy_exporter_default_volumes:
   - "{{ node_config_directory }}/prometheus-haproxy-exporter/:{{ container_config_directory }}/:ro"

ansible/roles/prometheus/templates/prometheus-server.json.j2

@@ -1,5 +1,9 @@
 {
+{% if not prometheus_use_v1 | bool %}
+    "command": "/opt/prometheus/prometheus --config.file /etc/prometheus/prometheus.yml --web.listen-address {{ api_interface_address | put_address_in_context('url') }}:{{ prometheus_port }} --web.external-url={{ internal_protocol }}://{{ kolla_internal_fqdn | put_address_in_context('url') }}:{{ prometheus_port }} --storage.tsdb.path /var/lib/prometheus{% if prometheus_cmdline_extras %} {{ prometheus_cmdline_extras }}{% endif %}",
+{% else %}
     "command": "/opt/prometheus/prometheus -config.file /etc/prometheus/prometheus.yml -web.listen-address {{ api_interface_address | put_address_in_context('url') }}:{{ prometheus_port }} -web.external-url={{ internal_protocol }}://{{ kolla_internal_fqdn | put_address_in_context('url') }}:{{ prometheus_port }} -log.format logger:stdout -storage.local.path /var/lib/prometheus{% if prometheus_cmdline_extras %} {{ prometheus_cmdline_extras }}{% endif %}",
+{% endif %}
     "config_files": [
         {
             "source": "{{ container_config_directory }}/prometheus.yml",

doc/source/reference/logging-and-monitoring/prometheus-guide.rst

@@ -20,6 +20,30 @@ and change the following:
 
    enable_prometheus: "yes"
 
+This will, by default, deploy Prometheus version 2.x. Since Prometheus 1.x data
+is not compatible with Prometheus 2.x and no automatic data migration is
+provided, any previous Prometheus 1.x deployment will be replaced and all its
+stored metrics will become inacessible (but still available in the old data
+volume: ``prometheus``; the new data volume defaults to ``prometheus_v2``).
+If you rely on Prometheus only as e.g. a source of alert notifications (in pair
+with Alertmanager), it might not be worth migrating old metrics and they could
+be discarded. Otherwise, it's either possible to use
+`remote storage <https://prometheus.io/docs/prometheus/latest/storage/#remote-storage-integrations>`_
+or scrape Kolla's Prometheus ``/federate`` endpoint with an external system.
+However, if you want to stay on 1.x series, set the following variable:
+
+.. code-block:: yaml
+
+   prometheus_use_v1: yes
+
+In order to remove leftover volume containing Prometheus 1.x data, execute:
+
+.. code-block:: console
+
+   docker volume rm prometheus
+
+on all hosts wherever Prometheus was previously deployed.
+
 Extending the default command line options
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

etc/kolla/globals.yml

@@ -702,6 +702,9 @@
 #  - host1:port1
 #  - host2:port2
 
+# Whether to keep using Prometheus server v1 (due to no data-preserving migration path to v2)
+#prometheus_use_v1: no
+
 #########
 # Freezer
 #########

releasenotes/notes/add-prometheus-v2-9f10c79986d3faf3.yaml

@@ -0,0 +1,17 @@
+---
+features:
+  - |
+    Prometheus version 2.x deployment added. This version is enabled by
+    default and replaces a forward-incompatible version 1.x. A variable
+    ``prometheus_use_v1`` can be set to ``yes`` to preserve version
+    1.x deployment with its data. Otherwise, Prometheus will start with
+    a new volume, ignoring all previously collected metrics.
+upgrade:
+  - |
+    New Prometheus version ignores previously stored metrics. If you
+    want to keep using 1.x with the old data, set ``prometheus_use_v1``
+    variable to ``yes``. The old data is not removed, please read the
+    docs for details. Please also make sure you adapt changes in command
+    line options if they were ever customized in your environment because
+    Prometheus 2.x has different syntax (``--option`` with double dashes
+    instead of ``-option``).