From 1bdd8ea98438f0f1c84515fe01ab31e47b3d7d15 Mon Sep 17 00:00:00 2001 From: Piotr Parczewski Date: Tue, 8 Dec 2020 11:39:52 +0100 Subject: [PATCH] Add Prometheus 2.x deployment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It is now possible to deploy either 1.x or 2.x version of Prometheus. The new 2.x version introduces breaking changes in terms of storage format and command line options. Change-Id: I80cc6f1947f3740ef04b29839bfa655b14fae146 Co-Authored-By: Radosław Piliszek --- ansible/roles/prometheus/defaults/main.yml | 11 +++++++-- .../templates/prometheus-server.json.j2 | 4 ++++ .../prometheus-guide.rst | 24 +++++++++++++++++++ etc/kolla/globals.yml | 3 +++ .../add-prometheus-v2-9f10c79986d3faf3.yaml | 17 +++++++++++++ 5 files changed, 57 insertions(+), 2 deletions(-) create mode 100644 releasenotes/notes/add-prometheus-v2-9f10c79986d3faf3.yaml diff --git a/ansible/roles/prometheus/defaults/main.yml b/ansible/roles/prometheus/defaults/main.yml index 431c687a22..507c04da0a 100644 --- a/ansible/roles/prometheus/defaults/main.yml +++ b/ansible/roles/prometheus/defaults/main.yml @@ -113,13 +113,20 @@ prometheus_mysql_exporter_database_user: "{% if use_preconfigured_databases | bo # For a list of modules see the alertmanager config. prometheus_blackbox_exporter_endpoints: [] +#################### +# Compatibility +#################### + +# Whether to keep using Prometheus server v1 (due to no data-preserving migration path to v2) +prometheus_use_v1: no + #################### # Docker #################### prometheus_install_type: "{{ kolla_install_type }}" prometheus_tag: "{{ openstack_tag }}" -prometheus_server_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ prometheus_install_type }}-prometheus-server" +prometheus_server_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ prometheus_install_type }}-{% if not prometheus_use_v1 | bool %}prometheus-v2-server{% else %}prometheus-server{% endif %}" prometheus_server_tag: "{{ prometheus_tag }}" prometheus_server_image_full: "{{ prometheus_server_image }}:{{ prometheus_server_tag }}" @@ -175,7 +182,7 @@ prometheus_server_default_volumes: - "{{ node_config_directory }}/prometheus-server/:{{ container_config_directory }}/:ro" - "/etc/localtime:/etc/localtime:ro" - "{{ '/etc/timezone:/etc/timezone:ro' if ansible_os_family == 'Debian' else '' }}" - - "prometheus:/var/lib/prometheus" + - "{{ 'prometheus_v2' if not prometheus_use_v1 | bool else 'prometheus' }}:/var/lib/prometheus" - "kolla_logs:/var/log/kolla/" prometheus_haproxy_exporter_default_volumes: - "{{ node_config_directory }}/prometheus-haproxy-exporter/:{{ container_config_directory }}/:ro" diff --git a/ansible/roles/prometheus/templates/prometheus-server.json.j2 b/ansible/roles/prometheus/templates/prometheus-server.json.j2 index 8249b76cf8..3391f70e86 100644 --- a/ansible/roles/prometheus/templates/prometheus-server.json.j2 +++ b/ansible/roles/prometheus/templates/prometheus-server.json.j2 @@ -1,5 +1,9 @@ { +{% if not prometheus_use_v1 | bool %} + "command": "/opt/prometheus/prometheus --config.file /etc/prometheus/prometheus.yml --web.listen-address {{ api_interface_address | put_address_in_context('url') }}:{{ prometheus_port }} --web.external-url={{ internal_protocol }}://{{ kolla_internal_fqdn | put_address_in_context('url') }}:{{ prometheus_port }} --storage.tsdb.path /var/lib/prometheus{% if prometheus_cmdline_extras %} {{ prometheus_cmdline_extras }}{% endif %}", +{% else %} "command": "/opt/prometheus/prometheus -config.file /etc/prometheus/prometheus.yml -web.listen-address {{ api_interface_address | put_address_in_context('url') }}:{{ prometheus_port }} -web.external-url={{ internal_protocol }}://{{ kolla_internal_fqdn | put_address_in_context('url') }}:{{ prometheus_port }} -log.format logger:stdout -storage.local.path /var/lib/prometheus{% if prometheus_cmdline_extras %} {{ prometheus_cmdline_extras }}{% endif %}", +{% endif %} "config_files": [ { "source": "{{ container_config_directory }}/prometheus.yml", diff --git a/doc/source/reference/logging-and-monitoring/prometheus-guide.rst b/doc/source/reference/logging-and-monitoring/prometheus-guide.rst index 59afdda5b2..ca5406fd14 100644 --- a/doc/source/reference/logging-and-monitoring/prometheus-guide.rst +++ b/doc/source/reference/logging-and-monitoring/prometheus-guide.rst @@ -20,6 +20,30 @@ and change the following: enable_prometheus: "yes" +This will, by default, deploy Prometheus version 2.x. Since Prometheus 1.x data +is not compatible with Prometheus 2.x and no automatic data migration is +provided, any previous Prometheus 1.x deployment will be replaced and all its +stored metrics will become inacessible (but still available in the old data +volume: ``prometheus``; the new data volume defaults to ``prometheus_v2``). +If you rely on Prometheus only as e.g. a source of alert notifications (in pair +with Alertmanager), it might not be worth migrating old metrics and they could +be discarded. Otherwise, it's either possible to use +`remote storage `_ +or scrape Kolla's Prometheus ``/federate`` endpoint with an external system. +However, if you want to stay on 1.x series, set the following variable: + +.. code-block:: yaml + + prometheus_use_v1: yes + +In order to remove leftover volume containing Prometheus 1.x data, execute: + +.. code-block:: console + + docker volume rm prometheus + +on all hosts wherever Prometheus was previously deployed. + Extending the default command line options ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/etc/kolla/globals.yml b/etc/kolla/globals.yml index 932a1c03bf..e6ee9b3485 100644 --- a/etc/kolla/globals.yml +++ b/etc/kolla/globals.yml @@ -702,6 +702,9 @@ # - host1:port1 # - host2:port2 +# Whether to keep using Prometheus server v1 (due to no data-preserving migration path to v2) +#prometheus_use_v1: no + ######### # Freezer ######### diff --git a/releasenotes/notes/add-prometheus-v2-9f10c79986d3faf3.yaml b/releasenotes/notes/add-prometheus-v2-9f10c79986d3faf3.yaml new file mode 100644 index 0000000000..60a1b7c5c4 --- /dev/null +++ b/releasenotes/notes/add-prometheus-v2-9f10c79986d3faf3.yaml @@ -0,0 +1,17 @@ +--- +features: + - | + Prometheus version 2.x deployment added. This version is enabled by + default and replaces a forward-incompatible version 1.x. A variable + ``prometheus_use_v1`` can be set to ``yes`` to preserve version + 1.x deployment with its data. Otherwise, Prometheus will start with + a new volume, ignoring all previously collected metrics. +upgrade: + - | + New Prometheus version ignores previously stored metrics. If you + want to keep using 1.x with the old data, set ``prometheus_use_v1`` + variable to ``yes``. The old data is not removed, please read the + docs for details. Please also make sure you adapt changes in command + line options if they were ever customized in your environment because + Prometheus 2.x has different syntax (``--option`` with double dashes + instead of ``-option``).