diff --git a/ansible/roles/prometheus/defaults/main.yml b/ansible/roles/prometheus/defaults/main.yml index 431c687a22..507c04da0a 100644 --- a/ansible/roles/prometheus/defaults/main.yml +++ b/ansible/roles/prometheus/defaults/main.yml @@ -113,13 +113,20 @@ prometheus_mysql_exporter_database_user: "{% if use_preconfigured_databases | bo # For a list of modules see the alertmanager config. prometheus_blackbox_exporter_endpoints: [] +#################### +# Compatibility +#################### + +# Whether to keep using Prometheus server v1 (due to no data-preserving migration path to v2) +prometheus_use_v1: no + #################### # Docker #################### prometheus_install_type: "{{ kolla_install_type }}" prometheus_tag: "{{ openstack_tag }}" -prometheus_server_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ prometheus_install_type }}-prometheus-server" +prometheus_server_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ prometheus_install_type }}-{% if not prometheus_use_v1 | bool %}prometheus-v2-server{% else %}prometheus-server{% endif %}" prometheus_server_tag: "{{ prometheus_tag }}" prometheus_server_image_full: "{{ prometheus_server_image }}:{{ prometheus_server_tag }}" @@ -175,7 +182,7 @@ prometheus_server_default_volumes: - "{{ node_config_directory }}/prometheus-server/:{{ container_config_directory }}/:ro" - "/etc/localtime:/etc/localtime:ro" - "{{ '/etc/timezone:/etc/timezone:ro' if ansible_os_family == 'Debian' else '' }}" - - "prometheus:/var/lib/prometheus" + - "{{ 'prometheus_v2' if not prometheus_use_v1 | bool else 'prometheus' }}:/var/lib/prometheus" - "kolla_logs:/var/log/kolla/" prometheus_haproxy_exporter_default_volumes: - "{{ node_config_directory }}/prometheus-haproxy-exporter/:{{ container_config_directory }}/:ro" diff --git a/ansible/roles/prometheus/templates/prometheus-server.json.j2 b/ansible/roles/prometheus/templates/prometheus-server.json.j2 index 8249b76cf8..3391f70e86 100644 --- a/ansible/roles/prometheus/templates/prometheus-server.json.j2 +++ b/ansible/roles/prometheus/templates/prometheus-server.json.j2 @@ -1,5 +1,9 @@ { +{% if not prometheus_use_v1 | bool %} + "command": "/opt/prometheus/prometheus --config.file /etc/prometheus/prometheus.yml --web.listen-address {{ api_interface_address | put_address_in_context('url') }}:{{ prometheus_port }} --web.external-url={{ internal_protocol }}://{{ kolla_internal_fqdn | put_address_in_context('url') }}:{{ prometheus_port }} --storage.tsdb.path /var/lib/prometheus{% if prometheus_cmdline_extras %} {{ prometheus_cmdline_extras }}{% endif %}", +{% else %} "command": "/opt/prometheus/prometheus -config.file /etc/prometheus/prometheus.yml -web.listen-address {{ api_interface_address | put_address_in_context('url') }}:{{ prometheus_port }} -web.external-url={{ internal_protocol }}://{{ kolla_internal_fqdn | put_address_in_context('url') }}:{{ prometheus_port }} -log.format logger:stdout -storage.local.path /var/lib/prometheus{% if prometheus_cmdline_extras %} {{ prometheus_cmdline_extras }}{% endif %}", +{% endif %} "config_files": [ { "source": "{{ container_config_directory }}/prometheus.yml", diff --git a/doc/source/reference/logging-and-monitoring/prometheus-guide.rst b/doc/source/reference/logging-and-monitoring/prometheus-guide.rst index 59afdda5b2..ca5406fd14 100644 --- a/doc/source/reference/logging-and-monitoring/prometheus-guide.rst +++ b/doc/source/reference/logging-and-monitoring/prometheus-guide.rst @@ -20,6 +20,30 @@ and change the following: enable_prometheus: "yes" +This will, by default, deploy Prometheus version 2.x. Since Prometheus 1.x data +is not compatible with Prometheus 2.x and no automatic data migration is +provided, any previous Prometheus 1.x deployment will be replaced and all its +stored metrics will become inacessible (but still available in the old data +volume: ``prometheus``; the new data volume defaults to ``prometheus_v2``). +If you rely on Prometheus only as e.g. a source of alert notifications (in pair +with Alertmanager), it might not be worth migrating old metrics and they could +be discarded. Otherwise, it's either possible to use +`remote storage `_ +or scrape Kolla's Prometheus ``/federate`` endpoint with an external system. +However, if you want to stay on 1.x series, set the following variable: + +.. code-block:: yaml + + prometheus_use_v1: yes + +In order to remove leftover volume containing Prometheus 1.x data, execute: + +.. code-block:: console + + docker volume rm prometheus + +on all hosts wherever Prometheus was previously deployed. + Extending the default command line options ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/etc/kolla/globals.yml b/etc/kolla/globals.yml index 932a1c03bf..e6ee9b3485 100644 --- a/etc/kolla/globals.yml +++ b/etc/kolla/globals.yml @@ -702,6 +702,9 @@ # - host1:port1 # - host2:port2 +# Whether to keep using Prometheus server v1 (due to no data-preserving migration path to v2) +#prometheus_use_v1: no + ######### # Freezer ######### diff --git a/releasenotes/notes/add-prometheus-v2-9f10c79986d3faf3.yaml b/releasenotes/notes/add-prometheus-v2-9f10c79986d3faf3.yaml new file mode 100644 index 0000000000..60a1b7c5c4 --- /dev/null +++ b/releasenotes/notes/add-prometheus-v2-9f10c79986d3faf3.yaml @@ -0,0 +1,17 @@ +--- +features: + - | + Prometheus version 2.x deployment added. This version is enabled by + default and replaces a forward-incompatible version 1.x. A variable + ``prometheus_use_v1`` can be set to ``yes`` to preserve version + 1.x deployment with its data. Otherwise, Prometheus will start with + a new volume, ignoring all previously collected metrics. +upgrade: + - | + New Prometheus version ignores previously stored metrics. If you + want to keep using 1.x with the old data, set ``prometheus_use_v1`` + variable to ``yes``. The old data is not removed, please read the + docs for details. Please also make sure you adapt changes in command + line options if they were ever customized in your environment because + Prometheus 2.x has different syntax (``--option`` with double dashes + instead of ``-option``).