From 37c2ab2aaafa06b40ba1c32a29699fbd8ee17b87 Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Fri, 8 Sep 2023 16:31:35 +0000 Subject: [PATCH] Support exposing prometheus_server externally This avoids the need to use a proxy, or some other means, to connect to Prometheus. This is disabled by default and can be enabled by setting enable_prometheus_server_external to true. Change-Id: Ia0af044ff436c2a204b357750a16ff49fcdfec45 --- ansible/group_vars/all.yml | 7 ++++ .../grafana/templates/prometheus.yaml.j2 | 3 ++ ansible/roles/prometheus/defaults/main.yml | 32 +++++++++++++++++++ ansible/roles/prometheus/tasks/config.yml | 18 +++++++++++ ansible/roles/prometheus/tasks/precheck.yml | 22 +++++++++++++ .../templates/prometheus-server.json.j2 | 8 ++++- .../templates/prometheus-web.yml.j2 | 4 +++ .../prometheus-guide.rst | 20 ++++++++++++ etc/kolla/passwords.yml | 3 ++ kolla_ansible/cmd/genpwd.py | 13 ++++++-- ...heus-on-external-api-78d5fff60f6e75a5.yaml | 16 ++++++++++ requirements.txt | 4 +++ tests/j2lint.py | 4 +++ tests/run-hashi-vault.yml | 6 ++-- tests/test-prometheus-opensearch.sh | 3 ++ zuul.d/base.yaml | 1 + 16 files changed, 159 insertions(+), 5 deletions(-) create mode 100644 ansible/roles/prometheus/templates/prometheus-web.yml.j2 create mode 100644 releasenotes/notes/expose-prometheus-on-external-api-78d5fff60f6e75a5.yaml diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml index 3c58363a48..5ec9f5da89 100644 --- a/ansible/group_vars/all.yml +++ b/ansible/group_vars/all.yml @@ -570,7 +570,10 @@ placement_api_port: "8780" placement_api_listen_port: "{{ placement_api_port }}" placement_api_public_port: "{{ haproxy_single_external_frontend_public_port if haproxy_single_external_frontend | bool else placement_api_port }}" +prometheus_external_fqdn: "{{ kolla_external_fqdn }}" +prometheus_internal_fqdn: "{{ kolla_internal_fqdn }}" prometheus_port: "9091" +prometheus_public_port: "{{ haproxy_single_external_frontend_public_port if haproxy_single_external_frontend | bool else prometheus_port }}" prometheus_node_exporter_port: "9100" prometheus_mysqld_exporter_port: "9104" prometheus_haproxy_exporter_port: "9101" @@ -1281,6 +1284,7 @@ enable_prometheus_etcd_integration: "{{ enable_prometheus | bool and enable_etcd enable_prometheus_msteams: "no" prometheus_alertmanager_user: "admin" +prometheus_grafana_user: "grafana" prometheus_scrape_interval: "60s" prometheus_openstack_exporter_interval: "{{ prometheus_scrape_interval }}" prometheus_openstack_exporter_timeout: "45s" @@ -1292,6 +1296,9 @@ prometheus_openstack_exporter_compute_api_version: "latest" prometheus_libvirt_exporter_interval: "60s" prometheus_msteams_webhook_url: +prometheus_public_endpoint: "{{ prometheus_external_fqdn | kolla_url(public_protocol, prometheus_public_port) }}" +prometheus_internal_endpoint: "{{ prometheus_internal_fqdn | kolla_url(internal_protocol, prometheus_port) }}" + ############ # Vitrage ############ diff --git a/ansible/roles/grafana/templates/prometheus.yaml.j2 b/ansible/roles/grafana/templates/prometheus.yaml.j2 index a0b1a4ae19..d4b3ac9669 100644 --- a/ansible/roles/grafana/templates/prometheus.yaml.j2 +++ b/ansible/roles/grafana/templates/prometheus.yaml.j2 @@ -4,6 +4,9 @@ datasources: - name: Prometheus type: prometheus access: proxy + basicAuth: true + basicAuthPassword: "{{ prometheus_grafana_password }}" + basicAuthUser: "{{ prometheus_grafana_user }}" orgId: 1 url: {{ grafana_prometheus_url }} version: 1 diff --git a/ansible/roles/prometheus/defaults/main.yml b/ansible/roles/prometheus/defaults/main.yml index cc5c8731ef..ae5d7292b3 100644 --- a/ansible/roles/prometheus/defaults/main.yml +++ b/ansible/roles/prometheus/defaults/main.yml @@ -14,6 +14,12 @@ prometheus_services: external: false port: "{{ prometheus_port }}" active_passive: "{{ prometheus_active_passive | bool }}" + prometheus_server_external: + enabled: "{{ enable_prometheus_server_external | bool }}" + mode: "http" + external: true + port: "{{ prometheus_public_port }}" + active_passive: "{{ prometheus_active_passive | bool }}" prometheus-node-exporter: container_name: prometheus_node_exporter group: prometheus-node-exporter @@ -132,6 +138,26 @@ prometheus_services: prometheus_external_labels: # : +#################### +# Server +#################### +enable_prometheus_server_external: false + +#################### +# Basic Auth +#################### +prometheus_basic_auth_users: "{{ prometheus_basic_auth_users_default + prometheus_basic_auth_users_extra }}" + +prometheus_basic_auth_users_default: + - username: admin + password: "{{ prometheus_password }}" + enabled: true + - username: "{{ prometheus_grafana_user }}" + password: "{{ prometheus_grafana_password }}" + enabled: "{{ enable_grafana }}" + +prometheus_basic_auth_users_extra: [] + #################### # Database #################### @@ -315,6 +341,12 @@ prometheus_openstack_exporter_disabled_object: "{{ '--disable-service.object-sto prometheus_openstack_exporter_disabled_lb: "{{ '--disable-service.load-balancer --disable-metric=neutron-loadbalancers --disable-metric=neutron-loadbalancers_not_active' if not enable_octavia | bool else '' }}" prometheus_openstack_exporter_disabled_items: "{{ [prometheus_openstack_exporter_disabled_volume, prometheus_openstack_exporter_disabled_dns, prometheus_openstack_exporter_disabled_object, prometheus_openstack_exporter_disabled_lb | trim] | join(' ') | trim }}" +prometheus_server_command: >- + /opt/prometheus/prometheus --web.config.file=/etc/prometheus/web.yml --config.file /etc/prometheus/prometheus.yml + --web.listen-address {{ api_interface_address | put_address_in_context('url') }}:{{ prometheus_port }} + --web.external-url={{ prometheus_public_endpoint if enable_prometheus_server_external else prometheus_internal_endpoint }} + --storage.tsdb.path /var/lib/prometheus{% if prometheus_cmdline_extras %} {{ prometheus_cmdline_extras }}{% endif %} + prometheus_blackbox_exporter_cmdline_extras: "" prometheus_cadvisor_cmdline_extras: "--docker_only --store_container_labels=false --disable_metrics=percpu,referenced_memory,cpu_topology,resctrl,udp,advtcp,sched,hugetlb,memory_numa,tcp,process" prometheus_elasticsearch_exporter_cmdline_extras: "" diff --git a/ansible/roles/prometheus/tasks/config.yml b/ansible/roles/prometheus/tasks/config.yml index f55f6b5baf..9ad3fa9116 100644 --- a/ansible/roles/prometheus/tasks/config.yml +++ b/ansible/roles/prometheus/tasks/config.yml @@ -97,6 +97,24 @@ notify: - Restart prometheus-server container +- name: Copying over prometheus web config file + become: true + vars: + service: "{{ prometheus_services['prometheus-server'] }}" + template: + src: "{{ item }}" + dest: "{{ node_config_directory }}/prometheus-server/web.yml" + mode: "0600" + when: + - inventory_hostname in groups[service.group] + - service.enabled | bool + with_first_found: + - "{{ node_custom_config }}/prometheus/{{ inventory_hostname }}/web.yml" + - "{{ node_custom_config }}/prometheus/web.yml" + - "{{ role_path }}/templates/prometheus-web.yml.j2" + notify: + - Restart prometheus-server container + - name: Copying over prometheus alertmanager config file become: true vars: diff --git a/ansible/roles/prometheus/tasks/precheck.yml b/ansible/roles/prometheus/tasks/precheck.yml index af0592a3a5..744bc4ad7c 100644 --- a/ansible/roles/prometheus/tasks/precheck.yml +++ b/ansible/roles/prometheus/tasks/precheck.yml @@ -25,6 +25,28 @@ check_mode: false register: container_facts +- name: Check that prometheus_bcrypt_salt is correctly set + assert: + that: + - prometheus_bcrypt_salt is defined + - prometheus_bcrypt_salt is string + - prometheus_bcrypt_salt | length == 22 + +- name: Check that prometheus_password is correctly set + assert: + that: + - prometheus_password is defined + - prometheus_password is string + - prometheus_password | length > 0 + +- name: Check that prometheus_grafana_password is correctly set + assert: + that: + - prometheus_grafana_password is defined + - prometheus_grafana_password is string + - prometheus_grafana_password | length > 0 + when: enable_grafana | bool + - name: Checking free port for Prometheus server wait_for: host: "{{ 'api' | kolla_address }}" diff --git a/ansible/roles/prometheus/templates/prometheus-server.json.j2 b/ansible/roles/prometheus/templates/prometheus-server.json.j2 index d57469ff2c..99ee0c7865 100644 --- a/ansible/roles/prometheus/templates/prometheus-server.json.j2 +++ b/ansible/roles/prometheus/templates/prometheus-server.json.j2 @@ -1,5 +1,5 @@ { - "command": "/opt/prometheus/prometheus --config.file /etc/prometheus/prometheus.yml --web.listen-address {{ api_interface_address | put_address_in_context('url') }}:{{ prometheus_port }} --web.external-url={{ internal_protocol }}://{{ kolla_internal_fqdn | put_address_in_context('url') }}:{{ prometheus_port }} --storage.tsdb.path /var/lib/prometheus{% if prometheus_cmdline_extras %} {{ prometheus_cmdline_extras }}{% endif %}", + "command": "{{ prometheus_server_command }}", "config_files": [ { "source": "{{ container_config_directory }}/prometheus.yml", @@ -7,6 +7,12 @@ "owner": "prometheus", "perm": "0600" }, + { + "source": "{{ container_config_directory }}/web.yml", + "dest": "/etc/prometheus/web.yml", + "owner": "prometheus", + "perm": "0600" + }, { "source": "{{ container_config_directory }}/extras/*", "dest": "/etc/prometheus/extras/", diff --git a/ansible/roles/prometheus/templates/prometheus-web.yml.j2 b/ansible/roles/prometheus/templates/prometheus-web.yml.j2 new file mode 100644 index 0000000000..67e0554285 --- /dev/null +++ b/ansible/roles/prometheus/templates/prometheus-web.yml.j2 @@ -0,0 +1,4 @@ +basic_auth_users: +{% for user in prometheus_basic_auth_users | selectattr('enabled') | list %} + {{ user.username }}: {{ user.password | password_hash('bcrypt', salt=prometheus_bcrypt_salt) }} +{% endfor %} diff --git a/doc/source/reference/logging-and-monitoring/prometheus-guide.rst b/doc/source/reference/logging-and-monitoring/prometheus-guide.rst index 1f814107a5..cc2f632988 100644 --- a/doc/source/reference/logging-and-monitoring/prometheus-guide.rst +++ b/doc/source/reference/logging-and-monitoring/prometheus-guide.rst @@ -34,6 +34,26 @@ In order to remove leftover volume containing Prometheus 1.x data, execute: on all hosts wherever Prometheus was previously deployed. +Basic Auth +~~~~~~~~~~ + +Prometheus is protected with basic HTTP authentication. Kolla-ansible will +create the following users: ``admin`` and ``grafana`` (if grafana is +enabled). The grafana username can be overidden using the variable +``prometheus_grafana_user``. The passwords are defined by the +``prometheus_password`` and ``prometheus_grafana_password`` variables in +``passwords.yml``. The list of basic auth users can be extended using the +``prometheus_basic_auth_users_extra`` variable: + +.. code-block:: yaml + + prometheus_basic_auth_users_extra: + - username: user + password: hello + enabled: true + +or completely overriden with the ``prometheus_basic_auth_users`` variable. + Extending the default command line options ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/etc/kolla/passwords.yml b/etc/kolla/passwords.yml index 3bca88f9fc..f59ab1cea3 100644 --- a/etc/kolla/passwords.yml +++ b/etc/kolla/passwords.yml @@ -247,6 +247,9 @@ redis_master_password: #################### prometheus_mysql_exporter_database_password: prometheus_alertmanager_password: +prometheus_password: +prometheus_grafana_password: +prometheus_bcrypt_salt: ############################### # OpenStack identity federation diff --git a/kolla_ansible/cmd/genpwd.py b/kolla_ansible/cmd/genpwd.py index 55dab376e5..89b7bbd0ab 100755 --- a/kolla_ansible/cmd/genpwd.py +++ b/kolla_ansible/cmd/genpwd.py @@ -20,6 +20,7 @@ import stat import string import sys +from ansible.utils.encrypt import random_salt from cryptography import fernet from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives.asymmetric import rsa @@ -56,7 +57,7 @@ def generate_RSA(bits=4096): def genpwd(passwords_file, length, uuid_keys, ssh_keys, blank_keys, - fernet_keys, hmac_md5_keys): + fernet_keys, hmac_md5_keys, bcrypt_keys): try: with open(passwords_file, 'r') as f: passwords = yaml.safe_load(f.read()) @@ -98,6 +99,11 @@ def genpwd(passwords_file, length, uuid_keys, ssh_keys, blank_keys, .hexdigest()) elif k in fernet_keys: passwords[k] = fernet.Fernet.generate_key().decode() + elif k in bcrypt_keys: + # NOTE(wszusmki) To be compatible with the ansible + # password_hash filter, we use the utility function from the + # ansible library. + passwords[k] = random_salt(22) else: passwords[k] = ''.join([ random.SystemRandom().choice( @@ -151,11 +157,14 @@ def main(): # Fernet keys fernet_keys = ['barbican_crypto_key'] + # bcrypt salts + bcrypt_keys = ['prometheus_bcrypt_salt'] + # length of password length = 40 genpwd(passwords_file, length, uuid_keys, ssh_keys, blank_keys, - fernet_keys, hmac_md5_keys) + fernet_keys, hmac_md5_keys, bcrypt_keys) if __name__ == '__main__': diff --git a/releasenotes/notes/expose-prometheus-on-external-api-78d5fff60f6e75a5.yaml b/releasenotes/notes/expose-prometheus-on-external-api-78d5fff60f6e75a5.yaml new file mode 100644 index 0000000000..d580e383a8 --- /dev/null +++ b/releasenotes/notes/expose-prometheus-on-external-api-78d5fff60f6e75a5.yaml @@ -0,0 +1,16 @@ +--- +features: + - | + Adds support for exposing Prometheus server on the external interface. + This is disabled by default and can be enabled by setting + ``enable_prometheus_server_external`` to ``true``. Basic auth is used to + protect the endpoint. + - | + Adds ``prometheus_external_fqdn`` and ``prometheus_internal_fqdn`` to + customise prometheus FQDNs. +upgrade: + - | + Prometheus now uses basic auth. The password is under the key + ``prometheus_password`` in the Kolla passwords file. The username is + ``admin``. The default set of users can be changed using the variable: + ``prometheus_basic_auth_users``. diff --git a/requirements.txt b/requirements.txt index 59147c1bd1..dcbbd1f10c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,3 +18,7 @@ jmespath>=0.9.3 # MIT # Hashicorp Vault hvac>=0.10.1 + +# Password hashing +bcrypt>=3.0.0 # Apache-2.0 +passlib[bcrypt]>=1.0.0 # BSD diff --git a/tests/j2lint.py b/tests/j2lint.py index 65189c38f9..d9cc6e8bf9 100755 --- a/tests/j2lint.py +++ b/tests/j2lint.py @@ -23,6 +23,7 @@ Simple j2 linter, useful for checking jinja2 template syntax Adapted for OpenStack Kolla/Kolla-Ansible purposes """ +from ansible.plugins.filter.core import get_encrypted_password from ansible.plugins.filter.core import to_json from functools import reduce from jinja2 import BaseLoader @@ -51,6 +52,9 @@ def check(template, out, err, env=Environment(loader=AbsolutePathLoader(), env.filters['bool'] = bool env.filters['hash'] = hash env.filters['to_json'] = to_json + # NOTE(wszumski): password_hash is mapped to the function: + # get_encrypted_password in ansible.filters.core. + env.filters['password_hash'] = get_encrypted_password env.filters['kolla_address'] = kolla_address env.filters['put_address_in_context'] = put_address_in_context env.get_template(template) diff --git a/tests/run-hashi-vault.yml b/tests/run-hashi-vault.yml index 2fa79663bc..cbeb4aa157 100644 --- a/tests/run-hashi-vault.yml +++ b/tests/run-hashi-vault.yml @@ -36,10 +36,12 @@ - name: install kolla-ansible and dependencies pip: - name: - - "{{ kolla_ansible_src_dir }}" executable: "pip3" extra_args: "-c {{ upper_constraints_file }} --user" + name: + - "{{ kolla_ansible_src_dir }}" + - "ansible-core{{ ansible_core_version_constraint }}" + - "ansible{{ ansible_version_constraint }}" - name: copy passwords.yml file copy: diff --git a/tests/test-prometheus-opensearch.sh b/tests/test-prometheus-opensearch.sh index 9ce8a76636..c1f8272c16 100755 --- a/tests/test-prometheus-opensearch.sh +++ b/tests/test-prometheus-opensearch.sh @@ -79,11 +79,14 @@ function check_prometheus { # Query prometheus graph, and check that the returned page looks like a # prometheus page. PROMETHEUS_URL=${OS_AUTH_URL%:*}:9091/graph + prometheus_password=$(awk '$1 == "prometheus_password:" { print $2 }' /etc/kolla/passwords.yml) output_path=$1 args=( --include --location --fail + --user + admin:$prometheus_password ) if [[ "$TLS_ENABLED" = "True" ]]; then args+=(--cacert $OS_CACERT) diff --git a/zuul.d/base.yaml b/zuul.d/base.yaml index fb84aa36af..04e84ba122 100644 --- a/zuul.d/base.yaml +++ b/zuul.d/base.yaml @@ -272,6 +272,7 @@ - job: name: kolla-ansible-hashi-vault-base + parent: kolla-ansible-variables run: tests/run-hashi-vault.yml required-projects: - openstack/kolla-ansible