diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml index 6959fd4056..8f67a23a62 100644 --- a/ansible/group_vars/all.yml +++ b/ansible/group_vars/all.yml @@ -397,6 +397,7 @@ ironic_inspector_port: "5050" ironic_inspector_public_port: "{{ haproxy_single_external_frontend_public_port if haproxy_single_external_frontend | bool else ironic_inspector_port }}" ironic_inspector_listen_port: "{{ ironic_inspector_port }}" ironic_http_port: "8089" +ironic_prometheus_exporter_port: "9608" iscsi_port: "3260" @@ -804,6 +805,7 @@ enable_ironic: "no" enable_ironic_neutron_agent: "{{ enable_neutron | bool and enable_ironic | bool }}" # TODO(yoctozepto): Remove the deprecated enable_ironic_pxe_uefi in Zed. enable_ironic_pxe_uefi: "no" +enable_ironic_prometheus_exporter: "{{ enable_ironic | bool and enable_prometheus | bool }}" enable_iscsid: "{{ enable_cinder | bool and enable_cinder_backend_iscsi | bool }}" enable_kuryr: "no" enable_magnum: "no" diff --git a/ansible/roles/ironic/defaults/main.yml b/ansible/roles/ironic/defaults/main.yml index 7fa8651aea..988439bbe3 100644 --- a/ansible/roles/ironic/defaults/main.yml +++ b/ansible/roles/ironic/defaults/main.yml @@ -85,6 +85,13 @@ ironic_services: image: "{{ ironic_dnsmasq_image_full }}" volumes: "{{ ironic_dnsmasq_default_volumes + ironic_dnsmasq_extra_volumes }}" dimensions: "{{ ironic_dnsmasq_dimensions }}" + ironic-prometheus-exporter: + container_name: ironic_prometheus_exporter + group: ironic-conductor + enabled: "{{ enable_ironic_prometheus_exporter }}" + image: "{{ ironic_prometheus_exporter_image_full }}" + volumes: "{{ ironic_prometheus_exporter_default_volumes + ironic_prometheus_exporter_extra_volumes }}" + dimensions: "{{ ironic_prometheus_exporter_dimensions }}" #################### # Config Validate @@ -146,12 +153,17 @@ ironic_dnsmasq_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ ironic_dnsmasq_tag: "{{ ironic_tag }}" ironic_dnsmasq_image_full: "{{ ironic_dnsmasq_image }}:{{ ironic_dnsmasq_tag }}" +ironic_prometheus_exporter_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/ironic-prometheus-exporter" +ironic_prometheus_exporter_tag: "{{ ironic_tag }}" +ironic_prometheus_exporter_image_full: "{{ ironic_prometheus_exporter_image }}:{{ ironic_prometheus_exporter_tag }}" + ironic_api_dimensions: "{{ default_container_dimensions }}" ironic_conductor_dimensions: "{{ default_container_dimensions }}" ironic_tftp_dimensions: "{{ default_container_dimensions }}" ironic_http_dimensions: "{{ default_container_dimensions }}" ironic_inspector_dimensions: "{{ default_container_dimensions }}" ironic_dnsmasq_dimensions: "{{ default_container_dimensions }}" +ironic_prometheus_exporter_dimensions: "{{ default_container_dimensions }}" ironic_api_enable_healthchecks: "{{ enable_container_healthchecks }}" ironic_api_healthcheck_interval: "{{ default_container_healthcheck_interval }}" @@ -222,6 +234,7 @@ ironic_conductor_default_volumes: - "kolla_logs:/var/log/kolla" - "ironic:/var/lib/ironic" - "{{ kolla_dev_repos_directory ~ '/ironic/ironic:/var/lib/kolla/venv/lib/python' ~ distro_python_version ~ '/site-packages/ironic' if ironic_dev_mode | bool else '' }}" + - "ironic_prometheus_exporter_data:/var/lib/ironic/metrics" ironic_tftp_default_volumes: - "{{ node_config_directory }}/ironic-tftp/:{{ container_config_directory }}/:ro" - "/etc/localtime:/etc/localtime:ro" @@ -247,6 +260,12 @@ ironic_dnsmasq_default_volumes: - "{{ '/etc/timezone:/etc/timezone:ro' if ansible_facts.os_family == 'Debian' else '' }}" - "kolla_logs:/var/log/kolla" - "ironic_inspector_dhcp_hosts:/etc/dnsmasq/dhcp-hostsdir:ro" +ironic_prometheus_exporter_default_volumes: + - "{{ node_config_directory }}/ironic-prometheus-exporter/:{{ container_config_directory }}/:ro" + - "/etc/localtime:/etc/localtime:ro" + - "{{ '/etc/timezone:/etc/timezone:ro' if ansible_facts.os_family == 'Debian' else '' }}" + - "kolla_logs:/var/log/kolla" + - "ironic_prometheus_exporter_data:/var/lib/ironic/metrics" ironic_extra_volumes: "{{ default_extra_volumes }}" ironic_api_extra_volumes: "{{ ironic_extra_volumes }}" @@ -255,6 +274,7 @@ ironic_tftp_extra_volumes: "{{ ironic_extra_volumes }}" ironic_http_extra_volumes: "{{ ironic_extra_volumes }}" ironic_inspector_extra_volumes: "{{ ironic_extra_volumes }}" ironic_dnsmasq_extra_volumes: "{{ ironic_extra_volumes }}" +ironic_prometheus_exporter_extra_volumes: "{{ ironic_extra_volumes }}" #################### # OpenStack @@ -288,6 +308,9 @@ ironic_enable_rolling_upgrade: "yes" ironic_upgrade_skip_wait_check: false ironic_inspector_kernel_cmdline_extras: [] ironic_inspector_pxe_filter: "{% if enable_neutron | bool %}dnsmasq{% else %}noop{% endif %}" +ironic_prometheus_exporter_data_dir: "/var/lib/ironic-prometheus-exporter/data" +ironic_prometheus_exporter_sensor_data_interval: 30 +ironic_prometheus_exporter_sensor_data_undeployed_nodes: "true" #################### diff --git a/ansible/roles/ironic/handlers/main.yml b/ansible/roles/ironic/handlers/main.yml index 95cf4e7648..8155cd4bf6 100644 --- a/ansible/roles/ironic/handlers/main.yml +++ b/ansible/roles/ironic/handlers/main.yml @@ -96,3 +96,18 @@ cap_add: "{{ service.cap_add }}" when: - kolla_action != "config" + +- name: Restart ironic-prometheus-exporter container + vars: + service_name: "ironic-prometheus-exporter" + service: "{{ ironic_services[service_name] }}" + become: true + kolla_docker: + action: "recreate_or_restart_container" + common_options: "{{ docker_common_options }}" + name: "{{ service.container_name }}" + image: "{{ service.image }}" + volumes: "{{ service.volumes }}" + dimensions: "{{ service.dimensions }}" + when: + - kolla_action != "config" diff --git a/ansible/roles/ironic/tasks/config.yml b/ansible/roles/ironic/tasks/config.yml index c5975ef1aa..464507ef0d 100644 --- a/ansible/roles/ironic/tasks/config.yml +++ b/ansible/roles/ironic/tasks/config.yml @@ -81,7 +81,7 @@ mode: "0660" become: true when: - - item.key in [ "ironic-api", "ironic-conductor" ] + - item.key in [ "ironic-api", "ironic-conductor", "ironic-prometheus-exporter" ] - inventory_hostname in groups[item.value.group] - item.value.enabled | bool with_dict: "{{ ironic_services }}" @@ -243,6 +243,24 @@ notify: - Restart ironic-http container +- name: Copying over ironic-prometheus-exporter-wsgi.conf + vars: + service: "{{ ironic_services['ironic-prometheus-exporter'] }}" + template: + src: "{{ item }}" + dest: "{{ node_config_directory }}/ironic-prometheus-exporter/ironic-prometheus-exporter-wsgi.conf" + mode: "0660" + become: true + with_first_found: + - "{{ node_config_directory }}/ironic/{{ inventory_hostname }}/ironic-prometheus-exporter-wsgi.conf" + - "{{ node_config_directory }}/ironic/ironic-prometheus-exporter-wsgi.conf" + - "ironic-prometheus-exporter-wsgi.conf.j2" + when: + - inventory_hostname in groups[service.group] + - service.enabled | bool + notify: + - "Restart ironic-prometheus-exporter container" + - name: Copying over existing Ironic policy file vars: services_require_policy_json: diff --git a/ansible/roles/ironic/tasks/precheck.yml b/ansible/roles/ironic/tasks/precheck.yml index 6de0731e0a..7f7af51c4f 100644 --- a/ansible/roles/ironic/tasks/precheck.yml +++ b/ansible/roles/ironic/tasks/precheck.yml @@ -49,6 +49,18 @@ - container_facts['ironic_http'] is not defined - inventory_hostname in groups['ironic-http'] +- name: Checking free port for Ironic Prometheus Exporter + wait_for: + host: "{{ api_interface_address }}" + port: "{{ ironic_prometheus_exporter_port }}" + connect_timeout: 1 + timeout: 1 + state: stopped + when: + - enable_ironic_prometheus_exporter | bool + - container_facts['ironic_prometheus_exporter'] is not defined + - inventory_hostname in groups['ironic-conductor'] + - name: Checking ironic-agent files exist for Ironic Inspector stat: path: "{{ ironic_agent_files_directory }}/ironic/{{ item }}" diff --git a/ansible/roles/ironic/templates/ironic-conductor.json.j2 b/ansible/roles/ironic/templates/ironic-conductor.json.j2 index f5a0477992..1cbc8eed5b 100644 --- a/ansible/roles/ironic/templates/ironic-conductor.json.j2 +++ b/ansible/roles/ironic/templates/ironic-conductor.json.j2 @@ -24,6 +24,11 @@ "path": "/var/lib/ironic", "owner": "ironic:ironic", "recurse": true - } + }{% if enable_ironic_prometheus_exporter | bool %}, + { + "path": "/var/lib/ironic/metrics", + "owner": "ironic:ironic", + "recurse": true + }{% endif %} ] } diff --git a/ansible/roles/ironic/templates/ironic-prometheus-exporter-wsgi.conf.j2 b/ansible/roles/ironic/templates/ironic-prometheus-exporter-wsgi.conf.j2 new file mode 100644 index 0000000000..ace7fd9a1d --- /dev/null +++ b/ansible/roles/ironic/templates/ironic-prometheus-exporter-wsgi.conf.j2 @@ -0,0 +1,38 @@ +{% set ironic_log_dir = '/var/log/kolla/ironic' %} +{% set python_path = '/var/lib/kolla/venv/lib/python' + distro_python_version + '/site-packages' %} +Listen {{ api_interface_address | put_address_in_context('url') }}:{{ ironic_prometheus_exporter_port }} + +ServerSignature Off +ServerTokens Prod +TraceEnable off + + + + AllowOverride None + Options None + Require all granted + + + +ErrorLog "{{ ironic_log_dir }}/apache-error.log" + + CustomLog "{{ ironic_log_dir }}/apache-access.log" common + + +{% if ironic_logging_debug | bool %} +LogLevel info +{% endif %} + + + ErrorLog "{{ ironic_log_dir }}/ironic-prometheus-exporter-wsgi-error.log" + LogFormat "%{X-Forwarded-For}i %l %u %t \"%r\" %>s %b %D \"%{Referer}i\" \"%{User-Agent}i\"" logformat + CustomLog "{{ ironic_log_dir }}/ironic-prometheus-exporter-wsgi-access.log" logformat + + WSGIDaemonProcess ironic-prometheus-exporter processes={{ openstack_service_workers }} threads=1 user=ironic display-name=%{GROUP} python-path={{ python_path }} + WSGIProcessGroup ironic-prometheus-exporter + WSGIScriptAlias / {{ python_path }}/ironic_prometheus_exporter/app/wsgi.py + WSGIApplicationGroup %{GLOBAL} + + Require all granted + + diff --git a/ansible/roles/ironic/templates/ironic-prometheus-exporter.json.j2 b/ansible/roles/ironic/templates/ironic-prometheus-exporter.json.j2 new file mode 100644 index 0000000000..c3b4bf2ee5 --- /dev/null +++ b/ansible/roles/ironic/templates/ironic-prometheus-exporter.json.j2 @@ -0,0 +1,31 @@ +{% set ironic_prometheus_exporter_cmd = 'apache2' if kolla_base_distro in ['ubuntu', 'debian'] else 'httpd' %} +{% set ironic_prometheus_exporter_dir = 'apache2/conf-enabled' if kolla_base_distro in ['ubuntu', 'debian'] else 'httpd/conf.d' %} +{ + "command": "/usr/sbin/{{ ironic_prometheus_exporter_cmd }} -DFOREGROUND", + "config_files": [ + { + "source": "{{ container_config_directory }}/ironic-prometheus-exporter-wsgi.conf", + "dest": "/etc/{{ ironic_prometheus_exporter_dir }}/ironic-prometheus-exporter-wsgi.conf", + "owner": "ironic", + "perm": "0600" + }, + { + "source": "{{ container_config_directory }}/ironic.conf", + "dest": "/etc/ironic/ironic.conf", + "owner": "ironic", + "perm": "0600" + } + ], + "permissions": [ + { + "path": "/var/log/kolla/ironic", + "owner": "ironic:ironic", + "recurse": true + }, + { + "path": "/var/lib/ironic/metrics", + "owner": "ironic:ironic", + "recurse": true + } + ] +} diff --git a/ansible/roles/ironic/templates/ironic.conf.j2 b/ansible/roles/ironic/templates/ironic.conf.j2 index 3c0a51b375..a48152d3d8 100644 --- a/ansible/roles/ironic/templates/ironic.conf.j2 +++ b/ansible/roles/ironic/templates/ironic.conf.j2 @@ -20,12 +20,20 @@ notification_level = info [oslo_messaging_notifications] transport_url = {{ notify_transport_url }} +{% if ironic_enabled_notification_topics or enable_ironic_prometheus_exporter | bool %} {% if ironic_enabled_notification_topics %} driver = messagingv2 topics = {{ ironic_enabled_notification_topics | map(attribute='name') | join(',') }} +{% endif %} +{% if enable_ironic_prometheus_exporter | bool %} +driver = prometheus_exporter +{% endif %} {% else %} driver = noop {% endif %} +{% if enable_ironic_prometheus_exporter | bool %} +location = /var/lib/ironic/metrics +{% endif %} [oslo_messaging_rabbit] heartbeat_in_pthread = {{ service_name == 'ironic-api' }} @@ -45,6 +53,11 @@ policy_file = {{ ironic_policy_file }} {% if service_name == 'ironic-conductor' %} [conductor] automated_clean=false +{% if enable_ironic_prometheus_exporter | bool %} +send_sensor_data = true +send_sensor_data_for_undeployed_nodes = {{ ironic_prometheus_exporter_sensor_data_undeployed_nodes }} +send_sensor_data_interval = {{ ironic_prometheus_exporter_sensor_data_interval }} +{% endif %} {% endif %} [database] diff --git a/ansible/roles/prometheus/templates/prometheus.yml.j2 b/ansible/roles/prometheus/templates/prometheus.yml.j2 index 43d76bada7..a67d6e77ff 100644 --- a/ansible/roles/prometheus/templates/prometheus.yml.j2 +++ b/ansible/roles/prometheus/templates/prometheus.yml.j2 @@ -214,6 +214,14 @@ scrape_configs: {% endfor %} {% endif %} +{% if enable_ironic_prometheus_exporter | bool %} + - job_name: ironic_prometheus_exporter + static_configs: +{% for host in groups['ironic-conductor'] %} + - targets: ["{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['ironic_prometheus_exporter_port'] }}"] +{% endfor %} +{% endif %} + {% if enable_prometheus_alertmanager | bool %} - job_name: alertmanager static_configs: diff --git a/etc/kolla/globals.yml b/etc/kolla/globals.yml index c87c73af58..71f1ea0a35 100644 --- a/etc/kolla/globals.yml +++ b/etc/kolla/globals.yml @@ -362,6 +362,7 @@ workaround_ansible_issue_8743: yes #enable_influxdb: "{{ enable_cloudkitty | bool and cloudkitty_storage_backend == 'influxdb' }}" #enable_ironic: "no" #enable_ironic_neutron_agent: "{{ enable_neutron | bool and enable_ironic | bool }}" +#enable_ironic_prometheus_exporter: "{{ enable_ironic | bool and enable_prometheus | bool }}" #enable_iscsid: "{{ enable_cinder | bool and enable_cinder_backend_iscsi | bool }}" #enable_kuryr: "no" #enable_magnum: "no" diff --git a/releasenotes/notes/add-ironic-prometheus-exporter-218a9985905602fd.yaml b/releasenotes/notes/add-ironic-prometheus-exporter-218a9985905602fd.yaml new file mode 100644 index 0000000000..9e8d0761a6 --- /dev/null +++ b/releasenotes/notes/add-ironic-prometheus-exporter-218a9985905602fd.yaml @@ -0,0 +1,8 @@ +--- +features: + - | + Adds support for deploying the ironic-prometheus-exporter, 'a Tool to + expose hardware sensor data in the Prometheus format through an HTTP + endpoint'. + See https://opendev.org/openstack/ironic-prometheus-exporter for more details + about the exporter.