Add ironic-prometheus-exporter

This commit adds the ironic-prometheus-exporter, following the
conventions used by the previously integrated exporters. '[The] Ironic
Prometheus Exporter is a Tool to expose hardware sensor data in the
Prometheus format through an HTTP endpoint.'[0]

Prometheus has been enabled in CI jobs to ensure test coverage.

[0] https://opendev.org/openstack/ironic-prometheus-exporter

Depends-On: https://review.opendev.org/c/openstack/kolla/+/874415

Change-Id: I6d421effd833d2e0524dd0b81736445c9a730ea9

ansible/group_vars/all.yml

@@ -397,6 +397,7 @@ ironic_inspector_port: "5050"
 ironic_inspector_public_port: "{{ haproxy_single_external_frontend_public_port if haproxy_single_external_frontend | bool else ironic_inspector_port }}"
 ironic_inspector_listen_port: "{{ ironic_inspector_port }}"
 ironic_http_port: "8089"
+ironic_prometheus_exporter_port: "9608"
 
 iscsi_port: "3260"
 
@@ -804,6 +805,7 @@ enable_ironic: "no"
 enable_ironic_neutron_agent: "{{ enable_neutron | bool and enable_ironic | bool }}"
 # TODO(yoctozepto): Remove the deprecated enable_ironic_pxe_uefi in Zed.
 enable_ironic_pxe_uefi: "no"
+enable_ironic_prometheus_exporter: "{{ enable_ironic | bool and enable_prometheus | bool }}"
 enable_iscsid: "{{ enable_cinder | bool and enable_cinder_backend_iscsi | bool }}"
 enable_kuryr: "no"
 enable_magnum: "no"

ansible/roles/ironic/defaults/main.yml

@@ -85,6 +85,13 @@ ironic_services:
     image: "{{ ironic_dnsmasq_image_full }}"
     volumes: "{{ ironic_dnsmasq_default_volumes + ironic_dnsmasq_extra_volumes }}"
     dimensions: "{{ ironic_dnsmasq_dimensions }}"
+  ironic-prometheus-exporter:
+    container_name: ironic_prometheus_exporter
+    group: ironic-conductor
+    enabled: "{{ enable_ironic_prometheus_exporter }}"
+    image: "{{ ironic_prometheus_exporter_image_full }}"
+    volumes: "{{ ironic_prometheus_exporter_default_volumes + ironic_prometheus_exporter_extra_volumes }}"
+    dimensions: "{{ ironic_prometheus_exporter_dimensions }}"
 
 ####################
 # Config Validate
@@ -146,12 +153,17 @@ ironic_dnsmasq_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{
 ironic_dnsmasq_tag: "{{ ironic_tag }}"
 ironic_dnsmasq_image_full: "{{ ironic_dnsmasq_image }}:{{ ironic_dnsmasq_tag }}"
 
+ironic_prometheus_exporter_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/ironic-prometheus-exporter"
+ironic_prometheus_exporter_tag: "{{ ironic_tag }}"
+ironic_prometheus_exporter_image_full: "{{ ironic_prometheus_exporter_image }}:{{ ironic_prometheus_exporter_tag }}"
+
 ironic_api_dimensions: "{{ default_container_dimensions }}"
 ironic_conductor_dimensions: "{{ default_container_dimensions }}"
 ironic_tftp_dimensions: "{{ default_container_dimensions }}"
 ironic_http_dimensions: "{{ default_container_dimensions }}"
 ironic_inspector_dimensions: "{{ default_container_dimensions }}"
 ironic_dnsmasq_dimensions: "{{ default_container_dimensions }}"
+ironic_prometheus_exporter_dimensions: "{{ default_container_dimensions }}"
 
 ironic_api_enable_healthchecks: "{{ enable_container_healthchecks }}"
 ironic_api_healthcheck_interval: "{{ default_container_healthcheck_interval }}"
@@ -222,6 +234,7 @@ ironic_conductor_default_volumes:
   - "kolla_logs:/var/log/kolla"
   - "ironic:/var/lib/ironic"
   - "{{ kolla_dev_repos_directory ~ '/ironic/ironic:/var/lib/kolla/venv/lib/python' ~ distro_python_version ~ '/site-packages/ironic' if ironic_dev_mode | bool else '' }}"
+  - "ironic_prometheus_exporter_data:/var/lib/ironic/metrics"
 ironic_tftp_default_volumes:
   - "{{ node_config_directory }}/ironic-tftp/:{{ container_config_directory }}/:ro"
   - "/etc/localtime:/etc/localtime:ro"
@@ -247,6 +260,12 @@ ironic_dnsmasq_default_volumes:
   - "{{ '/etc/timezone:/etc/timezone:ro' if ansible_facts.os_family == 'Debian' else '' }}"
   - "kolla_logs:/var/log/kolla"
   - "ironic_inspector_dhcp_hosts:/etc/dnsmasq/dhcp-hostsdir:ro"
+ironic_prometheus_exporter_default_volumes:
+  - "{{ node_config_directory }}/ironic-prometheus-exporter/:{{ container_config_directory }}/:ro"
+  - "/etc/localtime:/etc/localtime:ro"
+  - "{{ '/etc/timezone:/etc/timezone:ro' if ansible_facts.os_family == 'Debian' else '' }}"
+  - "kolla_logs:/var/log/kolla"
+  - "ironic_prometheus_exporter_data:/var/lib/ironic/metrics"
 
 ironic_extra_volumes: "{{ default_extra_volumes }}"
 ironic_api_extra_volumes: "{{ ironic_extra_volumes }}"
@@ -255,6 +274,7 @@ ironic_tftp_extra_volumes: "{{ ironic_extra_volumes }}"
 ironic_http_extra_volumes: "{{ ironic_extra_volumes }}"
 ironic_inspector_extra_volumes: "{{ ironic_extra_volumes }}"
 ironic_dnsmasq_extra_volumes: "{{ ironic_extra_volumes }}"
+ironic_prometheus_exporter_extra_volumes: "{{ ironic_extra_volumes }}"
 
 ####################
 # OpenStack
@@ -288,6 +308,9 @@ ironic_enable_rolling_upgrade: "yes"
 ironic_upgrade_skip_wait_check: false
 ironic_inspector_kernel_cmdline_extras: []
 ironic_inspector_pxe_filter: "{% if enable_neutron | bool %}dnsmasq{% else %}noop{% endif %}"
+ironic_prometheus_exporter_data_dir: "/var/lib/ironic-prometheus-exporter/data"
+ironic_prometheus_exporter_sensor_data_interval: 30
+ironic_prometheus_exporter_sensor_data_undeployed_nodes: "true"
 
 
 ####################

ansible/roles/ironic/handlers/main.yml

@@ -96,3 +96,18 @@
     cap_add: "{{ service.cap_add }}"
   when:
     - kolla_action != "config"
+
+- name: Restart ironic-prometheus-exporter container
+  vars:
+    service_name: "ironic-prometheus-exporter"
+    service: "{{ ironic_services[service_name] }}"
+  become: true
+  kolla_docker:
+    action: "recreate_or_restart_container"
+    common_options: "{{ docker_common_options }}"
+    name: "{{ service.container_name }}"
+    image: "{{ service.image }}"
+    volumes: "{{ service.volumes }}"
+    dimensions: "{{ service.dimensions }}"
+  when:
+    - kolla_action != "config"

ansible/roles/ironic/tasks/config.yml

@@ -81,7 +81,7 @@
     mode: "0660"
   become: true
   when:
-    - item.key in [ "ironic-api", "ironic-conductor" ]
+    - item.key in [ "ironic-api", "ironic-conductor", "ironic-prometheus-exporter" ]
     - inventory_hostname in groups[item.value.group]
     - item.value.enabled | bool
   with_dict: "{{ ironic_services }}"
@@ -243,6 +243,24 @@
   notify:
     - Restart ironic-http container
 
+- name: Copying over ironic-prometheus-exporter-wsgi.conf
+  vars:
+    service: "{{ ironic_services['ironic-prometheus-exporter'] }}"
+  template:
+    src: "{{ item }}"
+    dest: "{{ node_config_directory }}/ironic-prometheus-exporter/ironic-prometheus-exporter-wsgi.conf"
+    mode: "0660"
+  become: true
+  with_first_found:
+    - "{{ node_config_directory }}/ironic/{{ inventory_hostname }}/ironic-prometheus-exporter-wsgi.conf"
+    - "{{ node_config_directory }}/ironic/ironic-prometheus-exporter-wsgi.conf"
+    - "ironic-prometheus-exporter-wsgi.conf.j2"
+  when:
+    - inventory_hostname in groups[service.group]
+    - service.enabled | bool
+  notify:
+    - "Restart ironic-prometheus-exporter container"
+
 - name: Copying over existing Ironic policy file
   vars:
     services_require_policy_json:

ansible/roles/ironic/tasks/precheck.yml

@@ -49,6 +49,18 @@
     - container_facts['ironic_http'] is not defined
     - inventory_hostname in groups['ironic-http']
 
+- name: Checking free port for Ironic Prometheus Exporter
+  wait_for:
+    host: "{{ api_interface_address }}"
+    port: "{{ ironic_prometheus_exporter_port }}"
+    connect_timeout: 1
+    timeout: 1
+    state: stopped
+  when:
+    - enable_ironic_prometheus_exporter | bool
+    - container_facts['ironic_prometheus_exporter'] is not defined
+    - inventory_hostname in groups['ironic-conductor']
+
 - name: Checking ironic-agent files exist for Ironic Inspector
   stat:
     path: "{{ ironic_agent_files_directory }}/ironic/{{ item }}"

ansible/roles/ironic/templates/ironic-conductor.json.j2

@@ -24,6 +24,11 @@
             "path": "/var/lib/ironic",
             "owner": "ironic:ironic",
             "recurse": true
-        }
+        }{% if enable_ironic_prometheus_exporter | bool %},
+        {
+            "path": "/var/lib/ironic/metrics",
+            "owner": "ironic:ironic",
+            "recurse": true
+        }{% endif %}
     ]
 }

ansible/roles/ironic/templates/ironic-prometheus-exporter-wsgi.conf.j2

@@ -0,0 +1,38 @@
+{% set ironic_log_dir = '/var/log/kolla/ironic' %}
+{% set python_path = '/var/lib/kolla/venv/lib/python' + distro_python_version + '/site-packages' %}
+Listen {{ api_interface_address | put_address_in_context('url') }}:{{ ironic_prometheus_exporter_port }}
+
+ServerSignature Off
+ServerTokens Prod
+TraceEnable off
+
+<Directory "{{ python_path }}/ironic_prometheus_exporter/app">
+    <FilesMatch "^wsgi$">
+        AllowOverride None
+        Options None
+        Require all granted
+    </FilesMatch>
+</Directory>
+
+ErrorLog "{{ ironic_log_dir }}/apache-error.log"
+<IfModule log_config_module>
+    CustomLog "{{ ironic_log_dir }}/apache-access.log" common
+</IfModule>
+
+{% if ironic_logging_debug | bool %}
+LogLevel info
+{% endif %}
+
+<VirtualHost *:{{ ironic_prometheus_exporter_port }}>
+    ErrorLog "{{ ironic_log_dir }}/ironic-prometheus-exporter-wsgi-error.log"
+    LogFormat "%{X-Forwarded-For}i %l %u %t \"%r\" %>s %b %D \"%{Referer}i\" \"%{User-Agent}i\"" logformat
+    CustomLog "{{ ironic_log_dir }}/ironic-prometheus-exporter-wsgi-access.log" logformat
+
+    WSGIDaemonProcess ironic-prometheus-exporter processes={{ openstack_service_workers }} threads=1 user=ironic display-name=%{GROUP} python-path={{ python_path }}
+    WSGIProcessGroup ironic-prometheus-exporter
+    WSGIScriptAlias / {{ python_path }}/ironic_prometheus_exporter/app/wsgi.py
+    WSGIApplicationGroup %{GLOBAL}
+    <Location "/">
+        Require all granted
+    </Location>
+</VirtualHost>

ansible/roles/ironic/templates/ironic-prometheus-exporter.json.j2

@@ -0,0 +1,31 @@
+{% set ironic_prometheus_exporter_cmd = 'apache2' if kolla_base_distro in ['ubuntu', 'debian'] else 'httpd' %}
+{% set ironic_prometheus_exporter_dir = 'apache2/conf-enabled' if kolla_base_distro in ['ubuntu', 'debian'] else 'httpd/conf.d' %}
+{
+    "command": "/usr/sbin/{{ ironic_prometheus_exporter_cmd }} -DFOREGROUND",
+    "config_files": [
+        {
+            "source": "{{ container_config_directory }}/ironic-prometheus-exporter-wsgi.conf",
+            "dest": "/etc/{{ ironic_prometheus_exporter_dir }}/ironic-prometheus-exporter-wsgi.conf",
+            "owner": "ironic",
+            "perm": "0600"
+        },
+        {
+            "source": "{{ container_config_directory }}/ironic.conf",
+            "dest": "/etc/ironic/ironic.conf",
+            "owner": "ironic",
+            "perm": "0600"
+        }
+    ],
+    "permissions": [
+        {
+            "path": "/var/log/kolla/ironic",
+            "owner": "ironic:ironic",
+            "recurse": true
+        },
+        {
+            "path": "/var/lib/ironic/metrics",
+            "owner": "ironic:ironic",
+            "recurse": true
+        }
+  ]
+}

ansible/roles/ironic/templates/ironic.conf.j2

@@ -20,12 +20,20 @@ notification_level = info
 
 [oslo_messaging_notifications]
 transport_url = {{ notify_transport_url }}
+{% if ironic_enabled_notification_topics or enable_ironic_prometheus_exporter | bool %}
 {% if ironic_enabled_notification_topics %}
 driver = messagingv2
 topics = {{ ironic_enabled_notification_topics | map(attribute='name') | join(',') }}
+{% endif %}
+{% if enable_ironic_prometheus_exporter | bool %}
+driver = prometheus_exporter
+{% endif %}
 {% else %}
 driver = noop
 {% endif %}
+{% if enable_ironic_prometheus_exporter | bool %}
+location = /var/lib/ironic/metrics
+{% endif %}
 
 [oslo_messaging_rabbit]
 heartbeat_in_pthread = {{ service_name == 'ironic-api' }}
@@ -45,6 +53,11 @@ policy_file = {{ ironic_policy_file }}
 {% if service_name == 'ironic-conductor' %}
 [conductor]
 automated_clean=false
+{% if enable_ironic_prometheus_exporter | bool %}
+send_sensor_data = true
+send_sensor_data_for_undeployed_nodes = {{ ironic_prometheus_exporter_sensor_data_undeployed_nodes }}
+send_sensor_data_interval = {{ ironic_prometheus_exporter_sensor_data_interval }}
+{% endif %}
 {% endif %}
 
 [database]

ansible/roles/prometheus/templates/prometheus.yml.j2

@@ -214,6 +214,14 @@ scrape_configs:
 {% endfor %}
 {% endif %}
 
+{% if enable_ironic_prometheus_exporter | bool %}
+  - job_name: ironic_prometheus_exporter
+    static_configs:
+{% for host in groups['ironic-conductor'] %}
+      - targets: ["{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['ironic_prometheus_exporter_port'] }}"]
+{% endfor %}
+{% endif %}
+
 {% if enable_prometheus_alertmanager | bool %}
   - job_name: alertmanager
     static_configs:

etc/kolla/globals.yml

@@ -362,6 +362,7 @@ workaround_ansible_issue_8743: yes
 #enable_influxdb: "{{ enable_cloudkitty | bool and cloudkitty_storage_backend == 'influxdb' }}"
 #enable_ironic: "no"
 #enable_ironic_neutron_agent: "{{ enable_neutron | bool and enable_ironic | bool }}"
+#enable_ironic_prometheus_exporter: "{{ enable_ironic | bool and enable_prometheus | bool }}"
 #enable_iscsid: "{{ enable_cinder | bool and enable_cinder_backend_iscsi | bool }}"
 #enable_kuryr: "no"
 #enable_magnum: "no"

releasenotes/notes/add-ironic-prometheus-exporter-218a9985905602fd.yaml

@@ -0,0 +1,8 @@
+---
+features:
+  - |
+    Adds support for deploying the ironic-prometheus-exporter, 'a Tool to
+    expose hardware sensor data in the Prometheus format through an HTTP
+    endpoint'.
+    See https://opendev.org/openstack/ironic-prometheus-exporter for more details
+    about the exporter.