diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml index b42f2e2fae..4d79d42a98 100644 --- a/ansible/group_vars/all.yml +++ b/ansible/group_vars/all.yml @@ -452,6 +452,7 @@ prometheus_memcached_exporter_port: "9150" prometheus_rabbitmq_exporter_port: "15692" # Default cadvisor port of 8080 already in use prometheus_cadvisor_port: "18080" +prometheus_fluentd_integration_port: "24231" # Prometheus alertmanager ports prometheus_alertmanager_port: "9093" @@ -1099,6 +1100,7 @@ enable_prometheus_mysqld_exporter: "{{ enable_mariadb | bool }}" enable_prometheus_node_exporter: "{{ enable_prometheus | bool }}" enable_prometheus_memcached_exporter: "{{ enable_memcached | bool }}" enable_prometheus_cadvisor: "{{ enable_prometheus | bool }}" +enable_prometheus_fluentd_integration: "{{ enable_prometheus | bool and enable_fluentd | bool }}" enable_prometheus_alertmanager: "{{ enable_prometheus | bool }}" enable_prometheus_ceph_mgr_exporter: "no" enable_prometheus_openstack_exporter: "{{ enable_prometheus | bool }}" diff --git a/ansible/roles/common/tasks/config.yml b/ansible/roles/common/tasks/config.yml index 022570bff3..d03fdd5993 100644 --- a/ansible/roles/common/tasks/config.yml +++ b/ansible/roles/common/tasks/config.yml @@ -115,17 +115,29 @@ ( not enable_monasca | bool or not monasca_ingest_control_plane_logs | bool ) }} fluentd_version: "{{ fluentd_labels.images.0.ContainerConfig.Labels.fluentd_version | default('0.12') }}" # Inputs - fluentd_input_files: "{{ default_input_files | customise_fluentd(customised_input_files) }}" + fluentd_input_files: "{{ default_input_files_enabled | customise_fluentd(customised_input_files) }}" + default_input_files_enabled: "{{ default_input_files | selectattr('enabled') | map(attribute='name') | list }}" default_input_files: - - "conf/input/00-global.conf.j2" - - "conf/input/01-syslog.conf.j2" - - "conf/input/02-mariadb.conf.j2" - - "conf/input/03-rabbitmq.conf.j2" - - "conf/input/04-openstack-wsgi.conf.j2" - - "conf/input/05-libvirt.conf.j2" - - "conf/input/06-zookeeper.conf.j2" - - "conf/input/07-kafka.conf.j2" - - "conf/input/09-monasca.conf.j2" + - name: "conf/input/00-global.conf.j2" + enabled: true + - name: "conf/input/01-syslog.conf.j2" + enabled: true + - name: "conf/input/02-mariadb.conf.j2" + enabled: true + - name: "conf/input/03-rabbitmq.conf.j2" + enabled: true + - name: "conf/input/04-openstack-wsgi.conf.j2" + enabled: true + - name: "conf/input/05-libvirt.conf.j2" + enabled: true + - name: "conf/input/06-zookeeper.conf.j2" + enabled: true + - name: "conf/input/07-kafka.conf.j2" + enabled: true + - name: "conf/input/08-prometheus.conf.j2" + enabled: "{{ enable_prometheus_fluentd_integration | bool }}" + - name: "conf/input/09-monasca.conf.j2" + enabled: true customised_input_files: "{{ find_custom_fluentd_inputs.files | map(attribute='path') | list }}" # Filters fluentd_filter_files: "{{ default_filter_files | customise_fluentd(customised_filter_files) }}" diff --git a/ansible/roles/common/templates/conf/input/08-prometheus.conf.j2 b/ansible/roles/common/templates/conf/input/08-prometheus.conf.j2 new file mode 100644 index 0000000000..1aa5da895e --- /dev/null +++ b/ansible/roles/common/templates/conf/input/08-prometheus.conf.j2 @@ -0,0 +1,14 @@ + + @type prometheus + bind {{ api_interface_address }} + port {{ prometheus_fluentd_integration_port }} + metrics_path /metrics + + + + @type prometheus_output_monitor + interval 10 + + Hostname ${hostname} + + diff --git a/ansible/roles/prometheus/templates/prometheus.yml.j2 b/ansible/roles/prometheus/templates/prometheus.yml.j2 index 44e5226f5f..4dbce1852f 100644 --- a/ansible/roles/prometheus/templates/prometheus.yml.j2 +++ b/ansible/roles/prometheus/templates/prometheus.yml.j2 @@ -74,6 +74,15 @@ scrape_configs: {% endfor %} {% endif %} +{% if enable_prometheus_fluentd_integration | bool %} + - job_name: fluentd + static_configs: + - targets: +{% for host in groups['fluentd'] %} + - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_fluentd_integration_port'] }}' +{% endfor %} +{% endif %} + {% if enable_prometheus_ceph_mgr_exporter | bool %} - job_name: ceph_mgr_exporter honor_labels: true diff --git a/etc/kolla/globals.yml b/etc/kolla/globals.yml index a528d01bfa..74d8e67d86 100644 --- a/etc/kolla/globals.yml +++ b/etc/kolla/globals.yml @@ -694,6 +694,7 @@ #enable_prometheus_mysqld_exporter: "{{ enable_mariadb | bool }}" #enable_prometheus_node_exporter: "{{ enable_prometheus | bool }}" #enable_prometheus_cadvisor: "{{ enable_prometheus | bool }}" +#enable_prometheus_fluentd_integration: "{{ enable_prometheus | bool and enable fluentd | bool }}" #enable_prometheus_memcached: "{{ enable_prometheus | bool }}" #enable_prometheus_alertmanager: "{{ enable_prometheus | bool }}" #enable_prometheus_ceph_mgr_exporter: "no" diff --git a/releasenotes/notes/add-support-for-fluentd-exporter-ca02ff9182039e3e.yaml b/releasenotes/notes/add-support-for-fluentd-exporter-ca02ff9182039e3e.yaml new file mode 100644 index 0000000000..2896fd2e72 --- /dev/null +++ b/releasenotes/notes/add-support-for-fluentd-exporter-ca02ff9182039e3e.yaml @@ -0,0 +1,10 @@ +--- +features: + - | + Adds support for integrating Fluentd metrics into Prometheus. By + default this is now enabled when Prometheus is enabled. This behaviour + can be overridden via the `enable_prometheus_fluentd_integration` flag. By + default the integration provides metrics relating to the processing of logs + by Fluentd. These metrics can be useful for monitoring the status of the + Fluentd service. Additional metrics can also be extracted from logs via + custom Fluentd config.