openstack/kolla-ansible
Code Issues Proposed changes

Fluentd cleanup

Browse Source 
We no longer have to support multiple fluentd versions and rely
on the workaround.

Change-Id: Id4083a9f07317fc04467e52176850f3909703169
This commit is contained in:
Radosław Piliszek 2021-12-28 14:33:02 +00:00
parent 54bc017ae1
commit df2a29e483
13 changed files with 21 additions and 102 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
ansible/roles/common/tasks/config.yml
View File

@ -23,38 +23,6 @@
- paths
when: service | service_enabled_and_mapped_to_host
- name: Ensure fluentd image is present for label check
vars:
service_name: "fluentd"
service: "{{ common_services[service_name] }}"
become: true
kolla_docker:
action: "ensure_image"
common_options: "{{ docker_common_options }}"
image: "{{ service.image }}"
when:
- fluentd_version is not defined or fluentd_binary is not defined
- service | service_enabled_and_mapped_to_host
- name: Fetch fluentd image labels
vars:
service_name: "fluentd"
service: "{{ common_services[service_name] }}"
become: true
docker_image_info:
name: "{{ service.image }}"
register: fluentd_labels
when:
- fluentd_version is not defined or fluentd_binary is not defined
- service | service_enabled_and_mapped_to_host
- name: Set fluentd facts
set_fact:
fluentd_binary: "{{ fluentd_labels.images.0.ContainerConfig.Labels.fluentd_binary }}"
when:
- fluentd_binary is not defined
- common_services.fluentd | service_enabled_and_mapped_to_host
- include_tasks: copy-certs.yml
when:
- kolla_copy_ca_into_containers | bool
@ -112,7 +80,6 @@
{{ ( enable_elasticsearch | bool or
( elasticsearch_address != kolla_internal_fqdn )) and
( not enable_monasca | bool or not monasca_ingest_control_plane_logs | bool ) }}
fluentd_version: "{{ fluentd_labels.images.0.ContainerConfig.Labels.fluentd_version | default('0.12') }}"
# Inputs
fluentd_input_files: "{{ default_input_files_enabled | customise_fluentd(customised_input_files) }}"
default_input_files_enabled: "{{ default_input_files | selectattr('enabled') | map(attribute='name') | list }}"
@ -142,7 +109,7 @@
fluentd_filter_files: "{{ default_filter_files | customise_fluentd(customised_filter_files) }}"
default_filter_files:
- "conf/filter/00-record_transformer.conf.j2"
- "conf/filter/{{ '01-rewrite-0.14' if fluentd_version == '0.14' else '01-rewrite-0.12' }}.conf.j2"
- "conf/filter/01-rewrite.conf.j2"
- "conf/filter/02-parser.conf.j2"
customised_filter_files: "{{ find_custom_fluentd_filters.files | map(attribute='path') | list }}"
# Formats

44
ansible/roles/common/templates/conf/filter/01-rewrite-0.12.conf.j2
View File

@ -1,44 +0,0 @@
<match kolla.var.log.kolla.*.*.log>
@type rewrite_tag_filter
capitalize_regex_backreference yes
rewriterule1 programname ^(cinder-api-access|cloudkitty-api-access|gnocchi-api-access|horizon-access|keystone-apache-admin-access|keystone-apache-public-access|monasca-api-access|octavia-api-access|placement-api-access)$ apache_access
rewriterule2 programname ^(aodh_wsgi_access|barbican_api_uwsgi_access|zun_api_wsgi_access|vitrage_wsgi_access)$ wsgi_access
rewriterule3 programname ^(nova-api|nova-compute|nova-compute-ironic|nova-conductor|nova-manage|nova-novncproxy|nova-scheduler|nova-placement-api|placement-api|privsep-helper)$ openstack_python
rewriterule4 programname ^(sahara-api|sahara-engine)$ openstack_python
rewriterule5 programname ^(neutron-server|neutron-openvswitch-agent|neutron-ns-metadata-proxy|neutron-metadata-agent|neutron-l3-agent|neutron-dhcp-agent)$ openstack_python
rewriterule6 programname ^(magnum-conductor|magnum-api)$ openstack_python
rewriterule7 programname ^(keystone)$ openstack_python
rewriterule8 programname ^(heat-engine|heat-api|heat-api-cfn)$ openstack_python
rewriterule9 programname ^(glance-api)$ openstack_python
rewriterule10 programname ^(cloudkitty-storage-init|cloudkitty-processor|cloudkitty-dbsync|cloudkitty-api)$ openstack_python
rewriterule11 programname ^(ceilometer-polling|ceilometer-agent-notification)$ openstack_python
rewriterule12 programname ^(barbican-api|barbican-worker|barbican-keystone-listener|barbican-db-manage|app)$ openstack_python
rewriterule13 programname ^(aodh-notifier|aodh-listener|aodh-evaluator|aodh-dbsync)$ openstack_python
rewriterule14 programname ^(cinder-api|cinder-scheduler|cinder-manage|cinder-volume|cinder-backup|privsep-helper)$ openstack_python
rewriterule15 programname ^(mistral-server|mistral-engine|mistral-executor)$ openstack_python
rewriterule16 programname ^(designate-api|designate-central|designate-manage|designate-mdns|designate-sink|designate-worker)$ openstack_python
rewriterule17 programname ^(manila-api|manila-data|manila-manage|manila-share|manila-scheduler)$ openstack_python
rewriterule18 programname ^(trove-api|trove-conductor|trove-manage|trove-taskmanager)$ openstack_python
rewriterule20 programname ^(murano-api|murano-engine)$ openstack_python
rewriterule21 programname ^(senlin-api|senlin-conductor|senlin-engine|senlin-health-manager)$ openstack_python
rewriterule22 programname ^(watcher-api|watcher-applier|watcher-db-manage|watcher-decision-engine)$ openstack_python
rewriterule23 programname ^(freezer-api|freezer-api_access|freezer-manage)$ openstack_python
rewriterule24 programname ^(octavia-api|octavia-health-manager|octavia-housekeeping|octavia-worker)$ openstack_python
rewriterule25 programname ^(zun-api|zun-compute|zun-cni-daemon)$ openstack_python
rewriterule26 programname ^(kuryr-server)$ openstack_python
rewriterule27 programname ^(gnocchi-api|gnocchi-statsd|gnocchi-metricd|gnocchi-upgrade)$ openstack_python
rewriterule28 programname ^(ironic-api|ironic-conductor|ironic-inspector)$ openstack_python
rewriterule30 programname ^(tacker-server|tacker-conductor)$ openstack_python
rewriterule31 programname ^(vitrage-ml|vitrage-notifier|vitrage-graph|vitrage-persistor)$ openstack_python
rewriterule32 programname ^(blazar-api|blazar-manager)$ openstack_python
rewriterule33 programname ^(cyborg-api|cyborg-conductor|cyborg-agent)$ openstack_python
rewriterule34 programname ^(monasca-api|monasca-notification|monasca-persister|agent-collector|agent-forwarder|agent-statsd)$ openstack_python
rewriterule35 programname .+ unmatched
</match>
# Retag log messages from MariaDB according to log format
<match infra.mariadb>
@type rewrite_tag_filter
rewriterule1 Payload ^\d{6} infra.mariadb.mysqld_safe
rewriterule2 Payload ^\d{4}-\d{2}-\d{2} infra.mariadb.mysqld
</match>

0
ansible/roles/common/templates/conf/filter/01-rewrite-0.14.conf.j2 → ansible/roles/common/templates/conf/filter/01-rewrite.conf.j2
View File

2
ansible/roles/common/templates/conf/input/00-global.conf.j2
View File

@ -12,7 +12,7 @@
"/var/log/kolla/*/*-error.log",
"/var/log/kolla/*/*_access.log",
"/var/log/kolla/*/*_error.log"]
pos_file /var/run/{{ fluentd_binary }}/kolla-openstack.pos
pos_file /var/run/td-agent/kolla-openstack.pos
tag kolla.*
format multiline
format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3} \d+ \S+ \S+ \[(req-\S+ \S+ \S+ \S+ \S+ \S+|-)\]/

4
ansible/roles/common/templates/conf/input/02-mariadb.conf.j2
View File

@ -3,7 +3,7 @@
<source>
@type tail
path /var/log/kolla/mariadb/mariadb.log
pos_file /var/run/{{ fluentd_binary }}/mariadb.pos
pos_file /var/run/td-agent/mariadb.pos
tag infra.mariadb
format multiline
format_firstline /^(\d{4}-\d{2}-\d{2}|\d{6}) /
@ -13,7 +13,7 @@
<source>
@type tail
path /var/log/kolla/mariadb/xinetd.log
pos_file /var/run/{{ fluentd_binary }}/mariadb-xinetd.pos
pos_file /var/run/td-agent/mariadb-xinetd.pos
tag infra.mariadb-xinetd
format multiline
format_firstline /^\d{2}/\d{1,2}/\d{1,2}@\d{1,2}:\d{1,2}:\d{1,2}\: (START|EXIT)\: /

2
ansible/roles/common/templates/conf/input/03-rabbitmq.conf.j2
View File

@ -1,7 +1,7 @@
<source>
@type tail
path /var/log/kolla/rabbitmq/rabbit@{{ ansible_facts.hostname }}.log
pos_file /var/run/{{ fluentd_binary }}/rabbit.pos
pos_file /var/run/td-agent/rabbit.pos
tag infra.rabbit
format multiline
format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3}/

2
ansible/roles/common/templates/conf/input/04-openstack-wsgi.conf.j2
View File

@ -2,7 +2,7 @@
<source>
@type tail
path /var/log/kolla/*/*-access.log,/var/log/kolla/*/*-error.log,/var/log/kolla/*/*_access.log,/var/log/kolla/*/*_error.log
pos_file /var/run/{{ fluentd_binary }}/kolla-openstack-wsgi.pos
pos_file /var/run/td-agent/kolla-openstack-wsgi.pos
tag kolla.*
format /^(?<Payload>.*)$/
enable_watch_timer false

2
ansible/roles/common/templates/conf/input/05-libvirt.conf.j2
View File

@ -1,7 +1,7 @@
<source>
@type tail
path /var/log/kolla/libvirt/libvirtd.log
pos_file /var/run/{{ fluentd_binary }}/libvirt.pos
pos_file /var/run/td-agent/libvirt.pos
tag infra.libvirt
format /^(?<Timestamp>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3}\+\d{4}): (?<Pid>\d+): (?<log_level>\S+) : (?<Payload>.*)?$/
time_key Timestamp

2
ansible/roles/common/templates/conf/input/06-zookeeper.conf.j2
View File

@ -1,7 +1,7 @@
<source>
@type tail
path /var/log/kolla/zookeeper/zookeeper.log
pos_file /var/run/{{ fluentd_binary }}/zookeeper.pos
pos_file /var/run/td-agent/zookeeper.pos
tag infra.*
format multiline
format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3} \S+ \S+ \S+ .*$/

2
ansible/roles/common/templates/conf/input/07-kafka.conf.j2
View File

@ -1,7 +1,7 @@
<source>
@type tail
path /var/log/kolla/kafka/controller.log, /var/log/kolla/kafka/server.log, /var/log/kolla/kafka/state-change.log
pos_file /var/run/{{ fluentd_binary }}/kafka.pos
pos_file /var/run/td-agent/kafka.pos
tag infra.*
format multiline
format_firstline /^\[\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}\] \S+ .*$/

4
ansible/roles/common/templates/conf/input/09-monasca.conf.j2
View File

@ -1,7 +1,7 @@
<source>
@type tail
path /var/log/kolla/monasca/agent*.log
pos_file /var/run/{{ fluentd_binary }}/monasca-agent.pos
pos_file /var/run/td-agent/monasca-agent.pos
tag kolla.*
format multiline
format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} \S+ \| \S+ \| \S+ \| .*$/
@ -12,7 +12,7 @@
<source>
@type tail
path /var/log/kolla/monasca/grafana.log
pos_file /var/run/{{ fluentd_binary }}/monasca-grafana.pos
pos_file /var/run/td-agent/monasca-grafana.pos
tag infra.*
format multiline
format_firstline /^t=\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\+\d{4} lvl=\S+ msg=.*$/

18
ansible/roles/common/templates/fluentd.json.j2
View File

@ -1,17 +1,7 @@
{% set fluentd_user = fluentd_binary %}
{% set fluentd_dir = '/etc/' ~ fluentd_binary %}
{% if fluentd_binary == 'fluentd' %}
{% set fluentd_conf = 'fluent.conf' %}
{% if kolla_base_distro in ['ubuntu', 'debian'] %}
{% set fluentd_cmd = '/usr/local/bin/fluentd -c ' ~ fluentd_dir ~ '/' ~ fluentd_conf %}
{% else %}
{% set fluentd_cmd = '/usr/bin/fluentd -c ' ~ fluentd_dir ~ '/' ~ fluentd_conf %}
{% endif %}
{% elif fluentd_binary == 'td-agent' %}
{% set fluentd_conf = fluentd_binary ~ '.conf' %}
{% set fluentd_cmd = '/usr/sbin/td-agent' %}
{% endif %}
{% set fluentd_user = 'td-agent' %}
{% set fluentd_dir = '/etc/td-agent' %}
{% set fluentd_conf = 'td-agent.conf' %}
{% set fluentd_cmd = '/usr/sbin/td-agent' %}
{
"command": "{{ fluentd_cmd }} -o /var/log/kolla/fluentd/fluentd.log",

6
releasenotes/notes/fluentd-cleanup-efe1a668e36b0872.yaml Normal file
View File

@ -0,0 +1,6 @@
---
upgrade:
- |
``fluentd_binary`` and ``fluentd_version`` variables are no longer in use
as Kolla Ansible supports a single fluentd version across all supported
Kolla image flavours.