diff --git a/ansible/roles/common/tasks/config.yml b/ansible/roles/common/tasks/config.yml index 79851c2a4c..17e6310248 100644 --- a/ansible/roles/common/tasks/config.yml +++ b/ansible/roles/common/tasks/config.yml @@ -23,38 +23,6 @@ - paths when: service | service_enabled_and_mapped_to_host -- name: Ensure fluentd image is present for label check - vars: - service_name: "fluentd" - service: "{{ common_services[service_name] }}" - become: true - kolla_docker: - action: "ensure_image" - common_options: "{{ docker_common_options }}" - image: "{{ service.image }}" - when: - - fluentd_version is not defined or fluentd_binary is not defined - - service | service_enabled_and_mapped_to_host - -- name: Fetch fluentd image labels - vars: - service_name: "fluentd" - service: "{{ common_services[service_name] }}" - become: true - docker_image_info: - name: "{{ service.image }}" - register: fluentd_labels - when: - - fluentd_version is not defined or fluentd_binary is not defined - - service | service_enabled_and_mapped_to_host - -- name: Set fluentd facts - set_fact: - fluentd_binary: "{{ fluentd_labels.images.0.ContainerConfig.Labels.fluentd_binary }}" - when: - - fluentd_binary is not defined - - common_services.fluentd | service_enabled_and_mapped_to_host - - include_tasks: copy-certs.yml when: - kolla_copy_ca_into_containers | bool @@ -112,7 +80,6 @@ {{ ( enable_elasticsearch | bool or ( elasticsearch_address != kolla_internal_fqdn )) and ( not enable_monasca | bool or not monasca_ingest_control_plane_logs | bool ) }} - fluentd_version: "{{ fluentd_labels.images.0.ContainerConfig.Labels.fluentd_version | default('0.12') }}" # Inputs fluentd_input_files: "{{ default_input_files_enabled | customise_fluentd(customised_input_files) }}" default_input_files_enabled: "{{ default_input_files | selectattr('enabled') | map(attribute='name') | list }}" @@ -142,7 +109,7 @@ fluentd_filter_files: "{{ default_filter_files | customise_fluentd(customised_filter_files) }}" default_filter_files: - "conf/filter/00-record_transformer.conf.j2" - - "conf/filter/{{ '01-rewrite-0.14' if fluentd_version == '0.14' else '01-rewrite-0.12' }}.conf.j2" + - "conf/filter/01-rewrite.conf.j2" - "conf/filter/02-parser.conf.j2" customised_filter_files: "{{ find_custom_fluentd_filters.files | map(attribute='path') | list }}" # Formats diff --git a/ansible/roles/common/templates/conf/filter/01-rewrite-0.12.conf.j2 b/ansible/roles/common/templates/conf/filter/01-rewrite-0.12.conf.j2 deleted file mode 100644 index 0f4ac250a4..0000000000 --- a/ansible/roles/common/templates/conf/filter/01-rewrite-0.12.conf.j2 +++ /dev/null @@ -1,44 +0,0 @@ - - @type rewrite_tag_filter - capitalize_regex_backreference yes - rewriterule1 programname ^(cinder-api-access|cloudkitty-api-access|gnocchi-api-access|horizon-access|keystone-apache-admin-access|keystone-apache-public-access|monasca-api-access|octavia-api-access|placement-api-access)$ apache_access - rewriterule2 programname ^(aodh_wsgi_access|barbican_api_uwsgi_access|zun_api_wsgi_access|vitrage_wsgi_access)$ wsgi_access - rewriterule3 programname ^(nova-api|nova-compute|nova-compute-ironic|nova-conductor|nova-manage|nova-novncproxy|nova-scheduler|nova-placement-api|placement-api|privsep-helper)$ openstack_python - rewriterule4 programname ^(sahara-api|sahara-engine)$ openstack_python - rewriterule5 programname ^(neutron-server|neutron-openvswitch-agent|neutron-ns-metadata-proxy|neutron-metadata-agent|neutron-l3-agent|neutron-dhcp-agent)$ openstack_python - rewriterule6 programname ^(magnum-conductor|magnum-api)$ openstack_python - rewriterule7 programname ^(keystone)$ openstack_python - rewriterule8 programname ^(heat-engine|heat-api|heat-api-cfn)$ openstack_python - rewriterule9 programname ^(glance-api)$ openstack_python - rewriterule10 programname ^(cloudkitty-storage-init|cloudkitty-processor|cloudkitty-dbsync|cloudkitty-api)$ openstack_python - rewriterule11 programname ^(ceilometer-polling|ceilometer-agent-notification)$ openstack_python - rewriterule12 programname ^(barbican-api|barbican-worker|barbican-keystone-listener|barbican-db-manage|app)$ openstack_python - rewriterule13 programname ^(aodh-notifier|aodh-listener|aodh-evaluator|aodh-dbsync)$ openstack_python - rewriterule14 programname ^(cinder-api|cinder-scheduler|cinder-manage|cinder-volume|cinder-backup|privsep-helper)$ openstack_python - rewriterule15 programname ^(mistral-server|mistral-engine|mistral-executor)$ openstack_python - rewriterule16 programname ^(designate-api|designate-central|designate-manage|designate-mdns|designate-sink|designate-worker)$ openstack_python - rewriterule17 programname ^(manila-api|manila-data|manila-manage|manila-share|manila-scheduler)$ openstack_python - rewriterule18 programname ^(trove-api|trove-conductor|trove-manage|trove-taskmanager)$ openstack_python - rewriterule20 programname ^(murano-api|murano-engine)$ openstack_python - rewriterule21 programname ^(senlin-api|senlin-conductor|senlin-engine|senlin-health-manager)$ openstack_python - rewriterule22 programname ^(watcher-api|watcher-applier|watcher-db-manage|watcher-decision-engine)$ openstack_python - rewriterule23 programname ^(freezer-api|freezer-api_access|freezer-manage)$ openstack_python - rewriterule24 programname ^(octavia-api|octavia-health-manager|octavia-housekeeping|octavia-worker)$ openstack_python - rewriterule25 programname ^(zun-api|zun-compute|zun-cni-daemon)$ openstack_python - rewriterule26 programname ^(kuryr-server)$ openstack_python - rewriterule27 programname ^(gnocchi-api|gnocchi-statsd|gnocchi-metricd|gnocchi-upgrade)$ openstack_python - rewriterule28 programname ^(ironic-api|ironic-conductor|ironic-inspector)$ openstack_python - rewriterule30 programname ^(tacker-server|tacker-conductor)$ openstack_python - rewriterule31 programname ^(vitrage-ml|vitrage-notifier|vitrage-graph|vitrage-persistor)$ openstack_python - rewriterule32 programname ^(blazar-api|blazar-manager)$ openstack_python - rewriterule33 programname ^(cyborg-api|cyborg-conductor|cyborg-agent)$ openstack_python - rewriterule34 programname ^(monasca-api|monasca-notification|monasca-persister|agent-collector|agent-forwarder|agent-statsd)$ openstack_python - rewriterule35 programname .+ unmatched - - -# Retag log messages from MariaDB according to log format - - @type rewrite_tag_filter - rewriterule1 Payload ^\d{6} infra.mariadb.mysqld_safe - rewriterule2 Payload ^\d{4}-\d{2}-\d{2} infra.mariadb.mysqld - diff --git a/ansible/roles/common/templates/conf/filter/01-rewrite-0.14.conf.j2 b/ansible/roles/common/templates/conf/filter/01-rewrite.conf.j2 similarity index 100% rename from ansible/roles/common/templates/conf/filter/01-rewrite-0.14.conf.j2 rename to ansible/roles/common/templates/conf/filter/01-rewrite.conf.j2 diff --git a/ansible/roles/common/templates/conf/input/00-global.conf.j2 b/ansible/roles/common/templates/conf/input/00-global.conf.j2 index e31f1b9f6e..8b4afa5951 100644 --- a/ansible/roles/common/templates/conf/input/00-global.conf.j2 +++ b/ansible/roles/common/templates/conf/input/00-global.conf.j2 @@ -12,7 +12,7 @@ "/var/log/kolla/*/*-error.log", "/var/log/kolla/*/*_access.log", "/var/log/kolla/*/*_error.log"] - pos_file /var/run/{{ fluentd_binary }}/kolla-openstack.pos + pos_file /var/run/td-agent/kolla-openstack.pos tag kolla.* format multiline format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3} \d+ \S+ \S+ \[(req-\S+ \S+ \S+ \S+ \S+ \S+|-)\]/ diff --git a/ansible/roles/common/templates/conf/input/02-mariadb.conf.j2 b/ansible/roles/common/templates/conf/input/02-mariadb.conf.j2 index 8744db47aa..03726607a5 100644 --- a/ansible/roles/common/templates/conf/input/02-mariadb.conf.j2 +++ b/ansible/roles/common/templates/conf/input/02-mariadb.conf.j2 @@ -3,7 +3,7 @@ @type tail path /var/log/kolla/mariadb/mariadb.log - pos_file /var/run/{{ fluentd_binary }}/mariadb.pos + pos_file /var/run/td-agent/mariadb.pos tag infra.mariadb format multiline format_firstline /^(\d{4}-\d{2}-\d{2}|\d{6}) / @@ -13,7 +13,7 @@ @type tail path /var/log/kolla/mariadb/xinetd.log - pos_file /var/run/{{ fluentd_binary }}/mariadb-xinetd.pos + pos_file /var/run/td-agent/mariadb-xinetd.pos tag infra.mariadb-xinetd format multiline format_firstline /^\d{2}/\d{1,2}/\d{1,2}@\d{1,2}:\d{1,2}:\d{1,2}\: (START|EXIT)\: / diff --git a/ansible/roles/common/templates/conf/input/03-rabbitmq.conf.j2 b/ansible/roles/common/templates/conf/input/03-rabbitmq.conf.j2 index 91df08d37d..5846547079 100644 --- a/ansible/roles/common/templates/conf/input/03-rabbitmq.conf.j2 +++ b/ansible/roles/common/templates/conf/input/03-rabbitmq.conf.j2 @@ -1,7 +1,7 @@ @type tail path /var/log/kolla/rabbitmq/rabbit@{{ ansible_facts.hostname }}.log - pos_file /var/run/{{ fluentd_binary }}/rabbit.pos + pos_file /var/run/td-agent/rabbit.pos tag infra.rabbit format multiline format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3}/ diff --git a/ansible/roles/common/templates/conf/input/04-openstack-wsgi.conf.j2 b/ansible/roles/common/templates/conf/input/04-openstack-wsgi.conf.j2 index 130d298c43..73977d8cc0 100644 --- a/ansible/roles/common/templates/conf/input/04-openstack-wsgi.conf.j2 +++ b/ansible/roles/common/templates/conf/input/04-openstack-wsgi.conf.j2 @@ -2,7 +2,7 @@ @type tail path /var/log/kolla/*/*-access.log,/var/log/kolla/*/*-error.log,/var/log/kolla/*/*_access.log,/var/log/kolla/*/*_error.log - pos_file /var/run/{{ fluentd_binary }}/kolla-openstack-wsgi.pos + pos_file /var/run/td-agent/kolla-openstack-wsgi.pos tag kolla.* format /^(?.*)$/ enable_watch_timer false diff --git a/ansible/roles/common/templates/conf/input/05-libvirt.conf.j2 b/ansible/roles/common/templates/conf/input/05-libvirt.conf.j2 index 040b940621..b9713df78a 100644 --- a/ansible/roles/common/templates/conf/input/05-libvirt.conf.j2 +++ b/ansible/roles/common/templates/conf/input/05-libvirt.conf.j2 @@ -1,7 +1,7 @@ @type tail path /var/log/kolla/libvirt/libvirtd.log - pos_file /var/run/{{ fluentd_binary }}/libvirt.pos + pos_file /var/run/td-agent/libvirt.pos tag infra.libvirt format /^(?\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3}\+\d{4}): (?\d+): (?\S+) : (?.*)?$/ time_key Timestamp diff --git a/ansible/roles/common/templates/conf/input/06-zookeeper.conf.j2 b/ansible/roles/common/templates/conf/input/06-zookeeper.conf.j2 index 3193227fd7..061dad8dd6 100644 --- a/ansible/roles/common/templates/conf/input/06-zookeeper.conf.j2 +++ b/ansible/roles/common/templates/conf/input/06-zookeeper.conf.j2 @@ -1,7 +1,7 @@ @type tail path /var/log/kolla/zookeeper/zookeeper.log - pos_file /var/run/{{ fluentd_binary }}/zookeeper.pos + pos_file /var/run/td-agent/zookeeper.pos tag infra.* format multiline format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3} \S+ \S+ \S+ .*$/ diff --git a/ansible/roles/common/templates/conf/input/07-kafka.conf.j2 b/ansible/roles/common/templates/conf/input/07-kafka.conf.j2 index ad086ba899..a72db20a4d 100644 --- a/ansible/roles/common/templates/conf/input/07-kafka.conf.j2 +++ b/ansible/roles/common/templates/conf/input/07-kafka.conf.j2 @@ -1,7 +1,7 @@ @type tail path /var/log/kolla/kafka/controller.log, /var/log/kolla/kafka/server.log, /var/log/kolla/kafka/state-change.log - pos_file /var/run/{{ fluentd_binary }}/kafka.pos + pos_file /var/run/td-agent/kafka.pos tag infra.* format multiline format_firstline /^\[\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}\] \S+ .*$/ diff --git a/ansible/roles/common/templates/conf/input/09-monasca.conf.j2 b/ansible/roles/common/templates/conf/input/09-monasca.conf.j2 index 7da68f4b9d..1ae2b94681 100644 --- a/ansible/roles/common/templates/conf/input/09-monasca.conf.j2 +++ b/ansible/roles/common/templates/conf/input/09-monasca.conf.j2 @@ -1,7 +1,7 @@ @type tail path /var/log/kolla/monasca/agent*.log - pos_file /var/run/{{ fluentd_binary }}/monasca-agent.pos + pos_file /var/run/td-agent/monasca-agent.pos tag kolla.* format multiline format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} \S+ \| \S+ \| \S+ \| .*$/ @@ -12,7 +12,7 @@ @type tail path /var/log/kolla/monasca/grafana.log - pos_file /var/run/{{ fluentd_binary }}/monasca-grafana.pos + pos_file /var/run/td-agent/monasca-grafana.pos tag infra.* format multiline format_firstline /^t=\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\+\d{4} lvl=\S+ msg=.*$/ diff --git a/ansible/roles/common/templates/fluentd.json.j2 b/ansible/roles/common/templates/fluentd.json.j2 index 0df04b1aad..6c091e09a0 100644 --- a/ansible/roles/common/templates/fluentd.json.j2 +++ b/ansible/roles/common/templates/fluentd.json.j2 @@ -1,17 +1,7 @@ -{% set fluentd_user = fluentd_binary %} -{% set fluentd_dir = '/etc/' ~ fluentd_binary %} - -{% if fluentd_binary == 'fluentd' %} - {% set fluentd_conf = 'fluent.conf' %} - {% if kolla_base_distro in ['ubuntu', 'debian'] %} - {% set fluentd_cmd = '/usr/local/bin/fluentd -c ' ~ fluentd_dir ~ '/' ~ fluentd_conf %} - {% else %} - {% set fluentd_cmd = '/usr/bin/fluentd -c ' ~ fluentd_dir ~ '/' ~ fluentd_conf %} - {% endif %} -{% elif fluentd_binary == 'td-agent' %} - {% set fluentd_conf = fluentd_binary ~ '.conf' %} - {% set fluentd_cmd = '/usr/sbin/td-agent' %} -{% endif %} +{% set fluentd_user = 'td-agent' %} +{% set fluentd_dir = '/etc/td-agent' %} +{% set fluentd_conf = 'td-agent.conf' %} +{% set fluentd_cmd = '/usr/sbin/td-agent' %} { "command": "{{ fluentd_cmd }} -o /var/log/kolla/fluentd/fluentd.log", diff --git a/releasenotes/notes/fluentd-cleanup-efe1a668e36b0872.yaml b/releasenotes/notes/fluentd-cleanup-efe1a668e36b0872.yaml new file mode 100644 index 0000000000..0828c4aebc --- /dev/null +++ b/releasenotes/notes/fluentd-cleanup-efe1a668e36b0872.yaml @@ -0,0 +1,6 @@ +--- +upgrade: + - | + ``fluentd_binary`` and ``fluentd_version`` variables are no longer in use + as Kolla Ansible supports a single fluentd version across all supported + Kolla image flavours.