diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml index ff0e4d2f0f..30f4b31722 100644 --- a/ansible/group_vars/all.yml +++ b/ansible/group_vars/all.yml @@ -130,7 +130,7 @@ rgw_port: "6780" mistral_api_port: "8989" -kibana_port: "5601" +kibana_server_port: "5601" elasticsearch_port: "9200" @@ -184,7 +184,7 @@ enable_murano: "no" enable_ironic: "no" enable_magnum: "no" enable_mistral: "no" -enable_elk: "no" +enable_central_logging: "no" enable_mongodb: "no" enable_manila: "no" diff --git a/ansible/roles/common/tasks/config.yml b/ansible/roles/common/tasks/config.yml index 18f87c610a..8592801ab1 100644 --- a/ansible/roles/common/tasks/config.yml +++ b/ansible/roles/common/tasks/config.yml @@ -46,3 +46,11 @@ - "swift-object-updater" - "swift-proxy-server" - "swift-rsyncd" + +- name: Copying over heka elasticsearch config file + template: + src: "heka-{{ item }}.toml.j2" + dest: "{{ node_config_directory }}/heka/heka-{{ item }}.toml" + with_items: + - "elasticsearch" + when: "{{ enable_central_logging | bool }}" diff --git a/ansible/roles/common/templates/heka-elasticsearch.toml.j2 b/ansible/roles/common/templates/heka-elasticsearch.toml.j2 new file mode 100644 index 0000000000..7b9a351cb5 --- /dev/null +++ b/ansible/roles/common/templates/heka-elasticsearch.toml.j2 @@ -0,0 +1,16 @@ +[elasticsearch_json_encoder] +type = "ESJsonEncoder" +index = {{'"%{Type}-%{%Y.%m.%d}"'}} +es_index_from_timestamp = true +fields = ["Timestamp", "Type", "Logger", "Severity", "Payload", "Pid", "Hostname", "DynamicFields"] + +[elasticsearch_output] +type = "ElasticSearchOutput" +server = "{{ internal_protocol }}://{{ kolla_internal_vip_address }}:{{ elasticsearch_port }}" +message_matcher = "Type == 'log'" +encoder = "elasticsearch_json_encoder" +use_buffering = true + [elasticsearch_output.buffering] + max_buffer_size = 1073741824 # 1024 * 1024 * 1024 + max_file_size = 134217728 # 128 * 1024 * 1024 + full_action = "drop" diff --git a/ansible/roles/common/templates/heka-haproxy.toml.j2 b/ansible/roles/common/templates/heka-haproxy.toml.j2 index d933a6f554..f30dd32d57 100644 --- a/ansible/roles/common/templates/heka-haproxy.toml.j2 +++ b/ansible/roles/common/templates/heka-haproxy.toml.j2 @@ -3,12 +3,12 @@ [haproxy_file_output] type = "FileOutput" -message_matcher = "Type == 'Syslog' && Fields[programname] =~ /(?i:haproxy)/" +message_matcher = "Fields[programname] =~ /(?i:haproxy)/" path = "/var/log/kolla/haproxy/haproxy.log" encoder = "syslog_encoder" [keepalived_file_output] type = "FileOutput" -message_matcher = "Type == 'Syslog' && Fields[programname] =~ /(?i:keepalived)/" +message_matcher = "Fields[programname] =~ /(?i:keepalived)/" path = "/var/log/kolla/haproxy/keepalived.log" encoder = "syslog_encoder" diff --git a/ansible/roles/common/templates/heka-swift.toml.j2 b/ansible/roles/common/templates/heka-swift.toml.j2 index 3bb520e322..8ed4a1e236 100644 --- a/ansible/roles/common/templates/heka-swift.toml.j2 +++ b/ansible/roles/common/templates/heka-swift.toml.j2 @@ -3,6 +3,6 @@ [{{ item }}_file_output] type = "FileOutput" -message_matcher = "Type == 'Syslog' && Fields[programname] == '{{ item }}'" +message_matcher = "Fields[programname] == '{{ item }}'" path = "/var/log/kolla/swift/{{ item }}.log" encoder = "syslog_encoder" diff --git a/ansible/roles/common/templates/heka.json.j2 b/ansible/roles/common/templates/heka.json.j2 index f5bcad2979..332790231e 100644 --- a/ansible/roles/common/templates/heka.json.j2 +++ b/ansible/roles/common/templates/heka.json.j2 @@ -2,6 +2,13 @@ { "command": "/usr/bin/hekad -config=/etc/heka/", "config_files": [ + { + "source": "{{ container_config_directory }}/heka-elasticsearch.toml", + "dest": "/etc/heka/heka-elasticsearch.toml", + "owner": "heka", + "perm": "0600", + "optional": "True" + }, { "source": "{{ container_config_directory }}/heka-global.toml", "dest": "/etc/heka/heka-global.toml", diff --git a/ansible/roles/elasticsearch/defaults/main.yml b/ansible/roles/elasticsearch/defaults/main.yml index c7c5cc986b..947a1cb9f1 100644 --- a/ansible/roles/elasticsearch/defaults/main.yml +++ b/ansible/roles/elasticsearch/defaults/main.yml @@ -2,8 +2,11 @@ #################### # Elasticsearch #################### -elasticsearch_port: "{{ elasticsearch_port }}" -elasticsearch_host: "{{ kolla_internal_vip_address }}" +elasticsearch_cluster_name: "kolla_logging" + +#################### +# Docker +#################### elasticsearch_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ kolla_install_type }}-elasticsearch" elasticsearch_tag: "{{ openstack_release }}" elasticsearch_image_full: "{{ elasticsearch_image }}:{{ elasticsearch_tag }}" diff --git a/ansible/roles/elasticsearch/templates/elasticsearch.yml.j2 b/ansible/roles/elasticsearch/templates/elasticsearch.yml.j2 index eba662b63d..449fb3c8af 100644 --- a/ansible/roles/elasticsearch/templates/elasticsearch.yml.j2 +++ b/ansible/roles/elasticsearch/templates/elasticsearch.yml.j2 @@ -1,4 +1,17 @@ -network.host: {{ hostvars[inventory_hostname]['ansible_' + api_interface]['ipv4']['address'] }} +{% set num_nodes = groups['elasticsearch'] | length %} +{% set minimum_master_nodes = (num_nodes / 2 + 1) | round(0, 'floor') | int if num_nodes > 2 else 1 %} +{% set recover_after_nodes = (num_nodes * 2 / 3) | round(0, 'floor') | int if num_nodes > 1 else 1 %} +node.name: "{{ hostvars[inventory_hostname]['ansible_' + api_interface]['ipv4']['address'] }}" +network.host: "{{ hostvars[inventory_hostname]['ansible_' + api_interface]['ipv4']['address'] }}" +cluster.name: "{{ elasticsearch_cluster_name }}" +node.master: true +node.data: true +discovery.zen.ping.unicast.hosts: [{% for host in groups['elasticsearch'] %}"{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}"{% if not loop.last %},{% endif %}{% endfor %}] + +discovery.zen.minimum_master_nodes: {{ minimum_master_nodes }} +gateway.expected_nodes: {{ num_nodes }} +gateway.recover_after_time: "5m" +gateway.recover_after_nodes: {{ recover_after_nodes }} path.conf: "/etc/elasticsearch" path.data: "/var/lib/elasticsearch/data" path.logs: "/var/log/elasticsearch" diff --git a/ansible/roles/haproxy/templates/haproxy.cfg.j2 b/ansible/roles/haproxy/templates/haproxy.cfg.j2 index a9f1e8ac6b..56d34e79b4 100644 --- a/ansible/roles/haproxy/templates/haproxy.cfg.j2 +++ b/ansible/roles/haproxy/templates/haproxy.cfg.j2 @@ -376,3 +376,26 @@ listen radosgw_external {% endfor %} {% endif %} {% endif %} + +{% if enable_central_logging | bool %} +listen kibana + bind {{ kolla_internal_vip_address }}:{{ kibana_server_port }} +{% for host in groups['kibana'] %} + server {{ hostvars[host]['ansible_hostname'] }} {{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ kibana_server_port }} check inter 2000 rise 2 fall 5 +{% endfor %} +{% if haproxy_enable_external_vip | bool %} + +listen kibana_external + bind {{ kolla_external_vip_address }}:{{ kibana_server_port }} +{% for host in groups['kibana'] %} + server {{ hostvars[host]['ansible_hostname'] }} {{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ kibana_server_port }} check inter 2000 rise 2 fall 5 +{% endfor %} +{% endif %} + +listen elasticsearch + option dontlog-normal + bind {{ kolla_internal_vip_address }}:{{ elasticsearch_port }} +{% for host in groups['elasticsearch'] %} + server {{ hostvars[host]['ansible_hostname'] }} {{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ elasticsearch_port }} check inter 2000 rise 2 fall 5 +{% endfor %} +{% endif %} diff --git a/ansible/roles/kibana/defaults/main.yml b/ansible/roles/kibana/defaults/main.yml index c720b03069..e35e2f1e71 100644 --- a/ansible/roles/kibana/defaults/main.yml +++ b/ansible/roles/kibana/defaults/main.yml @@ -2,12 +2,10 @@ #################### # Kibana #################### -kibana_port: "{{ kibana_port }}" -kibana_host: "{{ kolla_internal_vip_address }}" -kibana_app_id: "discover" -kibana_request_timeout: 300000 -kibana_shard_timeout: 0 -kibana_verify_ssl: false +kibana_default_app_id: "discover" +kibana_elasticsearch_request_timeout: 300000 +kibana_elasticsearch_shard_timeout: 0 +kibana_elasticsearch_ssl_verify: false #################### @@ -16,9 +14,3 @@ kibana_verify_ssl: false kibana_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ kolla_install_type }}-kibana" kibana_tag: "{{ openstack_release }}" kibana_image_full: "{{ kibana_image }}:{{ kibana_tag }}" - - -#################### -# Elasticsearch -#################### -elasticsearch_preserve_host: "true" diff --git a/ansible/roles/kibana/templates/kibana.yml.j2 b/ansible/roles/kibana/templates/kibana.yml.j2 index 2fa66eec93..cc31234c9d 100644 --- a/ansible/roles/kibana/templates/kibana.yml.j2 +++ b/ansible/roles/kibana/templates/kibana.yml.j2 @@ -1,11 +1,10 @@ -port: {{ kibana_port }} -host: {{ kibana_host }} -elasticsearch_url: "{{ internal_protocol }}://{{ kolla_internal_fqdn }}:{{ elasticsearch_port }}" -elasticsearch_preserve_host: {{ elasticsearch_preserve_host }} -default_app_id: {{ kibana_app_id }} -request_timeout: {{ kibana_request_timeout }} -shard_timeout: {{ kibana_shard_timeout }} -verify_ssl: {{ kibana_verify_ssl }} +kibana.defaultAppId: "{{ kibana_default_app_id }}" +server.port: {{ kibana_server_port }} +server.host: "{{ hostvars[inventory_hostname]['ansible_' + api_interface]['ipv4']['address'] }}" +elasticsearch.url: "{{ internal_protocol }}://{{ hostvars[inventory_hostname]['ansible_' + api_interface]['ipv4']['address'] }}:{{ elasticsearch_port }}" +elasticsearch.requestTimeout: {{ kibana_elasticsearch_request_timeout }} +elasticsearch.shardTimeout: {{ kibana_elasticsearch_shard_timeout }} +elasticsearch.ssl.verify: {{ kibana_elasticsearch_ssl_verify }} bundled_plugin_ids: - plugins/dashboard/index - plugins/discover/index diff --git a/ansible/site.yml b/ansible/site.yml index cf1141239d..f84e40b9d5 100644 --- a/ansible/site.yml +++ b/ansible/site.yml @@ -30,13 +30,13 @@ roles: - { role: kibana, tags: kibana, - when: enable_elk | bool } + when: enable_central_logging | bool } - hosts: elasticsearch roles: - { role: elasticsearch, tags: elasticsearch, - when: enable_elk | bool } + when: enable_central_logging | bool } - hosts: memcached roles: diff --git a/docker/heka/plugins/decoders/os_swift_log.lua b/docker/heka/plugins/decoders/os_swift_log.lua index 440e9ab08b..0296f6e23d 100644 --- a/docker/heka/plugins/decoders/os_swift_log.lua +++ b/docker/heka/plugins/decoders/os_swift_log.lua @@ -20,7 +20,7 @@ local utils = require "os_utils" local msg = { Timestamp = nil, - Type = 'Syslog', + Type = 'log', Hostname = read_config("hostname"), Payload = nil, Pid = nil, diff --git a/docker/heka/plugins/decoders/os_syslog.lua b/docker/heka/plugins/decoders/os_syslog.lua index 34fdd0bcbd..94e2744f8b 100644 --- a/docker/heka/plugins/decoders/os_syslog.lua +++ b/docker/heka/plugins/decoders/os_syslog.lua @@ -20,7 +20,7 @@ local utils = require "os_utils" local msg = { Timestamp = nil, - Type = 'Syslog', + Type = 'log', Hostname = read_config("hostname"), Payload = nil, Pid = nil, diff --git a/tools/cleanup-containers b/tools/cleanup-containers index b5821d43e7..a860c6780e 100755 --- a/tools/cleanup-containers +++ b/tools/cleanup-containers @@ -33,7 +33,9 @@ else openvswitch_{vswitchd,db} \ rabbitmq{,_bootstrap} \ heka \ - swift_{account_{auditor,reaper,replicator,server},container_{auditor,replicator,server,updater},object_{auditor,expirer,replicator,server,updater},proxy_server,rsyncd} + swift_{account_{auditor,reaper,replicator,server},container_{auditor,replicator,server,updater},object_{auditor,expirer,replicator,server,updater},proxy_server,rsyncd} \ + elasticsearch \ + kibana ) ceph_osd_bootstrap=$(docker ps -a --filter "name=bootstrap_osd_*" --format "{{.Names}}") ceph_osd_containers=$(docker ps -a --filter "name=ceph_osd_*" --format "{{.Names}}") @@ -51,7 +53,8 @@ else mongodb \ haproxy_socket \ heka{,_socket} \ - kolla_logs + kolla_logs \ + elasticsearch ) fi