From 491aff0b88bc3dac13ea9180c952854fd0a67684 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=89ric=20Lemoine?= <elemoine@mirantis.com>
Date: Wed, 24 Feb 2016 16:05:36 +0100
Subject: [PATCH] Make Heka send logs to Elasticsearch

This patch includes changes relative to integrating Heka with
Elasticsearch and Kibana.

The main change is the addition of an Heka ElasticSearchOutput plugin
to make Heka send the logs it collects to Elasticsearch.

Since Logstash is not used the enable_elk deploy variable is renamed
to enable_central_logging.

If enable_central_logging is false then Elasticsearch and Kibana are
not started, and Heka won't attempt to send logs to Elasticsearch.

By default enable_central_logging is set to false. If
enable_central_logging is set to true after deployment then the Heka
container needs to be recreated (for Heka to get the new
configuration).

The Kibana configuration used property names that are deprecated in
Kibana 4.2. This is changed to use non-deprecated property names.

Previously logs read from files and from Syslog had a different Type
in Heka. This is changed to always use "log" for the Type. In this
way just one index instead of two is used in Elasticsearch, making
things easier to the user on the visualization side.

The HAProxy configuration is changed to add entries for Kibana.
Kibana server is now accessible via the internal VIP, and also via
the external VIP if there's one configured.

The HAProxy configuration is changed to add an entry for
Elasticsearch. So Elasticsearch is now accessible via the internal
VIP. Heka uses that channel for communicating with Elasticsearch.

Note that currently the Heka logs include "Plugin
elasticsearch_output" errors when Heka starts. This occurs when Heka
starts processing logs while Elasticsearch is not yet started. These
are transient errors that go away when Elasticsearch is ready. And
with buffering enabled on the ElasticSearchOuput plugin logs will be
buffered and then retransmitted when Elasticsearch is ready.

Change-Id: I6ff7a4f0ad04c4c666e174693a35ff49914280bb
Implements: blueprint central-logging-service
---
 ansible/group_vars/all.yml                    |  4 ++--
 ansible/roles/common/tasks/config.yml         |  8 +++++++
 .../templates/heka-elasticsearch.toml.j2      | 16 +++++++++++++
 .../common/templates/heka-haproxy.toml.j2     |  4 ++--
 .../roles/common/templates/heka-swift.toml.j2 |  2 +-
 ansible/roles/common/templates/heka.json.j2   |  7 ++++++
 ansible/roles/elasticsearch/defaults/main.yml |  7 ++++--
 .../templates/elasticsearch.yml.j2            | 15 +++++++++++-
 .../roles/haproxy/templates/haproxy.cfg.j2    | 23 +++++++++++++++++++
 ansible/roles/kibana/defaults/main.yml        | 16 ++++---------
 ansible/roles/kibana/templates/kibana.yml.j2  | 15 ++++++------
 ansible/site.yml                              |  4 ++--
 docker/heka/plugins/decoders/os_swift_log.lua |  2 +-
 docker/heka/plugins/decoders/os_syslog.lua    |  2 +-
 tools/cleanup-containers                      |  7 ++++--
 15 files changed, 98 insertions(+), 34 deletions(-)
 create mode 100644 ansible/roles/common/templates/heka-elasticsearch.toml.j2

diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml
index eca72ff3cd..cd1c292053 100644
--- a/ansible/group_vars/all.yml
+++ b/ansible/group_vars/all.yml
@@ -130,7 +130,7 @@ rgw_port: "6780"
 
 mistral_api_port: "8989"
 
-kibana_port: "5601"
+kibana_server_port: "5601"
 
 elasticsearch_port: "9200"
 
@@ -190,7 +190,7 @@ enable_murano: "no"
 enable_ironic: "no"
 enable_magnum: "no"
 enable_mistral: "no"
-enable_elk: "no"
+enable_central_logging: "no"
 enable_mongodb: "no"
 
 ironic_keystone_user: "ironic"
diff --git a/ansible/roles/common/tasks/config.yml b/ansible/roles/common/tasks/config.yml
index 18f87c610a..8592801ab1 100644
--- a/ansible/roles/common/tasks/config.yml
+++ b/ansible/roles/common/tasks/config.yml
@@ -46,3 +46,11 @@
     - "swift-object-updater"
     - "swift-proxy-server"
     - "swift-rsyncd"
+
+- name: Copying over heka elasticsearch config file
+  template:
+    src: "heka-{{ item }}.toml.j2"
+    dest: "{{ node_config_directory }}/heka/heka-{{ item }}.toml"
+  with_items:
+    - "elasticsearch"
+  when: "{{ enable_central_logging | bool }}"
diff --git a/ansible/roles/common/templates/heka-elasticsearch.toml.j2 b/ansible/roles/common/templates/heka-elasticsearch.toml.j2
new file mode 100644
index 0000000000..7b9a351cb5
--- /dev/null
+++ b/ansible/roles/common/templates/heka-elasticsearch.toml.j2
@@ -0,0 +1,16 @@
+[elasticsearch_json_encoder]
+type = "ESJsonEncoder"
+index = {{'"%{Type}-%{%Y.%m.%d}"'}}
+es_index_from_timestamp = true
+fields = ["Timestamp", "Type", "Logger", "Severity", "Payload", "Pid", "Hostname", "DynamicFields"]
+
+[elasticsearch_output]
+type = "ElasticSearchOutput"
+server = "{{ internal_protocol }}://{{ kolla_internal_vip_address }}:{{ elasticsearch_port }}"
+message_matcher = "Type == 'log'"
+encoder = "elasticsearch_json_encoder"
+use_buffering = true
+    [elasticsearch_output.buffering]
+    max_buffer_size = 1073741824  # 1024 * 1024 * 1024
+    max_file_size = 134217728  # 128 * 1024 * 1024
+    full_action = "drop"
diff --git a/ansible/roles/common/templates/heka-haproxy.toml.j2 b/ansible/roles/common/templates/heka-haproxy.toml.j2
index d933a6f554..f30dd32d57 100644
--- a/ansible/roles/common/templates/heka-haproxy.toml.j2
+++ b/ansible/roles/common/templates/heka-haproxy.toml.j2
@@ -3,12 +3,12 @@
 
 [haproxy_file_output]
 type = "FileOutput"
-message_matcher = "Type == 'Syslog' && Fields[programname] =~ /(?i:haproxy)/"
+message_matcher = "Fields[programname] =~ /(?i:haproxy)/"
 path = "/var/log/kolla/haproxy/haproxy.log"
 encoder = "syslog_encoder"
 
 [keepalived_file_output]
 type = "FileOutput"
-message_matcher = "Type == 'Syslog' && Fields[programname] =~ /(?i:keepalived)/"
+message_matcher = "Fields[programname] =~ /(?i:keepalived)/"
 path = "/var/log/kolla/haproxy/keepalived.log"
 encoder = "syslog_encoder"
diff --git a/ansible/roles/common/templates/heka-swift.toml.j2 b/ansible/roles/common/templates/heka-swift.toml.j2
index 3bb520e322..8ed4a1e236 100644
--- a/ansible/roles/common/templates/heka-swift.toml.j2
+++ b/ansible/roles/common/templates/heka-swift.toml.j2
@@ -3,6 +3,6 @@
 
 [{{ item }}_file_output]
 type = "FileOutput"
-message_matcher = "Type == 'Syslog' && Fields[programname] == '{{ item }}'"
+message_matcher = "Fields[programname] == '{{ item }}'"
 path = "/var/log/kolla/swift/{{ item }}.log"
 encoder = "syslog_encoder"
diff --git a/ansible/roles/common/templates/heka.json.j2 b/ansible/roles/common/templates/heka.json.j2
index f5bcad2979..332790231e 100644
--- a/ansible/roles/common/templates/heka.json.j2
+++ b/ansible/roles/common/templates/heka.json.j2
@@ -2,6 +2,13 @@
 {
     "command": "/usr/bin/hekad -config=/etc/heka/",
     "config_files": [
+        {
+            "source": "{{ container_config_directory }}/heka-elasticsearch.toml",
+            "dest": "/etc/heka/heka-elasticsearch.toml",
+            "owner": "heka",
+            "perm": "0600",
+            "optional": "True"
+        },
         {
             "source": "{{ container_config_directory }}/heka-global.toml",
             "dest": "/etc/heka/heka-global.toml",
diff --git a/ansible/roles/elasticsearch/defaults/main.yml b/ansible/roles/elasticsearch/defaults/main.yml
index c7c5cc986b..947a1cb9f1 100644
--- a/ansible/roles/elasticsearch/defaults/main.yml
+++ b/ansible/roles/elasticsearch/defaults/main.yml
@@ -2,8 +2,11 @@
 ####################
 # Elasticsearch
 ####################
-elasticsearch_port: "{{ elasticsearch_port }}"
-elasticsearch_host: "{{ kolla_internal_vip_address }}"
+elasticsearch_cluster_name: "kolla_logging"
+
+####################
+# Docker
+####################
 elasticsearch_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ kolla_install_type }}-elasticsearch"
 elasticsearch_tag: "{{ openstack_release }}"
 elasticsearch_image_full: "{{ elasticsearch_image }}:{{ elasticsearch_tag }}"
diff --git a/ansible/roles/elasticsearch/templates/elasticsearch.yml.j2 b/ansible/roles/elasticsearch/templates/elasticsearch.yml.j2
index eba662b63d..449fb3c8af 100644
--- a/ansible/roles/elasticsearch/templates/elasticsearch.yml.j2
+++ b/ansible/roles/elasticsearch/templates/elasticsearch.yml.j2
@@ -1,4 +1,17 @@
-network.host: {{ hostvars[inventory_hostname]['ansible_' + api_interface]['ipv4']['address'] }}
+{% set num_nodes = groups['elasticsearch'] | length %}
+{% set minimum_master_nodes = (num_nodes / 2 + 1) | round(0, 'floor') | int if num_nodes > 2 else 1 %}
+{% set recover_after_nodes = (num_nodes * 2 / 3) | round(0, 'floor') | int if num_nodes > 1 else 1 %}
+node.name: "{{ hostvars[inventory_hostname]['ansible_' + api_interface]['ipv4']['address'] }}"
+network.host: "{{ hostvars[inventory_hostname]['ansible_' + api_interface]['ipv4']['address'] }}"
+cluster.name: "{{ elasticsearch_cluster_name }}"
+node.master: true
+node.data: true
+discovery.zen.ping.unicast.hosts: [{% for host in groups['elasticsearch'] %}"{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}"{% if not loop.last %},{% endif %}{% endfor %}]
+
+discovery.zen.minimum_master_nodes: {{ minimum_master_nodes }}
+gateway.expected_nodes: {{ num_nodes }}
+gateway.recover_after_time: "5m"
+gateway.recover_after_nodes: {{ recover_after_nodes }}
 path.conf: "/etc/elasticsearch"
 path.data: "/var/lib/elasticsearch/data"
 path.logs: "/var/log/elasticsearch"
diff --git a/ansible/roles/haproxy/templates/haproxy.cfg.j2 b/ansible/roles/haproxy/templates/haproxy.cfg.j2
index 1ab7dd67db..f3b76253d4 100644
--- a/ansible/roles/haproxy/templates/haproxy.cfg.j2
+++ b/ansible/roles/haproxy/templates/haproxy.cfg.j2
@@ -328,3 +328,26 @@ listen radosgw_external
 {% endfor %}
 {% endif %}
 {% endif %}
+
+{% if enable_central_logging | bool %}
+listen kibana
+  bind {{ kolla_internal_vip_address }}:{{ kibana_server_port }}
+{% for host in groups['kibana'] %}
+  server {{ hostvars[host]['ansible_hostname'] }} {{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ kibana_server_port }} check inter 2000 rise 2 fall 5
+{% endfor %}
+{% if haproxy_enable_external_vip | bool %}
+
+listen kibana_external
+  bind {{ kolla_external_vip_address }}:{{ kibana_server_port }}
+{% for host in groups['kibana'] %}
+  server {{ hostvars[host]['ansible_hostname'] }} {{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ kibana_server_port }} check inter 2000 rise 2 fall 5
+{% endfor %}
+{% endif %}
+
+listen elasticsearch
+  option dontlog-normal
+  bind {{ kolla_internal_vip_address }}:{{ elasticsearch_port }}
+{% for host in groups['elasticsearch'] %}
+  server {{ hostvars[host]['ansible_hostname'] }} {{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ elasticsearch_port }} check inter 2000 rise 2 fall 5
+{% endfor %}
+{% endif %}
diff --git a/ansible/roles/kibana/defaults/main.yml b/ansible/roles/kibana/defaults/main.yml
index c720b03069..e35e2f1e71 100644
--- a/ansible/roles/kibana/defaults/main.yml
+++ b/ansible/roles/kibana/defaults/main.yml
@@ -2,12 +2,10 @@
 ####################
 # Kibana
 ####################
-kibana_port: "{{ kibana_port }}"
-kibana_host: "{{ kolla_internal_vip_address }}"
-kibana_app_id: "discover"
-kibana_request_timeout: 300000
-kibana_shard_timeout: 0
-kibana_verify_ssl: false
+kibana_default_app_id: "discover"
+kibana_elasticsearch_request_timeout: 300000
+kibana_elasticsearch_shard_timeout: 0
+kibana_elasticsearch_ssl_verify: false
 
 
 ####################
@@ -16,9 +14,3 @@ kibana_verify_ssl: false
 kibana_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ kolla_install_type }}-kibana"
 kibana_tag: "{{ openstack_release }}"
 kibana_image_full: "{{ kibana_image }}:{{ kibana_tag }}"
-
-
-####################
-# Elasticsearch
-####################
-elasticsearch_preserve_host: "true"
diff --git a/ansible/roles/kibana/templates/kibana.yml.j2 b/ansible/roles/kibana/templates/kibana.yml.j2
index 2fa66eec93..cc31234c9d 100644
--- a/ansible/roles/kibana/templates/kibana.yml.j2
+++ b/ansible/roles/kibana/templates/kibana.yml.j2
@@ -1,11 +1,10 @@
-port: {{ kibana_port }}
-host: {{ kibana_host }}
-elasticsearch_url: "{{ internal_protocol }}://{{ kolla_internal_fqdn }}:{{ elasticsearch_port }}"
-elasticsearch_preserve_host: {{ elasticsearch_preserve_host }}
-default_app_id: {{ kibana_app_id }}
-request_timeout: {{ kibana_request_timeout }}
-shard_timeout: {{ kibana_shard_timeout }}
-verify_ssl: {{ kibana_verify_ssl }}
+kibana.defaultAppId: "{{ kibana_default_app_id }}"
+server.port: {{ kibana_server_port }}
+server.host: "{{ hostvars[inventory_hostname]['ansible_' + api_interface]['ipv4']['address'] }}"
+elasticsearch.url: "{{ internal_protocol }}://{{ hostvars[inventory_hostname]['ansible_' + api_interface]['ipv4']['address'] }}:{{ elasticsearch_port }}"
+elasticsearch.requestTimeout: {{ kibana_elasticsearch_request_timeout }}
+elasticsearch.shardTimeout: {{ kibana_elasticsearch_shard_timeout }}
+elasticsearch.ssl.verify: {{ kibana_elasticsearch_ssl_verify }}
 bundled_plugin_ids:
  - plugins/dashboard/index
  - plugins/discover/index
diff --git a/ansible/site.yml b/ansible/site.yml
index ded4f6ef8b..39d469c3fc 100644
--- a/ansible/site.yml
+++ b/ansible/site.yml
@@ -30,13 +30,13 @@
   roles:
     - { role: kibana,
         tags: kibana,
-        when: enable_elk | bool }
+        when: enable_central_logging | bool }
 
 - hosts: elasticsearch
   roles:
     - { role: elasticsearch,
         tags: elasticsearch,
-        when: enable_elk | bool }
+        when: enable_central_logging | bool }
 
 - hosts: memcached
   roles:
diff --git a/docker/heka/plugins/decoders/os_swift_log.lua b/docker/heka/plugins/decoders/os_swift_log.lua
index 440e9ab08b..0296f6e23d 100644
--- a/docker/heka/plugins/decoders/os_swift_log.lua
+++ b/docker/heka/plugins/decoders/os_swift_log.lua
@@ -20,7 +20,7 @@ local utils = require "os_utils"
 
 local msg = {
     Timestamp   = nil,
-    Type        = 'Syslog',
+    Type        = 'log',
     Hostname    = read_config("hostname"),
     Payload     = nil,
     Pid         = nil,
diff --git a/docker/heka/plugins/decoders/os_syslog.lua b/docker/heka/plugins/decoders/os_syslog.lua
index 34fdd0bcbd..94e2744f8b 100644
--- a/docker/heka/plugins/decoders/os_syslog.lua
+++ b/docker/heka/plugins/decoders/os_syslog.lua
@@ -20,7 +20,7 @@ local utils = require "os_utils"
 
 local msg = {
     Timestamp   = nil,
-    Type        = 'Syslog',
+    Type        = 'log',
     Hostname    = read_config("hostname"),
     Payload     = nil,
     Pid         = nil,
diff --git a/tools/cleanup-containers b/tools/cleanup-containers
index b5821d43e7..a860c6780e 100755
--- a/tools/cleanup-containers
+++ b/tools/cleanup-containers
@@ -33,7 +33,9 @@ else
         openvswitch_{vswitchd,db} \
         rabbitmq{,_bootstrap} \
         heka \
-        swift_{account_{auditor,reaper,replicator,server},container_{auditor,replicator,server,updater},object_{auditor,expirer,replicator,server,updater},proxy_server,rsyncd}
+        swift_{account_{auditor,reaper,replicator,server},container_{auditor,replicator,server,updater},object_{auditor,expirer,replicator,server,updater},proxy_server,rsyncd} \
+        elasticsearch \
+        kibana
     )
     ceph_osd_bootstrap=$(docker ps -a --filter "name=bootstrap_osd_*" --format "{{.Names}}")
     ceph_osd_containers=$(docker ps -a --filter "name=ceph_osd_*" --format "{{.Names}}")
@@ -51,7 +53,8 @@ else
         mongodb \
         haproxy_socket \
         heka{,_socket} \
-        kolla_logs
+        kolla_logs \
+        elasticsearch
     )
 fi