Kibana visuals and dashboards for OpenStack

Commit to import Browbeat Kibana Visuals into a existing Kibana
infrastructure.

This commit will ship with two dashboards and mutliple visuals.

Fix for the ElasticSearch playbook install. The ElasticSearch playbook
was missing bits to listen on external.

Change-Id: I209af58ba81b1caf613ea669b3a9ce8f5530c40e

ansible/README.rst

@@ -73,6 +73,23 @@ file to ensure the correct parameters are passed.
 To install collectd on everything other than Openstack machines, view
 the `README for collectd-generic <README.collectd-generic.md>`__.
 
+Install Kibana Visuals
+''''''''''''''''''''''
+
+Prior to installing the Kibana visuals, please review install/group\_vars/all.yml
+file to ensure the correct parameters are passed.
+
+::
+    browbeat_path - Point to the browbeat directory, default is /home/stack/browbeat
+    es_ip -  Point to the ElasticSerach host, default is blank
+    es_kibana_index - Point to the correct Kibana index, default is .kibana
+
+To Install Kibana Visuals
+
+::
+
+    # ansible-playbook -i hosts install/kibana-visuals.yml
+
 Install Connmon
 '''''''''''''''
 

ansible/install/group_vars/all.yml

@@ -173,8 +173,10 @@ fluentd_debug_port: 24230
 # due to utilizing elasticsearch connectors, general
 # usage may want to disable this option due to security reasons
 # in which case you should set this to false
+es_ip:
 es_local_port: 9200
 es_listen_external: true
+es_kibana_index: .kibana
 ### install curator tool ###
 # curator is the recommended tool for managing elasticsearch indexes
 # https://www.elastic.co/guide/en/elasticsearch/client/curator/current/index.html

ansible/install/kibana-visuals.yml

@@ -0,0 +1,8 @@
+---
+#
+#
+
+- hosts: localhost
+  remote_user: "{{ local_remote_user }}"
+  roles:
+    - kibana-visualization

ansible/install/roles/elasticsearch/tasks/main.yml

@@ -110,6 +110,12 @@
 
 ## end firewall rules ##
 
+- name: Copy over ES Config
+  template:
+    src=elasticsearch.yml.j2
+    dest=/etc/elasticsearch/elasticsearch.yml
+  become: true
+
 - name: Start elasticsearch service
   command: systemctl start elasticsearch.service
   ignore_errors: true

ansible/install/roles/elasticsearch/templates/elasticsearch.yml.j2

@@ -0,0 +1,96 @@
+# ======================== Elasticsearch Configuration =========================
+#
+# NOTE: Elasticsearch comes with reasonable defaults for most settings.
+#       Before you set out to tweak and tune the configuration, make sure you
+#       understand what are you trying to accomplish and the consequences.
+#
+# The primary way of configuring a node is via this file. This template lists
+# the most important settings you may want to configure for a production cluster.
+#
+# Please see the documentation for further information on configuration options:
+# <http://www.elastic.co/guide/en/elasticsearch/reference/current/setup-configuration.html>
+#
+# ---------------------------------- Cluster -----------------------------------
+#
+# Use a descriptive name for your cluster:
+#
+# cluster.name: my-application
+#
+# ------------------------------------ Node ------------------------------------
+#
+# Use a descriptive name for the node:
+#
+# node.name: node-1
+#
+# Add custom attributes to the node:
+#
+# node.rack: r1
+#
+# ----------------------------------- Paths ------------------------------------
+#
+# Path to directory where to store the data (separate multiple locations by comma):
+#
+# path.data: /path/to/data
+#
+# Path to log files:
+#
+# path.logs: /path/to/logs
+#
+# ----------------------------------- Memory -----------------------------------
+#
+# Lock the memory on startup:
+#
+# bootstrap.memory_lock: true
+#
+# Make sure that the `ES_HEAP_SIZE` environment variable is set to about half the memory
+# available on the system and that the owner of the process is allowed to use this limit.
+#
+# Elasticsearch performs poorly when the system is swapping the memory.
+#
+# ---------------------------------- Network -----------------------------------
+#
+# Set the bind address to a specific IP (IPv4 or IPv6):
+#
+network.host: 0.0.0.0
+network.publish_host: {{es_ip}}
+http.port: {{es_local_port}}
+#
+# Set a custom port for HTTP:
+#
+#http.port: 9200
+#
+# For more information, see the documentation at:
+# <http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-network.html>
+#
+# --------------------------------- Discovery ----------------------------------
+#
+# Pass an initial list of hosts to perform discovery when new node is started:
+# The default list of hosts is ["127.0.0.1", "[::1]"]
+#
+# discovery.zen.ping.unicast.hosts: ["host1", "host2"]
+#
+# Prevent the "split brain" by configuring the majority of nodes (total number of nodes / 2 + 1):
+#
+# discovery.zen.minimum_master_nodes: 3
+#
+# For more information, see the documentation at:
+# <http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-discovery.html>
+#
+# ---------------------------------- Gateway -----------------------------------
+#
+# Block initial recovery after a full cluster restart until N nodes are started:
+#
+# gateway.recover_after_nodes: 3
+#
+# For more information, see the documentation at:
+# <http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-gateway.html>
+#
+# ---------------------------------- Various -----------------------------------
+#
+# Disable starting multiple nodes on a single system:
+#
+# node.max_local_storage_nodes: 1
+#
+# Require explicit names when deleting indices:
+#
+# action.destructive_requires_name: true

ansible/install/roles/kibana-visualization/tasks/main.yml

@@ -0,0 +1,31 @@
+---
+
+- name: Upload Searches
+  uri:
+    url: http://{{ es_ip }}:{{ es_local_port }}/{{ es_kibana_index }}/search/{{ item | basename | regex_replace('\.json','') }}
+    method: PUT
+    body: "{{ lookup('file', item) }}"
+    body_format: json
+  with_fileglob:
+  - "{{ browbeat_path }}/visualization/*/search/*"
+  ignore_errors: true
+
+- name: Upload Visuals
+  uri:
+    url: http://{{ es_ip }}:{{ es_local_port }}/{{ es_kibana_index }}/visualization/{{ item | basename | regex_replace('\.json','')}}
+    method: PUT
+    body: "{{ lookup('file', item) }}"
+    body_format: json
+  with_fileglob:
+  - "{{ browbeat_path }}/visualization/*/visualization/*"
+  ignore_errors: true
+
+- name: Upload Dashboards
+  uri:
+    url: http://{{ es_ip }}:{{ es_local_port }}/{{ es_kibana_index }}/dashboard/{{ item | basename | regex_replace('\.json','')}}
+    method: PUT
+    body: "{{ lookup('file', item) }}"
+    body_format: json
+  with_fileglob:
+  - "{{ browbeat_path }}/visualization/*/dashboard/*"
+  ignore_errors: true

visualization/Neutron/README.rst

@@ -0,0 +1,3 @@
+# OpenStack Browbeat Neutron Kibana page
+------------------------------------------
+

visualization/Neutron/dashboard/NeutronAPIDashboard.json

@@ -0,0 +1,13 @@
+{
+    "title": "Neutron API Dashboard",
+    "hits": 0,
+    "description": "",
+    "panelsJSON": "[{\"col\":10,\"id\":\"NeutronErrors\",\"panelIndex\":2,\"row\":1,\"size_x\":3,\"size_y\":2,\"type\":\"visualization\"},{\"col\":10,\"id\":\"NeutronResults\",\"panelIndex\":3,\"row\":3,\"size_x\":3,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"NeutronAPITimings\",\"panelIndex\":4,\"row\":1,\"size_x\":9,\"size_y\":7,\"type\":\"visualization\"},{\"col\":10,\"id\":\"NeutronNumberOfAPIWorkers\",\"panelIndex\":5,\"row\":6,\"size_x\":3,\"size_y\":2,\"type\":\"visualization\"}]",
+    "optionsJSON": "{\"darkTheme\":true}",
+    "uiStateJSON": "{\"P-5\":{\"vis\":{\"legendOpen\":true}}}",
+    "version": 1,
+    "timeRestore": false,
+    "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}}}]}"
+    }
+}

visualization/Neutron/visualization/NeutronAPITimings.json

@@ -0,0 +1,10 @@
+{
+    "title": "Neutron API - Top 10 - Average Timings",
+    "visState": "{\"title\":\"Neutron API - Average Timings\",\"type\":\"histogram\",\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"defaultYExtents\":false,\"mode\":\"grouped\",\"scale\":\"linear\",\"setYExtents\":false,\"shareYAxis\":true,\"times\":[],\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"type\":\"avg\",\"schema\":\"metric\",\"params\":{\"field\":\"raw\"}},{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"action\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\"}},{\"id\":\"4\",\"type\":\"terms\",\"schema\":\"split\",\"params\":{\"field\":\"software-metadata.software_details.openstack.config.neutron.openstack_neutron_api_workers\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\",\"row\":true}}],\"listeners\":{}}",
+    "uiStateJSON": "{}",
+    "description": "",
+    "version": 1,
+    "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"index\":\"[browbeat-rally-]YYYY.MM.DD\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
+    }
+}

visualization/Neutron/visualization/NeutronErrors.json

@@ -0,0 +1,10 @@
+{
+    "title": "Neutron errors: Top 10 Errors",
+    "visState": "{\"title\":\"Neutron errors: Top 10 Errors\",\"type\":\"pie\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"isDonut\":false},\"aggs\":[{\"id\":\"1\",\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"error_type\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}",
+    "uiStateJSON": "{}",
+    "description": "",
+    "version": 1,
+    "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"index\":\"[browbeat-rally-]YYYY.MM.DD\",\"query\":{\"query_string\":{\"query\":\"_type: error AND action_name: neutron*\",\"analyze_wildcard\":true}},\"filter\":[]}"
+    }
+}

visualization/Neutron/visualization/NeutronNumberOfAPIWorkers.json

@@ -0,0 +1,10 @@
+{
+    "title": "Neutron Results: Number of api workers",
+    "visState": "{\"title\":\"Neutron Results: Number of api workers\",\"type\":\"pie\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"isDonut\":false},\"aggs\":[{\"id\":\"1\",\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"software-metadata.software_details.openstack.config.neutron.openstack_neutron_api_workers\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}",
+    "uiStateJSON": "{}",
+    "description": "",
+    "version": 1,
+    "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"index\":\"[browbeat-rally-]YYYY.MM.DD\",\"query\":{\"query_string\":{\"query\":\"_type: result\",\"analyze_wildcard\":true}},\"filter\":[]}"
+    }
+}

visualization/Neutron/visualization/NeutronResults.json

@@ -0,0 +1,10 @@
+{
+    "title": "Neutron results: Top 10 results",
+    "visState": "{\"title\":\"Neutron errors: Top 10 Errors\",\"type\":\"pie\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"isDonut\":false},\"aggs\":[{\"id\":\"1\",\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"action\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}",
+    "uiStateJSON": "{}",
+    "description": "",
+    "version": 1,
+    "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"index\":\"[browbeat-rally-]YYYY.MM.DD\",\"query\":{\"query_string\":{\"query\":\"_type: result\",\"analyze_wildcard\":true}},\"filter\":[]}"
+    }
+}

visualization/OpenStack-Workers/README.rst

@@ -0,0 +1,2 @@
+# OpenStack Browbeat Worker Kibana page
+----------------------------------------

visualization/OpenStack-Workers/dashboard/OpenStack-Worker.json

@@ -0,0 +1,12 @@
+{
+    "title": "OpenStack - Worker Dashboard",
+    "hits": 0,
+    "description": "",
+    "panelsJSON": "[{\"col\":1,\"id\":\"NovaAPI\",\"row\":1,\"size_x\":12,\"size_y\":5,\"type\":\"visualization\"},{\"col\":1,\"id\":\"NeutronAPI\",\"row\":2,\"size_x\":12,\"size_y\":5,\"type\":\"visualization\"},{\"col\":1,\"id\":\"NeutronTimes\",\"row\":3,\"size_x\":12,\"size_y\":5,\"type\":\"visualization\"},{\"col\":1,\"id\":\"KeystoneAPI\",\"row\":4,\"size_x\":12,\"size_y\":5,\"type\":\"visualization\"}]",
+    "optionsJSON": "{\"darkTheme\":true}",
+    "version": 1,
+    "timeRestore": false,
+    "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}}}]}"
+    }
+}

visualization/OpenStack-Workers/visualization/KeystoneAPI.json

@@ -0,0 +1,9 @@
+{
+    "title": "OpenStack Performance: Keystone API workers",
+    "visState": "{\"type\":\"histogram\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":false,\"scale\":\"linear\",\"mode\":\"grouped\",\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"type\":\"percentiles\",\"schema\":\"metric\",\"params\":{\"field\":\"raw\",\"percents\":[99]}},{\"id\": \"3\",\"type\": \"terms\",\"schema\": \"group\",\"params\": {\"field\": \"software-metadata.software_details.openstack.config.keystone.openstack_keystone_admin_workers_processes\",\"size\": 20,\"order\": \"desc\",\"orderBy\": \"1.99\"}},{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"action\",\"size\":16,\"order\":\"desc\",\"orderBy\":\"1.99\"}},{\"id\":\"4\",\"type\":\"filters\",\"schema\":\"split\",\"params\":{\"filters\":[{\"input\":{\"query\":{\"query_string\":{\"query\":\"action: authenticate.*\",\"analyze_wildcard\":true}}},\"label\":\"\"}],\"row\":true}}],\"listeners\":{}}",
+    "description": "",
+    "version": 1,
+    "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"index\":\"[browbeat-rally-]YYYY.MM.DD\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
+    }
+}

visualization/OpenStack-Workers/visualization/NeutronAPI.json

@@ -0,0 +1,9 @@
+{
+    "title": "OpenStack Performance: Neutron API workers",
+    "visState": "{\"type\":\"histogram\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"scale\":\"linear\",\"mode\":\"grouped\",\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"type\":\"percentiles\",\"schema\":\"metric\",\"params\":{\"field\":\"raw\",\"percents\":[99]}},{\"id\": \"3\",\"type\": \"terms\",\"schema\": \"group\",\"params\": {\"field\": \"software-metadata.software_details.openstack.config.neutron.openstack_neutron_api_workers\",\"size\": 20,\"order\": \"desc\",\"orderBy\": \"1.99\"}},{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"action\",\"size\":16,\"order\":\"desc\",\"orderBy\":\"1.99\"}},{\"id\":\"4\",\"type\":\"filters\",\"schema\":\"split\",\"params\":{\"filters\":[{\"input\":{\"query\":{\"query_string\":{\"query\":\"action: neutron.*\",\"analyze_wildcard\":true}}},\"label\":\"\"}],\"row\":true}}],\"listeners\":{}}",
+    "description": "",
+    "version": 1,
+    "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"index\":\"[browbeat-rally-]YYYY.MM.DD\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
+    }
+}

visualization/OpenStack-Workers/visualization/NeutronTimes.json

@@ -0,0 +1,11 @@
+{
+    "title": "OpenStack Performance: Neutron Times",
+      "visState": "{\"title\":\"OpenStack Performance: Neutron API Response Times @ Differnet loads\",\"type\":\"histogram\",\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"defaultYExtents\":false,\"mode\":\"grouped\",\"scale\":\"linear\",\"setYExtents\":false,\"shareYAxis\":true,\"times\":[],\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"type\":\"percentiles\",\"schema\":\"metric\",\"params\":{\"field\":\"raw\",\"percents\":[99]}},{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"action\",\"size\":20,\"order\":\"desc\",\"orderBy\":\"1.99\"}},{\"id\":\"4\",\"type\":\"terms\",\"schema\":\"split\",\"params\":{\"field\":\"software-metadata.software_details.openstack.config.neutron.openstack_neutron_api_workers\",\"size\":20,\"order\":\"desc\",\"orderBy\":\"1.99\",\"row\":false}},{\"id\":\"3\",\"type\":\"histogram\",\"schema\":\"group\",\"params\":{\"field\":\"rally_setup.kw.runner.times\",\"interval\":20,\"min_doc_count\":false,\"extended_bounds\":{}}}],\"listeners\":{}}",
+    "description": "",
+    "version": 1,
+    "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"index\":\"[browbeat-rally-]YYYY.MM.DD\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"action: neutron.*\"}},\"filter\":[]}"
+    }
+}
+
+

visualization/OpenStack-Workers/visualization/NovaAPI.json

@@ -0,0 +1,9 @@
+{
+    "title": "OpenStack Performance: Nova API workers",
+    "visState": "{\"type\":\"histogram\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":false,\"scale\":\"linear\",\"mode\":\"grouped\",\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"type\":\"percentiles\",\"schema\":\"metric\",\"params\":{\"field\":\"raw\",\"percents\":[99]}},{\"id\": \"3\",\"type\": \"terms\",\"schema\": \"group\",\"params\": {\"field\": \"software-metadata.software_details.openstack.config.nova.openstack_nova_api_workers\",\"size\": 20,\"order\": \"desc\",\"orderBy\": \"1.99\"}},{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"action\",\"size\":16,\"order\":\"desc\",\"orderBy\":\"1.99\"}},{\"id\":\"4\",\"type\":\"filters\",\"schema\":\"split\",\"params\":{\"filters\":[{\"input\":{\"query\":{\"query_string\":{\"query\":\"action: nova.*\",\"analyze_wildcard\":true}}},\"label\":\"\"}],\"row\":true}}],\"listeners\":{}}",
+    "description": "",
+    "version": 1,
+    "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"index\":\"[browbeat-rally-]YYYY.MM.DD\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
+    }
+}

visualization/README.rst

@@ -0,0 +1,18 @@
+# OpenStack Browbeat Kibna visualization
+-----------------------------------------
+To import these Visualizations, you need Kibana 4.1 or greater.
+
+## How to install?
+-------------------
+Use the provided ansible playbook in ansible/install in order to install the Kibana Viz.
+
+## Assumptions
+--------------
+This work assumes you are using :
+
+- Browbeat to run Rally workloads
+- Browbeat Gather scripts for Metadata
+
+Without the two pieces of information above, your milage may vary.
+
+Also, this work assumes you are using the default browbeat-rally-YYYY.MM.DD ElasticSearch index. If that is not the case, update the jsons.