Merge pull request #116 from stackhpc/feature/configure_grafana

Support configuring Grafana

.gitignore

@@ -57,6 +57,7 @@ ansible/roles/mrlesmithjr.manage-lvm/
 ansible/roles/MichaelRigart.interfaces/
 ansible/roles/stackhpc.drac/
 ansible/roles/stackhpc.drac-facts/
+ansible/roles/stackhpc.grafana-conf/
 ansible/roles/stackhpc.libvirt-host/
 ansible/roles/stackhpc.libvirt-vm/
 ansible/roles/stackhpc.os-flavors/

ansible/group_vars/all/grafana

@@ -0,0 +1,45 @@
+---
+###############################################################################
+# Grafana configuration.
+
+# Grafana local admin user name. If you are deploying Monasca Grafana this
+# should not conflict with an OpenStack user name.
+grafana_local_admin_user_name: "admin"
+
+# Path to git repo containing Grafana dashboards. Eg.
+# https://github.com/stackhpc/grafana-reference-dashboards.git
+grafana_monitoring_node_dashboard_repo:
+
+# Dashboard repo version. Optional, defaults to 'HEAD'.
+grafana_monitoring_node_dashboard_repo_version:
+
+# Path to which Grafana dashboards will be cloned to a monitoring node
+grafana_monitoring_node_dashboard_repo_checkout_path: "{{ source_checkout_path ~ '/grafana-dashboards' }}"
+
+# The path, relative to the grafana_monitoring_node_dashboard_repo_checkout_path
+# containing the dashboards. Eg. /prometheus/control_plane
+grafana_monitoring_node_dashboard_repo_path:
+
+# The Grafana organisation for the control plane. Note that for Monasca
+# Grafana with domain support the format is:
+# organisation_name@openstack_domain
+grafana_control_plane_organisation: "control_plane"
+
+# A dict of datasources to configure. See the stackhpc.grafana-conf role
+# for all supported datasources. Example:
+#
+# grafana_datasources:
+#   monasca_api:
+#     port: 8082
+#     host: monasca-api
+#   monasca_log_api:
+#     port: 5607
+#     host: monasca-log-api
+#   elasticsearch:
+#     port: 9200
+#     host: monasca-elasticsearch
+#     project_id: "some_id"
+#
+grafana_datasources: {}
+
+###############################################################################

ansible/overcloud-grafana-configure.yml

@@ -0,0 +1,41 @@
+---
+- name: Check whether Grafana is enabled
+  hosts: overcloud
+  tags:
+    - grafana
+  tasks:
+    - name: Create monitoring group with grafana enabled
+      group_by:
+        key: "monitoring_with_grafana_enabled_{{ kolla_enable_grafana | bool }}"
+
+- name: Configure control plane monitoring for Grafana
+  # Only required to run on a single host.
+  hosts: monitoring_with_grafana_enabled_True[0]
+  gather_facts: False
+  tags:
+    - grafana
+  pre_tasks:
+    - name: Set fact for the VIP address
+      set_fact:
+        kolla_internal_vip_address: "{{ internal_net_name | net_vip_address }}"
+      when: kolla_enable_haproxy | bool
+
+    - name: Set fact for the VIP address
+      set_fact:
+        kolla_internal_vip_address: "{{ internal_net_name | net_ip }}"
+      when: not kolla_enable_haproxy | bool
+
+    - name: Include Kolla passwords for Grafana local admin account credentials
+      include_vars: "{{ kayobe_config_path }}/kolla/passwords.yml"
+  roles:
+    - role: stackhpc.grafana-conf
+      grafana_conf_organisation: "{{ grafana_control_plane_organisation }}"
+      grafana_conf_grafana_admin_user: "{{ grafana_local_admin_user_name }}"
+      grafana_conf_grafana_admin_pass: "{{ grafana_admin_password }}"
+      grafana_conf_grafana_dashboard_repo:
+        repo: "{{ grafana_monitoring_node_dashboard_repo }}"
+        version: "{{ grafana_monitoring_node_dashboard_repo_version }}"
+        checkout_path: "{{ grafana_monitoring_node_dashboard_repo_checkout_path }}"
+        relative_path: "{{ grafana_monitoring_node_dashboard_repo_path }}"
+      grafana_conf_grafana_url: "http://{{ kolla_internal_vip_address }}:3000"
+      grafana_conf_grafana_datasources: "{{ grafana_datasources }}"

ansible/roles/kolla-openstack/templates/grafana.ini.j2

@@ -1,5 +1,14 @@
 # {{ ansible_managed }}
 
+[security]
+; If an OpenStack user with the same name as the admin user logs into
+; Grafana it overwrites user data in the Grafana database, breaking
+; the local admin account, and preventing admin API calls to Grafana. To
+; reduce the chance of this happening we rename the local admin user here.
+; Note that this only affects the Monasca fork of Grafana.
+;admin_user = admin
+admin_user = {{ grafana_local_admin_user_name }}
+
 {% if kolla_extra_grafana %}
 #######################
 # Extra configuration

doc/source/deployment.rst

@@ -379,3 +379,4 @@ This will perform the following tasks:
 - Register Ironic Python Agent (IPA) images with glance
 - Register introspection rules with ironic inspector
 - Register a provisioning network and subnet with neutron
+- Configure Grafana organisations, dashboards and datasources

etc/kayobe/grafana.yml

@@ -0,0 +1,47 @@
+---
+###############################################################################
+# Grafana configuration.
+
+# Grafana local admin user name. If you are deploying Monasca Grafana this
+# should not conflict with an OpenStack user name.
+#grafana_local_admin_user_name:
+
+# Path to git repo containing Grafana dashboards. Eg.
+# https://github.com/stackhpc/grafana-reference-dashboards.git
+#grafana_monitoring_node_dashboard_repo:
+
+# Dashboard repo version. Optional, defaults to 'HEAD'.
+#grafana_monitoring_node_dashboard_repo_version:
+
+# Path to which Grafana dashboards will be cloned to a monitoring node
+#grafana_monitoring_node_dashboard_repo_checkout_path:
+
+# The path, relative to the grafana_monitoring_node_dashboard_repo_checkout_path
+# containing the dashboards. Eg. /prometheus/control_plane
+#grafana_monitoring_node_dashboard_repo_path:
+
+# The Grafana organisation for the control plane. Note that for Monasca
+# Grafana with domain support the format is:
+# organisation_name@openstack_domain
+#grafana_control_plane_organisation:
+
+# A dict of datasources to configure. See the stackhpc.grafana-conf role
+# for all supported datasources. Example:
+#
+# grafana_datasources:
+#   monasca_api:
+#     port: 8082
+#     host: monasca-api
+#   monasca_log_api:
+#     port: 5607
+#     host: monasca-log-api
+#   elasticsearch:
+#     port: 9200
+#     host: monasca-elasticsearch
+#     project_id: "some_id"
+#
+#grafana_datasources:
+
+###############################################################################
+# Dummy variable to allow Ansible to accept this file.
+workaround_ansible_issue_8743: yes

kayobe/cli/commands.py

@@ -1087,6 +1087,7 @@ class OvercloudPostConfigure(KayobeAnsibleMixin, VaultMixin, Command):
     * Register ironic inspector introspection rules with the overcloud
       inspector service.
     * Register a provisioning network with glance.
+    * Configure Grafana for control plane.
     """
 
     def take_action(self, parsed_args):
@@ -1094,7 +1095,7 @@ class OvercloudPostConfigure(KayobeAnsibleMixin, VaultMixin, Command):
         playbooks = _build_playbook_list(
             "overcloud-ipa-images", "overcloud-introspection-rules",
             "overcloud-introspection-rules-dell-lldp-workaround",
-            "provision-net")
+            "provision-net", "overcloud-grafana-configure")
         self.run_kayobe_playbooks(parsed_args, playbooks)
 
 

kayobe/tests/unit/cli/test_commands.py

@@ -672,6 +672,30 @@ class TestCase(unittest.TestCase):
         ]
         self.assertEqual(expected_calls, mock_run.call_args_list)
 
+    @mock.patch.object(commands.KayobeAnsibleMixin,
+                       "run_kayobe_playbooks")
+    def test_overcloud_post_configure(self, mock_run):
+        command = commands.OvercloudPostConfigure(TestApp(), [])
+        parser = command.get_parser("test")
+        parsed_args = parser.parse_args([])
+
+        result = command.run(parsed_args)
+        self.assertEqual(0, result)
+
+        expected_calls = [
+            mock.call(
+                mock.ANY,
+                [
+                    'ansible/overcloud-ipa-images.yml',
+                    'ansible/overcloud-introspection-rules.yml',
+                    'ansible/overcloud-introspection-rules-dell-lldp-workaround.yml',  # noqa
+                    'ansible/provision-net.yml',
+                    'ansible/overcloud-grafana-configure.yml'
+                ],
+            ),
+        ]
+        self.assertEqual(expected_calls, mock_run.call_args_list)
+
     @mock.patch.object(commands.KayobeAnsibleMixin,
                        "run_kayobe_playbooks")
     def test_baremetal_compute_inspect(self, mock_run):

requirements.yml

@@ -10,6 +10,7 @@
   name: singleplatform-eng.users
 - src: stackhpc.drac
 - src: stackhpc.drac-facts
+- src: stackhpc.grafana-conf
 - src: stackhpc.libvirt-host
 - src: stackhpc.libvirt-vm
 - src: stackhpc.os-flavors