Add NRPE checks for services

Adds the nrpe-external-master layer and checks for services managed by
this charm.

func-test-pr: https://github.com/openstack-charmers/zaza-openstack-tests/pull/433

Change-Id: I8f45a612cf4e6df7618edcd8930f2bb199fa69fd
Closes-Bug: #1896674

src/config.yaml

@@ -30,3 +30,17 @@ options:
 
       The Open vSwitch ovsdb-server default of 5 seconds may not be sufficient
       depending on type and load of the CMS you want to connect to OVN.
+  nagios_context:
+    default: "juju"
+    type: string
+    description: |
+      A string that will be prepended to instance name to set the host name
+      in nagios. So for instance the hostname would be something like:
+          juju-myservice-0
+      If you're running multiple environments with the same services in them
+      this allows you to differentiate between them.
+  nagios_servicegroups:
+    default: ""
+    type: string
+    description: |
+      Comma separated list of nagios servicegroups for the service checks.

src/layer.yaml

@@ -1,6 +1,7 @@
 includes:
   - layer:openstack
   - layer:leadership
+  - interface:nrpe-external-master
   - interface:ovsdb
   - interface:ovsdb-cluster
   - interface:ovsdb-cms

src/lib/charm/openstack/ovn_central.py

@@ -17,6 +17,7 @@ import os
 import subprocess
 
 import charmhelpers.core as ch_core
+import charmhelpers.contrib.charmsupport.nrpe as nrpe
 import charmhelpers.contrib.network.ovs.ovn as ch_ovn
 import charmhelpers.contrib.network.ovs.ovsdb as ch_ovsdb
 from charmhelpers.contrib.network import ufw as ch_ufw
@@ -65,6 +66,7 @@ class BaseOVNCentralCharm(charms_openstack.charm.OpenStackCharm):
     name = 'ovn-central'
     packages = ['ovn-central']
     services = ['ovn-central']
+    nrpe_check_services = []
     release_pkg = 'ovn-central'
     configuration_class = OVNCentralConfigurationAdapter
     required_relations = [PEER_RELATION, CERT_RELATION]
@@ -471,6 +473,15 @@ class BaseOVNCentralCharm(charms_openstack.charm.OpenStackCharm):
         for rule in sorted(delete_rules, reverse=True):
             ch_ufw.modify_access(None, dst=None, action='delete', index=rule)
 
+    def render_nrpe(self):
+        """Configure Nagios NRPE checks."""
+        hostname = nrpe.get_nagios_hostname()
+        current_unit = nrpe.get_nagios_unit_name()
+        charm_nrpe = nrpe.NRPE(hostname=hostname)
+        nrpe.add_init_service_checks(
+            charm_nrpe, self.nrpe_check_services, current_unit)
+        charm_nrpe.write()
+
 
 class TrainOVNCentralCharm(BaseOVNCentralCharm):
     # OpenvSwitch and OVN is distributed as part of the Ubuntu Cloud Archive
@@ -507,6 +518,11 @@ class TrainOVNCentralCharm(BaseOVNCentralCharm):
             '/lib/systemd/system/ovn-nb-ovsdb.service': [],
             '/lib/systemd/system/ovn-sb-ovsdb.service': [],
         })
+        self.nrpe_check_services = [
+            'ovn-northd',
+            'ovn-nb-ovsdb',
+            'ovn-sb-ovsdb',
+        ]
 
     def install(self):
         """Override charm install method.
@@ -546,6 +562,11 @@ class UssuriOVNCentralCharm(BaseOVNCentralCharm):
             'ovn-ovsdb-server-nb',
             'ovn-ovsdb-server-sb',
         ])
+        self.nrpe_check_services = [
+            'ovn-northd',
+            'ovn-ovsdb-server-nb',
+            'ovn-ovsdb-server-sb',
+        ]
 
     def install(self):
         """Override charm install method."""

src/metadata.yaml

@@ -35,6 +35,9 @@ provides:
     interface: ovsdb-cms
   ovsdb-server:
     interface: ovsdb-cluster
+  nrpe-external-master:
+    interface: nrpe-external-master
+    scope: container
 peers:
   ovsdb-peer:
     interface: ovsdb-cluster

src/reactive/ovn_central_handlers.py

@@ -198,3 +198,15 @@ def render():
                 ovsdb_peer.db_sb_admin_port)
             reactive.set_flag('config.rendered')
         ovn_charm.assess_status()
+
+
+@reactive.when_none('charm.paused', 'is-update-status-hook')
+@reactive.when('config.rendered')
+@reactive.when_any('config.changed.nagios_context',
+                   'config.changed.nagios_servicegroups',
+                   'endpoint.nrpe-external-master.changed',
+                   'nrpe-external-master.available')
+def configure_nrpe():
+    """Handle config-changed for NRPE options."""
+    with charm.provide_charm_instance() as charm_instance:
+        charm_instance.render_nrpe()

src/tests/bundles/bionic-train.yaml

@@ -4,6 +4,8 @@ relations:
   - mysql:shared-db
 - - ovn-central:certificates
   - vault:certificates
+- - 'nrpe:nrpe-external-master'
+  - 'ovn-central:nrpe-external-master'
 applications:
   mysql:
     charm: cs:~openstack-charmers-next/percona-cluster
@@ -17,3 +19,5 @@ applications:
     num_units: 3
     options:
       source: cloud:bionic-train
+  nrpe:
+    charm: cs:nrpe

src/tests/bundles/bionic-ussuri.yaml

@@ -4,6 +4,8 @@ relations:
   - mysql:shared-db
 - - ovn-central:certificates
   - vault:certificates
+- - 'nrpe:nrpe-external-master'
+  - 'ovn-central:nrpe-external-master'
 applications:
   mysql:
     charm: cs:~openstack-charmers-next/percona-cluster
@@ -17,3 +19,5 @@ applications:
     num_units: 3
     options:
       source: cloud:bionic-ussuri/proposed
+  nrpe:
+    charm: cs:nrpe

src/tests/bundles/focal-ussuri.yaml

@@ -48,6 +48,10 @@ applications:
       - '5'
       - '6'
 
+  nrpe:
+    charm: cs:nrpe
+
+
 relations:
 
   - - 'vault:shared-db'
@@ -57,3 +61,5 @@ relations:
 
   - - 'ovn-central:certificates'
     - 'vault:certificates'
+  - - 'nrpe:nrpe-external-master'
+    - 'ovn-central:nrpe-external-master'

src/tests/bundles/focal-victoria.yaml

@@ -48,6 +48,9 @@ applications:
       - '5'
       - '6'
 
+  nrpe:
+    charm: cs:nrpe
+
 relations:
 
   - - 'vault:shared-db'
@@ -57,3 +60,5 @@ relations:
 
   - - 'ovn-central:certificates'
     - 'vault:certificates'
+  - - 'nrpe:nrpe-external-master'
+    - 'ovn-central:nrpe-external-master'

src/tests/bundles/groovy-victoria.yaml

@@ -6,6 +6,8 @@ relations:
   - vault-mysql-router:shared-db
 - - ovn-central:certificates
   - vault:certificates
+- - 'nrpe:nrpe-external-master'
+  - 'ovn-central:nrpe-external-master'
 applications:
   mysql-innodb-cluster:
     charm: cs:~openstack-charmers-next/mysql-innodb-cluster
@@ -19,3 +21,5 @@ applications:
     series: groovy
     charm: cs:~openstack-charmers-next/ovn-central
     num_units: 3
+  nrpe:
+    charm: cs:nrpe

src/tests/tests.yaml

@@ -15,6 +15,10 @@ target_deploy_status:
   vault:
     workload-status: blocked
     workload-status-message: Vault needs to be initialized
+  nrpe:
+    workload-status: blocked
+    workload-status-message: "Nagios server not configured or related"
+
 # Note that full end to end tests are performed with OVN in the
 # neutron-api-plugin-ovn and octavia charm gates
 configure:

unit_tests/__init__.py

@@ -36,6 +36,7 @@ class _fake_decorator(object):
 sys.modules['charmhelpers.contrib.network.ovs'] = mock.MagicMock()
 sys.modules['charmhelpers.contrib.network.ovs.ovn'] = mock.MagicMock()
 sys.modules['charmhelpers.contrib.network.ovs.ovsdb'] = mock.MagicMock()
+sys.modules['charmhelpers.contrib.charmsupport.nrpe'] = mock.MagicMock()
 charms = mock.MagicMock()
 sys.modules['charms'] = charms
 charms.leadership = mock.MagicMock()

unit_tests/test_lib_charms_ovn_central.py

@@ -410,3 +410,19 @@ class TestOVNCentralCharm(Helper):
         self.ch_ufw.modify_access.assert_has_calls([
             mock.call(None, dst=None, action='delete', index=42)
         ])
+
+    def test_render_nrpe(self):
+        self.patch_object(ovn_central.nrpe, 'NRPE')
+        self.patch_object(ovn_central.nrpe, 'add_init_service_checks')
+        self.target.render_nrpe()
+        # Note that this list is valid for Ussuri
+        self.add_init_service_checks.assert_has_calls([
+            mock.call().add_init_service_checks(
+                mock.ANY,
+                ['ovn-northd', 'ovn-ovsdb-server-nb', 'ovn-ovsdb-server-sb'],
+                mock.ANY
+            ),
+        ])
+        self.NRPE.assert_has_calls([
+            mock.call().write(),
+        ])

unit_tests/test_reactive_ovn_central_handlers.py

@@ -45,6 +45,7 @@ class TestRegisteredHooks(test_utils.TestRegisteredHooks):
                 'maybe_do_upgrade': ('is-update-status-hook',),
                 'publish_addr_to_clients': ('is-update-status-hook',),
                 'render': ('is-update-status-hook',),
+                'configure_nrpe': ('charm.paused', 'is-update-status-hook',),
             },
             'when': {
                 'announce_leader_ready': ('config.rendered',
@@ -71,6 +72,13 @@ class TestRegisteredHooks(test_utils.TestRegisteredHooks):
                            'leadership.set.sb_cid',
                            'certificates.connected',
                            'certificates.available',),
+                'configure_nrpe': ('config.rendered',),
+            },
+            'when_any': {
+                'configure_nrpe': ('config.changed.nagios_context',
+                                   'config.changed.nagios_servicegroups',
+                                   'endpoint.nrpe-external-master.changed',
+                                   'nrpe-external-master.available',),
             },
         }
         # test that the hooks were registered via the