openstack/charm-aodh
Code Issues Proposed changes

Add NRPE checks for services

Browse Source 
Adds the nrpe-external-master layer and checks for services managed by
this charm.

func-test-pr: https://github.com/openstack-charmers/zaza-openstack-tests/pull/445

Change-Id: I9109078bb0d07159de2024cb836d659a533e50c7
Closes-Bug: #1750486
This commit is contained in:
Xav Paice 2020-10-13 15:50:22 +13:00
parent 729c46410c
commit fbb7777638
22 changed files with 198 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
src/config.yaml
View File

@ -7,3 +7,17 @@ options:
override YAML files in the service's policy.d directory. The resource
file should be a ZIP file containing at least one yaml file with a .yaml
or .yml extension. If False then remove the overrides.
nagios_context:
default: "juju"
type: string
description: |
A string that will be prepended to instance name to set the host name
in nagios. So for instance the hostname would be something like:
juju-myservice-0
If you're running multiple environments with the same services in them
this allows you to differentiate between them.
nagios_servicegroups:
default: ""
type: string
description: |
Comma separated list of nagios servicegroups for the service checks.

2
src/layer.yaml
View File

@ -1,4 +1,4 @@
includes: ['layer:openstack-api', 'interface:mongodb']
includes: ['layer:openstack-api', 'interface:mongodb', 'interface:nrpe-external-master']
options:
basic:
use_venv: True

15
src/lib/charm/openstack/aodh.py
View File

@ -17,6 +17,7 @@ import os
import subprocess
import charmhelpers.core.host as ch_host
import charmhelpers.contrib.charmsupport.nrpe as nrpe
import charms_openstack.charm
import charms_openstack.adapters
@ -131,6 +132,15 @@ class AodhCharm(charms_openstack.plugins.PolicydOverridePlugin,
subprocess.check_call(['systemctl', 'daemon-reload'])
ch_host.service_restart('aodh-api')
def render_nrpe_checks(self):
"""Configure Nagios NRPE checks."""
hostname = nrpe.get_nagios_hostname()
current_unit = nrpe.get_nagios_unit_name()
charm_nrpe = nrpe.NRPE(hostname=hostname)
nrpe.add_init_service_checks(
charm_nrpe, self.services, current_unit)
charm_nrpe.write()
class AodhCharmNewton(AodhCharm):
"""Newton uses the aodh-api standalone systemd. If the systemd definition
@ -290,3 +300,8 @@ def reload_and_restart():
"""Reload systemd and restart aodh API when override file changes
"""
AodhCharm.singleton.reload_and_restart()
def render_nrpe():
"""Render NRPE service monitors."""
AodhCharm.singleton.render_nrpe_checks()

4
src/metadata.yaml
View File

@ -22,6 +22,10 @@ subordinate: false
requires:
mongodb:
interface: mongodb
provides:
nrpe-external-master:
interface: nrpe-external-master
scope: container
resources:
policyd-override:
type: file

11
src/reactive/aodh_handlers.py
View File

@ -104,3 +104,14 @@ def run_db_migration():
@reactive.when('ha.connected')
def cluster_connected(hacluster):
aodh.configure_ha_resources(hacluster)
@reactive.when_none('charm.paused', 'is-update-status-hook')
@reactive.when('config.complete')
@reactive.when_any('config.changed.nagios_context',
'config.changed.nagios_servicegroups',
'endpoint.nrpe-external-master.changed',
'nrpe-external-master.available')
def configure_nrpe():
"""Handle config-changed for NRPE options."""
aodh.render_nrpe()

4
src/tests/bundles/bionic-queens.yaml
View File

@ -92,6 +92,8 @@ applications:
num_units: 1
to:
- '12'
nrpe:
charm: cs:nrpe
relations:
- - 'keystone:shared-db'
- 'percona-cluster:shared-db'
@ -119,3 +121,5 @@ relations:
- 'ceph-mon:client'
- - 'gnocchi:coordinator-memcached'
- 'memcached:cache'
- - 'aodh:nrpe-external-master'
- 'nrpe:nrpe-external-master'

4
src/tests/bundles/bionic-rocky.yaml
View File

@ -92,6 +92,8 @@ applications:
num_units: 1
to:
- '12'
nrpe:
charm: cs:nrpe
relations:
- - 'keystone:shared-db'
- 'percona-cluster:shared-db'
@ -119,3 +121,5 @@ relations:
- 'ceph-mon:client'
- - 'gnocchi:coordinator-memcached'
- 'memcached:cache'
- - 'aodh:nrpe-external-master'
- 'nrpe:nrpe-external-master'

4
src/tests/bundles/bionic-stein.yaml
View File

@ -92,6 +92,8 @@ applications:
num_units: 1
to:
- '12'
nrpe:
charm: cs:nrpe
relations:
- - 'keystone:shared-db'
- 'percona-cluster:shared-db'
@ -119,3 +121,5 @@ relations:
- 'ceph-mon:client'
- - 'gnocchi:coordinator-memcached'
- 'memcached:cache'
- - 'aodh:nrpe-external-master'
- 'nrpe:nrpe-external-master'

4
src/tests/bundles/bionic-train.yaml
View File

@ -92,6 +92,8 @@ applications:
num_units: 1
to:
- '12'
nrpe:
charm: cs:nrpe
relations:
- - 'keystone:shared-db'
- 'percona-cluster:shared-db'
@ -119,3 +121,5 @@ relations:
- 'ceph-mon:client'
- - 'gnocchi:coordinator-memcached'
- 'memcached:cache'
- - 'aodh:nrpe-external-master'
- 'nrpe:nrpe-external-master'

4
src/tests/bundles/bionic-ussuri.yaml
View File

@ -92,6 +92,8 @@ applications:
num_units: 1
to:
- '12'
nrpe:
charm: cs:nrpe
relations:
- - 'keystone:shared-db'
- 'percona-cluster:shared-db'
@ -119,3 +121,5 @@ relations:
- 'ceph-mon:client'
- - 'gnocchi:coordinator-memcached'
- 'memcached:cache'
- - 'aodh:nrpe-external-master'
- 'nrpe:nrpe-external-master'

6
src/tests/bundles/focal-ussuri.yaml
View File

@ -115,6 +115,9 @@ applications:
to:
- '14'
nrpe:
charm: cs:nrpe
relations:
- - 'keystone:shared-db'
@ -162,3 +165,6 @@ relations:
- - 'gnocchi:coordinator-memcached'
- 'memcached:cache'
- - 'aodh:nrpe-external-master'
- 'nrpe:nrpe-external-master'

6
src/tests/bundles/focal-victoria.yaml
View File

@ -115,6 +115,9 @@ applications:
to:
- '14'
nrpe:
charm: cs:nrpe
relations:
- - 'keystone:shared-db'
@ -162,3 +165,6 @@ relations:
- - 'gnocchi:coordinator-memcached'
- 'memcached:cache'
- - 'aodh:nrpe-external-master'
- 'nrpe:nrpe-external-master'

6
src/tests/bundles/groovy-victoria.yaml
View File

@ -116,6 +116,9 @@ applications:
to:
- '14'
nrpe:
charm: cs:nrpe
relations:
- - 'keystone:shared-db'
@ -163,3 +166,6 @@ relations:
- - 'gnocchi:coordinator-memcached'
- 'memcached:cache'
- - 'aodh:nrpe-external-master'
- 'nrpe:nrpe-external-master'

4
src/tests/bundles/trusty-mitaka.yaml
View File

@ -56,6 +56,8 @@ applications:
num_units: 1
to:
- '5'
nrpe:
charm: cs:nrpe
relations:
- - 'keystone:shared-db'
- 'percona-cluster:shared-db'
@ -75,3 +77,5 @@ relations:
- 'mongodb:database'
- - 'ceilometer:identity-service'
- 'keystone:identity-service'
- - 'aodh:nrpe-external-master'
- 'nrpe:nrpe-external-master'

4
src/tests/bundles/xenial-mitaka.yaml
View File

@ -56,6 +56,8 @@ applications:
num_units: 1
to:
- '5'
nrpe:
charm: cs:nrpe
relations:
- - 'keystone:shared-db'
- 'percona-cluster:shared-db'
@ -75,3 +77,5 @@ relations:
- 'mongodb:database'
- - 'ceilometer:identity-service'
- 'keystone:identity-service'
- - 'aodh:nrpe-external-master'
- 'nrpe:nrpe-external-master'

4
src/tests/bundles/xenial-ocata.yaml
View File

@ -98,6 +98,8 @@ applications:
num_units: 1
to:
- '13'
nrpe:
charm: cs:nrpe
relations:
- - 'keystone:shared-db'
- 'percona-cluster:shared-db'
@ -129,3 +131,5 @@ relations:
- 'ceph-mon:client'
- - 'gnocchi:coordinator-memcached'
- 'memcached:cache'
- - 'aodh:nrpe-external-master'
- 'nrpe:nrpe-external-master'

4
src/tests/bundles/xenial-pike.yaml
View File

@ -98,6 +98,8 @@ applications:
num_units: 1
to:
- '13'
nrpe:
charm: cs:nrpe
relations:
- - 'keystone:shared-db'
- 'percona-cluster:shared-db'
@ -129,3 +131,5 @@ relations:
- 'ceph-mon:client'
- - 'gnocchi:coordinator-memcached'
- 'memcached:cache'
- - 'aodh:nrpe-external-master'
- 'nrpe:nrpe-external-master'

4
src/tests/bundles/xenial-queens.yaml
View File

@ -92,6 +92,8 @@ applications:
num_units: 1
to:
- '12'
nrpe:
charm: cs:nrpe
relations:
- - 'keystone:shared-db'
- 'percona-cluster:shared-db'
@ -119,3 +121,5 @@ relations:
- 'ceph-mon:client'
- - 'gnocchi:coordinator-memcached'
- 'memcached:cache'
- - 'aodh:nrpe-external-master'
- 'nrpe:nrpe-external-master'

3
src/tests/tests.yaml
View File

@ -27,6 +27,9 @@ target_deploy_status:
mongodb:
workload-status: unknown
workload-status-message: ''
nrpe:
workload-status: blocked
workload-status-message: "Nagios server not configured or related"
tests_options:
force_deploy:
- groovy-victoria

2
unit_tests/__init__.py
View File

@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import mock
import sys
sys.path.append('src')
@ -20,3 +21,4 @@ sys.path.append('src/lib')
# Mock out charmhelpers so that we can test without it.
import charms_openstack.test_mocks # noqa
charms_openstack.test_mocks.mock_charmhelpers()
sys.modules['charmhelpers.contrib.charmsupport.nrpe'] = mock.MagicMock()

118
unit_tests/test_aodh_handlers.py
View File

@ -15,41 +15,63 @@
import unittest
import mock
import charms_openstack.test_utils as test_utils
import reactive.aodh_handlers as handlers
_when_args = {}
_when_not_args = {}
class TestRegisteredHooks(test_utils.TestRegisteredHooks):
"""Run tests to ensure hooks are registered."""
def mock_hook_factory(d):
def mock_hook(*args, **kwargs):
def inner(f):
# remember what we were passed. Note that we can't actually
# determine the class we're attached to, as the decorator only gets
# the function.
try:
d[f.__name__].append(dict(args=args, kwargs=kwargs))
except KeyError:
d[f.__name__] = [dict(args=args, kwargs=kwargs)]
return f
return inner
return mock_hook
def test_registered_hooks(self):
"""Test that the correct hooks are registered."""
defaults = [
'charm.installed',
'config.changed',
'charm.default-select-release',
'update-status',
'upgrade-charm',
]
hook_set = {
'when': {
'setup_amqp_req': ('amqp.connected', ),
'setup_database': ('shared-db.connected', ),
'setup_endpoint': ('identity-service.connected', ),
'render_unclustered': ('charm.installed',
'shared-db.available',
'identity-service.available',
'amqp.available',),
'render_clustered': ('charm.installed',
'shared-db.available',
'identity-service.available',
'amqp.available',
'cluster.available',),
'run_db_migration': ('charm.installed',
'config.complete', ),
'cluster_connected': ('ha.connected', ),
'configure_nrpe': ('config.complete', ),
},
'when_not': {
'install_packages': ('charm.installed', ),
'render_unclustered': ('cluster.available', ),
'run_db_migration': ('db.synced', ),
},
'when_none': {
'configure_nrpe': ('charm.paused', 'is-update-status-hook', ),
},
'when_any': {
'configure_nrpe': ('config.changed.nagios_context',
'config.changed.nagios_servicegroups',
'endpoint.nrpe-external-master.changed',
'nrpe-external-master.available', ),
},
}
self.registered_hooks_test_helper(handlers, hook_set, defaults)
class TestAodhHandlers(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls._patched_when = mock.patch('charms.reactive.when',
mock_hook_factory(_when_args))
cls._patched_when_started = cls._patched_when.start()
cls._patched_when_not = mock.patch('charms.reactive.when_not',
mock_hook_factory(_when_not_args))
cls._patched_when_not_started = cls._patched_when_not.start()
# force requires to rerun the mock_hook decorator:
# try except is Python2/Python3 compatibility as Python3 has moved
# reload to importlib.
@ -61,12 +83,6 @@ class TestAodhHandlers(unittest.TestCase):
@classmethod
def tearDownClass(cls):
cls._patched_when.stop()
cls._patched_when_started = None
cls._patched_when = None
cls._patched_when_not.stop()
cls._patched_when_not_started = None
cls._patched_when_not = None
# and fix any breakage we did to the module
try:
reload(handlers)
@ -94,46 +110,6 @@ class TestAodhHandlers(unittest.TestCase):
self._patches_start[attr] = started
setattr(self, attr, started)
def test_registered_hooks(self):
# test that the hooks actually registered the relation expressions that
# are meaningful for this interface: this is to handle regressions.
# The keys are the function names that the hook attaches to.
when_patterns = {
'setup_amqp_req': ('amqp.connected', ),
'setup_database': ('shared-db.connected', ),
'setup_endpoint': ('identity-service.connected', ),
'render_unclustered': ('charm.installed',
'shared-db.available',
'identity-service.available',
'amqp.available',),
'render_clustered': ('charm.installed',
'shared-db.available',
'identity-service.available',
'amqp.available',
'cluster.available',),
'run_db_migration': ('charm.installed',
'config.complete', ),
'cluster_connected': ('ha.connected', ),
}
when_not_patterns = {
'install_packages': ('charm.installed', ),
'render_unclustered': ('cluster.available', ),
'run_db_migration': ('db.synced', ),
}
# check the when hooks are attached to the expected functions
for t, p in [(_when_args, when_patterns),
(_when_not_args, when_not_patterns)]:
for f, args in t.items():
# check that function is in patterns
self.assertTrue(f in p.keys(),
"{} not found".format(f))
# check that the lists are equal
items = []
for a in args:
items += a['args'][:]
self.assertEqual(sorted(items), sorted(p[f]),
"{}: incorrect state registration".format(f))
def test_install_packages(self):
self.patch(handlers.aodh, 'install')
self.patch(handlers.reactive, 'set_state')

43
unit_tests/test_lib_charm_openstack_aodh.py
View File

@ -123,6 +123,29 @@ class TestAodhCharm(Helper):
self.check_call.assert_called_once_with(['systemctl', 'daemon-reload'])
self.service_restart.assert_called_once_with('aodh-api')
def test_render_nrpe(self):
"""Test NRPE renders correctly pre Ocata."""
self.patch_object(aodh.nrpe, 'NRPE')
self.patch_object(aodh.nrpe, 'add_init_service_checks')
services = ['aodh-api',
'aodh-evaluator',
'aodh-notifier',
'aodh-listener',
]
target = aodh.AodhCharmNewton()
target.render_nrpe_checks()
# Note that this list is valid for Ussuri
self.add_init_service_checks.assert_has_calls([
mock.call().add_init_service_checks(
mock.ANY,
services,
mock.ANY
),
])
self.NRPE.assert_has_calls([
mock.call().write(),
])
class TestAodhCharmOcata(Helper):
@ -136,3 +159,23 @@ class TestAodhCharmOcata(Helper):
self.init_is_systemd.return_value = True
aodh.AodhCharmOcata.reload_and_restart()
self.check_call.assert_called_once_with(['systemctl', 'daemon-reload'])
def test_render_nrpe(self):
"""Test NRPE renders correctly in Ocata."""
self.patch_object(aodh.nrpe, 'NRPE')
self.patch_object(aodh.nrpe, 'add_init_service_checks')
services = ['aodh-evaluator', 'aodh-notifier',
'aodh-listener', 'apache2']
target = aodh.AodhCharmOcata()
target.render_nrpe_checks()
# Note that this list is valid for Ussuri
self.add_init_service_checks.assert_has_calls([
mock.call().add_init_service_checks(
mock.ANY,
services,
mock.ANY
),
])
self.NRPE.assert_has_calls([
mock.call().write(),
])