diff --git a/src/actions.yaml b/src/actions.yaml new file mode 100644 index 0000000..54c58d7 --- /dev/null +++ b/src/actions.yaml @@ -0,0 +1,27 @@ +restart-services: + description: | + Restarts services this charm manages. + params: + deferred-only: + type: boolean + default: false + description: | + Restart all deferred services. + services: + type: string + default: "" + description: | + List of services to restart. + run-hooks: + type: boolean + default: true + description: | + Run any hooks which have been deferred. +run-deferred-hooks: + description: | + Run deferable hooks and restart services. + . + NOTE: Service will be restarted as needed irrespective of enable-auto-restarts +show-deferred-events: + descrpition: | + Show the outstanding restarts diff --git a/src/actions/os_deferred_event_actions.py b/src/actions/os_deferred_event_actions.py new file mode 100755 index 0000000..fa17211 --- /dev/null +++ b/src/actions/os_deferred_event_actions.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +# Copyright 2021 Canonical Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys + +# Load modules from $CHARM_DIR/lib +sys.path.append('lib') + +from charms.layer import basic +basic.bootstrap_charm_deps() + +import charmhelpers.contrib.openstack.deferred_events as deferred_events +import charmhelpers.contrib.openstack.utils as os_utils +import charmhelpers.core.hookenv as hookenv +import charms_openstack.bus +import charms_openstack.charm +import charms.reactive as reactive + +charms_openstack.bus.discover() + + +def restart_services(args): + """Restart services. + + :param args: Unused + :type args: List[str] + """ + deferred_only = hookenv.action_get("deferred-only") + services = hookenv.action_get("services").split() + # Check input + if deferred_only and services: + hookenv.action_fail("Cannot set deferred-only and services") + return + if not (deferred_only or services): + hookenv.action_fail("Please specify deferred-only or services") + return + if deferred_only: + os_utils.restart_services_action(deferred_only=True) + else: + os_utils.restart_services_action(services=services) + with charms_openstack.charm.provide_charm_instance() as charm_instance: + charm_instance._assess_status() + + +def show_deferred_events(args): + """Show the deferred events. + + :param args: Unused + :type args: List[str] + """ + os_utils.show_deferred_events_action_helper() + + +def run_deferred_hooks(args): + """Run deferred hooks. + + :param args: Unused + :type args: List[str] + """ + deferred_methods = deferred_events.get_deferred_hooks() + ovsdb = reactive.endpoint_from_flag('ovsdb.available') + with charms_openstack.charm.provide_charm_instance() as charm_instance: + if ('install' in deferred_methods or + 'configure_ovs' in deferred_methods): + charm_instance.install(check_deferred_events=False) + if 'configure_ovs' in deferred_methods: + charm_instance.render_with_interfaces( + charms_openstack.charm.optional_interfaces( + (ovsdb,), + 'nova-compute.connected', + 'amqp.connected')) + charm_instance.configure_ovs( + ','.join(ovsdb.db_sb_connection_strs), + reactive.is_flag_set('config.changed.disable-mlockall'), + check_deferred_events=False) + charm_instance._assess_status() + + +# Actions to function mapping, to allow for illegal python action names that +# can map to a python function. +ACTIONS = { + "restart-services": restart_services, + "show-deferred-events": show_deferred_events, + "run-deferred-hooks": run_deferred_hooks +} + + +def main(args): + hookenv._run_atstart() + action_name = os.path.basename(args[0]) + try: + action = ACTIONS[action_name] + except KeyError: + return "Action %s undefined" % action_name + else: + try: + action(args) + except Exception as e: + hookenv.action_fail(str(e)) + hookenv._run_atexit() + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/src/actions/restart-services b/src/actions/restart-services new file mode 100755 index 0000000..09e534c --- /dev/null +++ b/src/actions/restart-services @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +# Copyright 2021 Canonical Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + + +sys.path.append('actions') + + +import os_deferred_event_actions + + +if __name__ == "__main__": + sys.exit(os_deferred_event_actions.main(sys.argv)) diff --git a/src/actions/run-deferred-hooks b/src/actions/run-deferred-hooks new file mode 100755 index 0000000..09e534c --- /dev/null +++ b/src/actions/run-deferred-hooks @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +# Copyright 2021 Canonical Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + + +sys.path.append('actions') + + +import os_deferred_event_actions + + +if __name__ == "__main__": + sys.exit(os_deferred_event_actions.main(sys.argv)) diff --git a/src/actions/show-deferred-events b/src/actions/show-deferred-events new file mode 100755 index 0000000..09e534c --- /dev/null +++ b/src/actions/show-deferred-events @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +# Copyright 2021 Canonical Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + + +sys.path.append('actions') + + +import os_deferred_event_actions + + +if __name__ == "__main__": + sys.exit(os_deferred_event_actions.main(sys.argv)) diff --git a/src/config.yaml b/src/config.yaml index eb2a7eb..187b3bb 100644 --- a/src/config.yaml +++ b/src/config.yaml @@ -63,3 +63,9 @@ options: type: string description: | Comma separated list of nagios servicegroups for the service checks. + enable-auto-restarts: + type: boolean + default: True + description: | + Allow the charm and packages to restart services automatically when + required. diff --git a/src/lib/charm/openstack/ovn_central.py b/src/lib/charm/openstack/ovn_central.py index bb9db1b..f27ee0f 100644 --- a/src/lib/charm/openstack/ovn_central.py +++ b/src/lib/charm/openstack/ovn_central.py @@ -23,6 +23,7 @@ import charmhelpers.contrib.charmsupport.nrpe as nrpe import charmhelpers.contrib.network.ovs.ovn as ch_ovn import charmhelpers.contrib.network.ovs.ovsdb as ch_ovsdb from charmhelpers.contrib.network import ufw as ch_ufw +import charmhelpers.contrib.openstack.deferred_events as deferred_events import charms.reactive as reactive @@ -92,6 +93,43 @@ class BaseOVNCentralCharm(charms_openstack.charm.OpenStackCharm): } super().__init__(**kwargs) + def restart_on_change(self): + """Restart the services in the self.restart_map{} attribute if any of + the files identified by the keys changes for the wrapped call. + + Usage: + + with restart_on_change(restart_map, ...): + do_stuff_that_might_trigger_a_restart() + ... + """ + return ch_core.host.restart_on_change( + self.full_restart_map, + stopstart=True, + restart_functions=getattr(self, 'restart_functions', None), + can_restart_now_f=deferred_events.check_and_record_restart_request, + post_svc_restart_f=deferred_events.process_svc_restart) + + @property + def deferable_services(self): + """Services which should be stopped from restarting. + + All services from self.services are deferable. But the charm may + install a package which install a service that the charm does not add + to its restart_map. In that case it will be missing from + self.services. However one of the jobs of deferred events is to ensure + that packages updates outside of charms also do not restart services. + To ensure there is a complete list take the services from self.services + and also add in a known list of networking services. + + NOTE: It does not matter if one of the services in the list is not + installed on the system. + """ + svcs = self.services[:] + svcs.extend(['ovn-ovsdb-server-nb', 'ovn-ovsdb-server-nb', + 'ovn-northd', 'ovn-central']) + return list(set(svcs)) + def install(self, service_masks=None): """Extend the default install method. @@ -116,6 +154,16 @@ class BaseOVNCentralCharm(charms_openstack.charm.OpenStackCharm): self.configure_source() super().install() + def configure_deferred_restarts(self): + if 'enable-auto-restarts' in ch_core.hookenv.config().keys(): + deferred_events.configure_deferred_restarts( + self.deferable_services) + # Reactive charms execute perm missing. + os.chmod( + '/var/lib/charm/{}/policy-rc.d'.format( + ch_core.hookenv.service_name()), + 0o755) + def states_to_check(self, required_relations=None): """Override parent method to add custom messaging. @@ -617,6 +665,33 @@ class BaseOVNCentralCharm(charms_openstack.charm.OpenStackCharm): charm_nrpe, self.nrpe_check_services, current_unit) charm_nrpe.write() + def custom_assess_status_check(self): + """Report deferred events in charm status message.""" + state = None + message = None + deferred_events.check_restart_timestamps() + events = collections.defaultdict(set) + for e in deferred_events.get_deferred_events(): + events[e.action].add(e.service) + for action, svcs in events.items(): + svc_msg = "Services queued for {}: {}".format( + action, ', '.join(sorted(svcs))) + state = 'active' + if message: + message = "{}. {}".format(message, svc_msg) + else: + message = svc_msg + deferred_hooks = deferred_events.get_deferred_hooks() + if deferred_hooks: + state = 'active' + svc_msg = "Hooks skipped due to disabled auto restarts: {}".format( + ', '.join(sorted(deferred_hooks))) + if message: + message = "{}. {}".format(message, svc_msg) + else: + message = svc_msg + return state, message + class TrainOVNCentralCharm(BaseOVNCentralCharm): # OpenvSwitch and OVN is distributed as part of the Ubuntu Cloud Archive diff --git a/src/reactive/ovn_central_handlers.py b/src/reactive/ovn_central_handlers.py index 2c4a795..0e0dd33 100644 --- a/src/reactive/ovn_central_handlers.py +++ b/src/reactive/ovn_central_handlers.py @@ -210,3 +210,9 @@ def configure_nrpe(): """Handle config-changed for NRPE options.""" with charm.provide_charm_instance() as charm_instance: charm_instance.render_nrpe() + + +@reactive.when_not('is-update-status-hook') +def configure_deferred_restarts(): + with charm.provide_charm_instance() as instance: + instance.configure_deferred_restarts() diff --git a/src/tests/tests.yaml b/src/tests/tests.yaml index 4f6b234..72c2579 100644 --- a/src/tests/tests.yaml +++ b/src/tests/tests.yaml @@ -23,6 +23,7 @@ target_deploy_status: configure: - zaza.openstack.charm_tests.vault.setup.auto_initialize_no_validation tests: +- zaza.openstack.charm_tests.ovn.tests.OVNCentralDeferredRestartTest - zaza.openstack.charm_tests.ovn.tests.CentralCharmOperationTest tests_options: force_deploy: diff --git a/unit_tests/__init__.py b/unit_tests/__init__.py index 0ed404f..cf7bc6f 100644 --- a/unit_tests/__init__.py +++ b/unit_tests/__init__.py @@ -37,6 +37,8 @@ sys.modules['charmhelpers.contrib.network.ovs'] = mock.MagicMock() sys.modules['charmhelpers.contrib.network.ovs.ovn'] = mock.MagicMock() sys.modules['charmhelpers.contrib.network.ovs.ovsdb'] = mock.MagicMock() sys.modules['charmhelpers.contrib.charmsupport.nrpe'] = mock.MagicMock() +sys.modules[ + 'charmhelpers.contrib.openstack.deferred_events'] = mock.MagicMock() charms = mock.MagicMock() sys.modules['charms'] = charms charms.leadership = mock.MagicMock() @@ -66,6 +68,8 @@ charms.ovn = mock.MagicMock() sys.modules['charms.ovn'] = charms.ovn charms.ovn_charm = mock.MagicMock() sys.modules['charms.ovn_charm'] = charms.ovn +charms.layer = mock.MagicMock() +sys.modules['charms.layer'] = charms.layer keystoneauth1 = mock.MagicMock() sys.modules['keystoneauth1'] = keystoneauth1 netaddr = mock.MagicMock() diff --git a/unit_tests/test_actions_os_deferred_event_actions.py b/unit_tests/test_actions_os_deferred_event_actions.py new file mode 100644 index 0000000..c81ede6 --- /dev/null +++ b/unit_tests/test_actions_os_deferred_event_actions.py @@ -0,0 +1,140 @@ +# Copyright 2021 Canonical Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import unittest.mock as mock +import actions.os_deferred_event_actions as os_deferred_event_actions +import charms_openstack.test_utils as test_utils + + +class TestOSDeferredEventActions(test_utils.PatchHelper): + + def setUp(self): + super().setUp() + self.patch_object(os_deferred_event_actions.hookenv, 'action_get') + self.action_config = {} + self.action_get.side_effect = lambda x: self.action_config.get(x) + self.patch_object(os_deferred_event_actions.hookenv, 'action_fail') + + self.patch_object( + os_deferred_event_actions.charms_openstack.charm, + 'provide_charm_instance') + self.charm_instance = mock.MagicMock() + self.provide_charm_instance.return_value.__enter__.return_value = \ + self.charm_instance + + def test_restart_services(self): + self.patch_object( + os_deferred_event_actions.os_utils, + 'restart_services_action') + + self.action_config = { + 'deferred-only': True, + 'services': ''} + os_deferred_event_actions.restart_services(['restart-services']) + self.charm_instance._assess_status.assert_called_once_with() + self.restart_services_action.assert_called_once_with( + deferred_only=True) + + self.charm_instance.reset_mock() + self.restart_services_action.reset_mock() + + self.action_config = { + 'deferred-only': False, + 'services': 'svcA svcB'} + os_deferred_event_actions.restart_services(['restart-services']) + self.charm_instance._assess_status.assert_called_once_with() + self.restart_services_action.assert_called_once_with( + services=['svcA', 'svcB']) + + self.charm_instance.reset_mock() + self.restart_services_action.reset_mock() + + self.action_config = { + 'deferred-only': True, + 'services': 'svcA svcB'} + os_deferred_event_actions.restart_services(['restart-services']) + self.action_fail.assert_called_once_with( + 'Cannot set deferred-only and services') + + self.charm_instance.reset_mock() + self.restart_services_action.reset_mock() + self.action_fail.reset_mock() + + self.action_config = { + 'deferred-only': False, + 'services': ''} + os_deferred_event_actions.restart_services(['restart-services']) + self.action_fail.assert_called_once_with( + 'Please specify deferred-only or services') + + def test_show_deferred_events(self): + self.patch_object( + os_deferred_event_actions.os_utils, + 'show_deferred_events_action_helper') + os_deferred_event_actions.show_deferred_events( + ['show-deferred-events']) + self.show_deferred_events_action_helper.assert_called_once_with() + + def test_run_deferred_hooks(self): + self.patch_object( + os_deferred_event_actions.deferred_events, + 'get_deferred_hooks') + self.patch_object( + os_deferred_event_actions.reactive, + 'endpoint_from_flag') + self.patch_object( + os_deferred_event_actions.reactive, + 'is_flag_set') + self.patch_object( + os_deferred_event_actions.charms_openstack.charm, + 'optional_interfaces') + interfaces_mock = mock.MagicMock() + self.optional_interfaces.return_value = interfaces_mock + self.is_flag_set.return_value = True + ovsdb_available = mock.MagicMock() + ovsdb_available.db_sb_connection_strs = ['constrA', 'connstrB'] + self.endpoint_from_flag.return_value = ovsdb_available + + self.get_deferred_hooks.return_value = ['install'] + os_deferred_event_actions.run_deferred_hooks(['run-deferred-hooks']) + self.charm_instance.install.assert_called_once_with( + check_deferred_events=False) + self.assertFalse(self.charm_instance.configure_ovs.called) + self.assertFalse( + self.charm_instance.render_with_interfaces.called) + self.charm_instance._assess_status.assert_called_once_with() + + self.charm_instance.reset_mock() + + self.get_deferred_hooks.return_value = ['install', 'configure_ovs'] + os_deferred_event_actions.run_deferred_hooks(['run-deferred-hooks']) + self.charm_instance.install.assert_called_once_with( + check_deferred_events=False) + self.charm_instance.render_with_interfaces.assert_called_once_with( + interfaces_mock) + self.charm_instance.configure_ovs.assert_called_once_with( + 'constrA,connstrB', + True, + check_deferred_events=False) + self.charm_instance._assess_status.assert_called_once_with() + + self.charm_instance.reset_mock() + + self.get_deferred_hooks.return_value = [] + os_deferred_event_actions.run_deferred_hooks(['run-deferred-hooks']) + self.assertFalse(self.charm_instance.install.configure_ovs.called) + self.assertFalse(self.charm_instance.configure_ovs.called) + self.assertFalse(self.charm_instance.render_with_interfaces.called) + self.charm_instance._assess_status.assert_called_once_with() diff --git a/unit_tests/test_lib_charms_ovn_central.py b/unit_tests/test_lib_charms_ovn_central.py index 054b9a3..6461f81 100644 --- a/unit_tests/test_lib_charms_ovn_central.py +++ b/unit_tests/test_lib_charms_ovn_central.py @@ -539,3 +539,36 @@ class TestOVNCentralCharm(Helper): self.NRPE.assert_has_calls([ mock.call().write(), ]) + + def test_configure_deferred_restarts(self): + self.patch_object( + ovn_central.ch_core.hookenv, + 'config', + return_value={'enable-auto-restarts': True}) + self.patch_object( + ovn_central.ch_core.hookenv, + 'service_name', + return_value='myapp') + self.patch_object( + ovn_central.deferred_events, + 'configure_deferred_restarts') + self.patch_object(ovn_central.os, 'chmod') + self.target.configure_deferred_restarts() + self.configure_deferred_restarts.assert_called_once_with( + ['ovn-central', 'ovn-ovsdb-server-nb', 'ovn-northd', + 'ovn-ovsdb-server-sb']) + + self.chmod.assert_called_once_with( + '/var/lib/charm/myapp/policy-rc.d', + 493) + + def test_configure_deferred_restarts_unsupported(self): + self.patch_object( + ovn_central.ch_core.hookenv, + 'config', + return_value={}) + self.patch_object( + ovn_central.deferred_events, + 'configure_deferred_restarts') + self.target.configure_deferred_restarts() + self.assertFalse(self.configure_deferred_restarts.called) diff --git a/unit_tests/test_reactive_ovn_central_handlers.py b/unit_tests/test_reactive_ovn_central_handlers.py index deb29e9..b07bfdb 100644 --- a/unit_tests/test_reactive_ovn_central_handlers.py +++ b/unit_tests/test_reactive_ovn_central_handlers.py @@ -80,6 +80,9 @@ class TestRegisteredHooks(test_utils.TestRegisteredHooks): 'endpoint.nrpe-external-master.changed', 'nrpe-external-master.available',), }, + 'when_not': { + 'configure_deferred_restarts': ('is-update-status-hook',), + }, } # test that the hooks were registered via the # reactive.ovn_handlers