Implementation of deferred restarts

Add deferred restart actions and config.

Change-Id: I74274ab33781ae3a980efb8cfae7ff0cb922f69d

src/actions.yaml

@@ -0,0 +1,27 @@
+restart-services:
+  description: |
+    Restarts services this charm manages.
+  params:
+    deferred-only:
+      type: boolean
+      default: false
+      description: |
+        Restart all deferred services.
+    services:
+      type: string
+      default: ""
+      description: |
+        List of services to restart.
+    run-hooks:
+      type: boolean
+      default: true
+      description: |
+        Run any hooks which have been deferred.
+run-deferred-hooks:
+  description: |
+    Run deferable hooks and restart services.
+    .
+    NOTE: Service will be restarted as needed irrespective of enable-auto-restarts
+show-deferred-events:
+    descrpition: |
+        Show the outstanding restarts

src/actions/os_deferred_event_actions.py

@@ -0,0 +1,117 @@
+#!/usr/bin/env python3
+# Copyright 2021 Canonical Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import sys
+
+# Load modules from $CHARM_DIR/lib
+sys.path.append('lib')
+
+from charms.layer import basic
+basic.bootstrap_charm_deps()
+
+import charmhelpers.contrib.openstack.deferred_events as deferred_events
+import charmhelpers.contrib.openstack.utils as os_utils
+import charmhelpers.core.hookenv as hookenv
+import charms_openstack.bus
+import charms_openstack.charm
+import charms.reactive as reactive
+
+charms_openstack.bus.discover()
+
+
+def restart_services(args):
+    """Restart services.
+
+    :param args: Unused
+    :type args: List[str]
+    """
+    deferred_only = hookenv.action_get("deferred-only")
+    services = hookenv.action_get("services").split()
+    # Check input
+    if deferred_only and services:
+        hookenv.action_fail("Cannot set deferred-only and services")
+        return
+    if not (deferred_only or services):
+        hookenv.action_fail("Please specify deferred-only or services")
+        return
+    if deferred_only:
+        os_utils.restart_services_action(deferred_only=True)
+    else:
+        os_utils.restart_services_action(services=services)
+    with charms_openstack.charm.provide_charm_instance() as charm_instance:
+        charm_instance._assess_status()
+
+
+def show_deferred_events(args):
+    """Show the deferred events.
+
+    :param args: Unused
+    :type args: List[str]
+    """
+    os_utils.show_deferred_events_action_helper()
+
+
+def run_deferred_hooks(args):
+    """Run deferred hooks.
+
+    :param args: Unused
+    :type args: List[str]
+    """
+    deferred_methods = deferred_events.get_deferred_hooks()
+    ovsdb = reactive.endpoint_from_flag('ovsdb.available')
+    with charms_openstack.charm.provide_charm_instance() as charm_instance:
+        if ('install' in deferred_methods or
+                'configure_ovs' in deferred_methods):
+            charm_instance.install(check_deferred_events=False)
+        if 'configure_ovs' in deferred_methods:
+            charm_instance.render_with_interfaces(
+                charms_openstack.charm.optional_interfaces(
+                    (ovsdb,),
+                    'nova-compute.connected',
+                    'amqp.connected'))
+            charm_instance.configure_ovs(
+                ','.join(ovsdb.db_sb_connection_strs),
+                reactive.is_flag_set('config.changed.disable-mlockall'),
+                check_deferred_events=False)
+        charm_instance._assess_status()
+
+
+# Actions to function mapping, to allow for illegal python action names that
+# can map to a python function.
+ACTIONS = {
+    "restart-services": restart_services,
+    "show-deferred-events": show_deferred_events,
+    "run-deferred-hooks": run_deferred_hooks
+}
+
+
+def main(args):
+    hookenv._run_atstart()
+    action_name = os.path.basename(args[0])
+    try:
+        action = ACTIONS[action_name]
+    except KeyError:
+        return "Action %s undefined" % action_name
+    else:
+        try:
+            action(args)
+        except Exception as e:
+            hookenv.action_fail(str(e))
+    hookenv._run_atexit()
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv))

src/actions/restart-services

@@ -0,0 +1,26 @@
+#!/usr/bin/env python3
+# Copyright 2021 Canonical Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+
+
+sys.path.append('actions')
+
+
+import os_deferred_event_actions
+
+
+if __name__ == "__main__":
+    sys.exit(os_deferred_event_actions.main(sys.argv))

src/actions/run-deferred-hooks

@@ -0,0 +1,26 @@
+#!/usr/bin/env python3
+# Copyright 2021 Canonical Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+
+
+sys.path.append('actions')
+
+
+import os_deferred_event_actions
+
+
+if __name__ == "__main__":
+    sys.exit(os_deferred_event_actions.main(sys.argv))

src/actions/show-deferred-events

@@ -0,0 +1,26 @@
+#!/usr/bin/env python3
+# Copyright 2021 Canonical Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+
+
+sys.path.append('actions')
+
+
+import os_deferred_event_actions
+
+
+if __name__ == "__main__":
+    sys.exit(os_deferred_event_actions.main(sys.argv))

src/config.yaml

@@ -63,3 +63,9 @@ options:
     type: string
     description: |
       Comma separated list of nagios servicegroups for the service checks.
+  enable-auto-restarts:
+    type: boolean
+    default: True
+    description: |
+      Allow the charm and packages to restart services automatically when
+      required.

src/lib/charm/openstack/ovn_central.py

@@ -23,6 +23,7 @@ import charmhelpers.contrib.charmsupport.nrpe as nrpe
 import charmhelpers.contrib.network.ovs.ovn as ch_ovn
 import charmhelpers.contrib.network.ovs.ovsdb as ch_ovsdb
 from charmhelpers.contrib.network import ufw as ch_ufw
+import charmhelpers.contrib.openstack.deferred_events as deferred_events
 
 import charms.reactive as reactive
 
@@ -92,6 +93,43 @@ class BaseOVNCentralCharm(charms_openstack.charm.OpenStackCharm):
         }
         super().__init__(**kwargs)
 
+    def restart_on_change(self):
+        """Restart the services in the self.restart_map{} attribute if any of
+        the files identified by the keys changes for the wrapped call.
+
+        Usage:
+
+           with restart_on_change(restart_map, ...):
+               do_stuff_that_might_trigger_a_restart()
+               ...
+        """
+        return ch_core.host.restart_on_change(
+            self.full_restart_map,
+            stopstart=True,
+            restart_functions=getattr(self, 'restart_functions', None),
+            can_restart_now_f=deferred_events.check_and_record_restart_request,
+            post_svc_restart_f=deferred_events.process_svc_restart)
+
+    @property
+    def deferable_services(self):
+        """Services which should be stopped from restarting.
+
+        All services from self.services are deferable. But the charm may
+        install a package which install a service that the charm does not add
+        to its restart_map. In that case it will be missing from
+        self.services. However one of the jobs of deferred events is to ensure
+        that packages updates outside of charms also do not restart services.
+        To ensure there is a complete list take the services from self.services
+        and also add in a known list of networking services.
+
+        NOTE: It does not matter if one of the services in the list is not
+        installed on the system.
+        """
+        svcs = self.services[:]
+        svcs.extend(['ovn-ovsdb-server-nb', 'ovn-ovsdb-server-nb',
+                     'ovn-northd', 'ovn-central'])
+        return list(set(svcs))
+
     def install(self, service_masks=None):
         """Extend the default install method.
 
@@ -116,6 +154,16 @@ class BaseOVNCentralCharm(charms_openstack.charm.OpenStackCharm):
         self.configure_source()
         super().install()
 
+    def configure_deferred_restarts(self):
+        if 'enable-auto-restarts' in ch_core.hookenv.config().keys():
+            deferred_events.configure_deferred_restarts(
+                self.deferable_services)
+            # Reactive charms execute perm missing.
+            os.chmod(
+                '/var/lib/charm/{}/policy-rc.d'.format(
+                    ch_core.hookenv.service_name()),
+                0o755)
+
     def states_to_check(self, required_relations=None):
         """Override parent method to add custom messaging.
 
@@ -617,6 +665,33 @@ class BaseOVNCentralCharm(charms_openstack.charm.OpenStackCharm):
             charm_nrpe, self.nrpe_check_services, current_unit)
         charm_nrpe.write()
 
+    def custom_assess_status_check(self):
+        """Report deferred events in charm status message."""
+        state = None
+        message = None
+        deferred_events.check_restart_timestamps()
+        events = collections.defaultdict(set)
+        for e in deferred_events.get_deferred_events():
+            events[e.action].add(e.service)
+        for action, svcs in events.items():
+            svc_msg = "Services queued for {}: {}".format(
+                action, ', '.join(sorted(svcs)))
+            state = 'active'
+            if message:
+                message = "{}. {}".format(message, svc_msg)
+            else:
+                message = svc_msg
+        deferred_hooks = deferred_events.get_deferred_hooks()
+        if deferred_hooks:
+            state = 'active'
+            svc_msg = "Hooks skipped due to disabled auto restarts: {}".format(
+                ', '.join(sorted(deferred_hooks)))
+            if message:
+                message = "{}. {}".format(message, svc_msg)
+            else:
+                message = svc_msg
+        return state, message
+
 
 class TrainOVNCentralCharm(BaseOVNCentralCharm):
     # OpenvSwitch and OVN is distributed as part of the Ubuntu Cloud Archive

src/reactive/ovn_central_handlers.py

@@ -210,3 +210,9 @@ def configure_nrpe():
     """Handle config-changed for NRPE options."""
     with charm.provide_charm_instance() as charm_instance:
         charm_instance.render_nrpe()
+
+
+@reactive.when_not('is-update-status-hook')
+def configure_deferred_restarts():
+    with charm.provide_charm_instance() as instance:
+        instance.configure_deferred_restarts()

src/tests/tests.yaml

@@ -23,6 +23,7 @@ target_deploy_status:
 configure:
 - zaza.openstack.charm_tests.vault.setup.auto_initialize_no_validation
 tests:
+- zaza.openstack.charm_tests.ovn.tests.OVNCentralDeferredRestartTest
 - zaza.openstack.charm_tests.ovn.tests.CentralCharmOperationTest
 tests_options:
   force_deploy:

unit_tests/__init__.py

@@ -37,6 +37,8 @@ sys.modules['charmhelpers.contrib.network.ovs'] = mock.MagicMock()
 sys.modules['charmhelpers.contrib.network.ovs.ovn'] = mock.MagicMock()
 sys.modules['charmhelpers.contrib.network.ovs.ovsdb'] = mock.MagicMock()
 sys.modules['charmhelpers.contrib.charmsupport.nrpe'] = mock.MagicMock()
+sys.modules[
+    'charmhelpers.contrib.openstack.deferred_events'] = mock.MagicMock()
 charms = mock.MagicMock()
 sys.modules['charms'] = charms
 charms.leadership = mock.MagicMock()
@@ -66,6 +68,8 @@ charms.ovn = mock.MagicMock()
 sys.modules['charms.ovn'] = charms.ovn
 charms.ovn_charm = mock.MagicMock()
 sys.modules['charms.ovn_charm'] = charms.ovn
+charms.layer = mock.MagicMock()
+sys.modules['charms.layer'] = charms.layer
 keystoneauth1 = mock.MagicMock()
 sys.modules['keystoneauth1'] = keystoneauth1
 netaddr = mock.MagicMock()

unit_tests/test_actions_os_deferred_event_actions.py

@@ -0,0 +1,140 @@
+# Copyright 2021 Canonical Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import unittest.mock as mock
+import actions.os_deferred_event_actions as os_deferred_event_actions
+import charms_openstack.test_utils as test_utils
+
+
+class TestOSDeferredEventActions(test_utils.PatchHelper):
+
+    def setUp(self):
+        super().setUp()
+        self.patch_object(os_deferred_event_actions.hookenv, 'action_get')
+        self.action_config = {}
+        self.action_get.side_effect = lambda x: self.action_config.get(x)
+        self.patch_object(os_deferred_event_actions.hookenv, 'action_fail')
+
+        self.patch_object(
+            os_deferred_event_actions.charms_openstack.charm,
+            'provide_charm_instance')
+        self.charm_instance = mock.MagicMock()
+        self.provide_charm_instance.return_value.__enter__.return_value = \
+            self.charm_instance
+
+    def test_restart_services(self):
+        self.patch_object(
+            os_deferred_event_actions.os_utils,
+            'restart_services_action')
+
+        self.action_config = {
+            'deferred-only': True,
+            'services': ''}
+        os_deferred_event_actions.restart_services(['restart-services'])
+        self.charm_instance._assess_status.assert_called_once_with()
+        self.restart_services_action.assert_called_once_with(
+            deferred_only=True)
+
+        self.charm_instance.reset_mock()
+        self.restart_services_action.reset_mock()
+
+        self.action_config = {
+            'deferred-only': False,
+            'services': 'svcA svcB'}
+        os_deferred_event_actions.restart_services(['restart-services'])
+        self.charm_instance._assess_status.assert_called_once_with()
+        self.restart_services_action.assert_called_once_with(
+            services=['svcA', 'svcB'])
+
+        self.charm_instance.reset_mock()
+        self.restart_services_action.reset_mock()
+
+        self.action_config = {
+            'deferred-only': True,
+            'services': 'svcA svcB'}
+        os_deferred_event_actions.restart_services(['restart-services'])
+        self.action_fail.assert_called_once_with(
+            'Cannot set deferred-only and services')
+
+        self.charm_instance.reset_mock()
+        self.restart_services_action.reset_mock()
+        self.action_fail.reset_mock()
+
+        self.action_config = {
+            'deferred-only': False,
+            'services': ''}
+        os_deferred_event_actions.restart_services(['restart-services'])
+        self.action_fail.assert_called_once_with(
+            'Please specify deferred-only or services')
+
+    def test_show_deferred_events(self):
+        self.patch_object(
+            os_deferred_event_actions.os_utils,
+            'show_deferred_events_action_helper')
+        os_deferred_event_actions.show_deferred_events(
+            ['show-deferred-events'])
+        self.show_deferred_events_action_helper.assert_called_once_with()
+
+    def test_run_deferred_hooks(self):
+        self.patch_object(
+            os_deferred_event_actions.deferred_events,
+            'get_deferred_hooks')
+        self.patch_object(
+            os_deferred_event_actions.reactive,
+            'endpoint_from_flag')
+        self.patch_object(
+            os_deferred_event_actions.reactive,
+            'is_flag_set')
+        self.patch_object(
+            os_deferred_event_actions.charms_openstack.charm,
+            'optional_interfaces')
+        interfaces_mock = mock.MagicMock()
+        self.optional_interfaces.return_value = interfaces_mock
+        self.is_flag_set.return_value = True
+        ovsdb_available = mock.MagicMock()
+        ovsdb_available.db_sb_connection_strs = ['constrA', 'connstrB']
+        self.endpoint_from_flag.return_value = ovsdb_available
+
+        self.get_deferred_hooks.return_value = ['install']
+        os_deferred_event_actions.run_deferred_hooks(['run-deferred-hooks'])
+        self.charm_instance.install.assert_called_once_with(
+            check_deferred_events=False)
+        self.assertFalse(self.charm_instance.configure_ovs.called)
+        self.assertFalse(
+            self.charm_instance.render_with_interfaces.called)
+        self.charm_instance._assess_status.assert_called_once_with()
+
+        self.charm_instance.reset_mock()
+
+        self.get_deferred_hooks.return_value = ['install', 'configure_ovs']
+        os_deferred_event_actions.run_deferred_hooks(['run-deferred-hooks'])
+        self.charm_instance.install.assert_called_once_with(
+            check_deferred_events=False)
+        self.charm_instance.render_with_interfaces.assert_called_once_with(
+            interfaces_mock)
+        self.charm_instance.configure_ovs.assert_called_once_with(
+            'constrA,connstrB',
+            True,
+            check_deferred_events=False)
+        self.charm_instance._assess_status.assert_called_once_with()
+
+        self.charm_instance.reset_mock()
+
+        self.get_deferred_hooks.return_value = []
+        os_deferred_event_actions.run_deferred_hooks(['run-deferred-hooks'])
+        self.assertFalse(self.charm_instance.install.configure_ovs.called)
+        self.assertFalse(self.charm_instance.configure_ovs.called)
+        self.assertFalse(self.charm_instance.render_with_interfaces.called)
+        self.charm_instance._assess_status.assert_called_once_with()

unit_tests/test_lib_charms_ovn_central.py

@@ -539,3 +539,36 @@ class TestOVNCentralCharm(Helper):
         self.NRPE.assert_has_calls([
             mock.call().write(),
         ])
+
+    def test_configure_deferred_restarts(self):
+        self.patch_object(
+            ovn_central.ch_core.hookenv,
+            'config',
+            return_value={'enable-auto-restarts': True})
+        self.patch_object(
+            ovn_central.ch_core.hookenv,
+            'service_name',
+            return_value='myapp')
+        self.patch_object(
+            ovn_central.deferred_events,
+            'configure_deferred_restarts')
+        self.patch_object(ovn_central.os, 'chmod')
+        self.target.configure_deferred_restarts()
+        self.configure_deferred_restarts.assert_called_once_with(
+            ['ovn-central', 'ovn-ovsdb-server-nb', 'ovn-northd',
+             'ovn-ovsdb-server-sb'])
+
+        self.chmod.assert_called_once_with(
+            '/var/lib/charm/myapp/policy-rc.d',
+            493)
+
+    def test_configure_deferred_restarts_unsupported(self):
+        self.patch_object(
+            ovn_central.ch_core.hookenv,
+            'config',
+            return_value={})
+        self.patch_object(
+            ovn_central.deferred_events,
+            'configure_deferred_restarts')
+        self.target.configure_deferred_restarts()
+        self.assertFalse(self.configure_deferred_restarts.called)

unit_tests/test_reactive_ovn_central_handlers.py

@@ -80,6 +80,9 @@ class TestRegisteredHooks(test_utils.TestRegisteredHooks):
                                    'endpoint.nrpe-external-master.changed',
                                    'nrpe-external-master.available',),
             },
+            'when_not': {
+                'configure_deferred_restarts': ('is-update-status-hook',),
+            },
         }
         # test that the hooks were registered via the
         # reactive.ovn_handlers