Implementation of deferred restarts

Add deferred restart actions and config.

Change-Id: I6aeceae10b7a25ba5bfd6f2cb07f510a9481d0ba

actions.yaml

@@ -24,3 +24,30 @@ resume:
     descrpition: Resume the neutron-gateway unit.
 security-checklist:
   description: Validate the running configuration against the OpenStack security guides checklist
+restart-services:
+  description: |
+    Restarts services this charm manages.
+  params:
+    deferred-only:
+      type: boolean
+      default: false
+      description: |
+        Restart all deferred services.
+    services:
+      type: string
+      default: ""
+      description: |
+        List of services to restart.
+    run-hooks:
+      type: boolean
+      default: true
+      description: |
+        Run any hooks which have been deferred.
+run-deferred-hooks:
+  description: |
+    Run deferable hooks and restart services.
+    .
+    NOTE: Service will be restarted as needed irrespective of enable-auto-restarts
+show-deferred-events:
+    descrpition: |
+        Show the outstanding restarts

actions/actions.py

@@ -15,8 +15,15 @@ def _add_path(path):
 _add_path(_hooks_dir)
 
 
-from charmhelpers.core.hookenv import action_fail
+import charmhelpers.contrib.openstack.utils as os_utils
+from charmhelpers.core.hookenv import (
+    DEBUG,
+    action_get,
+    action_fail,
+    log,
+)
 from neutron_utils import (
+    assess_status,
     pause_unit_helper,
     resume_unit_helper,
     register_configs,
@@ -36,9 +43,55 @@ def resume(args):
     resume_unit_helper(register_configs())
 
 
+def restart(args):
+    """Restart services.
+
+    :param args: Unused
+    :type args: List[str]
+    """
+    deferred_only = action_get("deferred-only")
+    services = action_get("services").split()
+    # Check input
+    if deferred_only and services:
+        action_fail("Cannot set deferred-only and services")
+        return
+    if not (deferred_only or services):
+        action_fail("Please specify deferred-only or services")
+        return
+    if action_get('run-hooks'):
+        log("Charm does not defer any hooks at present", DEBUG)
+    if deferred_only:
+        os_utils.restart_services_action(deferred_only=True)
+    else:
+        os_utils.restart_services_action(services=services)
+    assess_status(register_configs())
+
+
+def run_deferred_hooks(args):
+    """Run deferred hooks.
+
+    :param args: Unused
+    :type args: List[str]
+    """
+    # Charm defers restarts on a case-by-case basis so no full
+    # hook deferalls are needed.
+    action_fail("Charm does not defer any hooks at present")
+
+
+def show_deferred_events(args):
+    """Show the deferred events.
+
+    :param args: Unused
+    :type args: List[str]
+    """
+    os_utils.show_deferred_events_action_helper()
+
+
 # A dictionary of all the defined actions to callables (which take
 # parsed arguments).
-ACTIONS = {"pause": pause, "resume": resume}
+ACTIONS = {"pause": pause, "resume": resume, "restart-services": restart,
+           "show-deferred-events": show_deferred_events,
+           "run-deferred-hooks": run_deferred_hooks}
 
 
 def main(args):

actions/restart-services

@@ -0,0 +1 @@
+actions.py

actions/run-deferred-hooks

@@ -0,0 +1 @@
+actions.py

actions/show-deferred-events

@@ -0,0 +1 @@
+actions.py

config.yaml

@@ -382,3 +382,9 @@ options:
       by sending a ping and if that fails they trigger a vrrp transition. This
       option defines how frequently this check is performed. Setting this value
       to 0 will disable the healthchecks.
+  enable-auto-restarts:
+    type: boolean
+    default: True
+    description: |
+      Allow the charm and packages to restart services automatically when
+      required.

hooks/neutron_hooks.py

@@ -12,8 +12,11 @@ from charmhelpers.core.hookenv import (
     UnregisteredHookError,
     status_set,
 )
-from charmhelpers.core.host import service_restart
 from charmhelpers.core.unitdata import kv
+from charmhelpers.contrib.openstack.deferred_events import (
+    configure_deferred_restarts,
+    deferrable_svc_restart,
+)
 from charmhelpers.fetch import (
     apt_update,
     apt_install,
@@ -33,8 +36,8 @@ from charmhelpers.contrib.hahelpers.apache import (
 from charmhelpers.contrib.openstack.utils import (
     configure_installation_source,
     openstack_upgrade_available,
-    pausable_restart_on_change as restart_on_change,
     is_unit_paused_set,
+    os_restart_on_change as restart_on_change,
     series_upgrade_prepare,
     series_upgrade_complete,
 )
@@ -76,6 +79,7 @@ from neutron_utils import (
     disable_neutron_lbaas,
     remove_old_packages,
     deprecated_services,
+    deferrable_services,
 )
 
 hooks = Hooks()
@@ -140,6 +144,7 @@ def install():
 @restart_on_change(restart_map)
 @harden()
 def config_changed():
+    configure_deferred_restarts(deferrable_services())
     if not config('action-managed-upgrade'):
         if openstack_upgrade_available(NEUTRON_COMMON):
             status_set('maintenance', 'Running openstack upgrade')
@@ -205,7 +210,7 @@ def upgrade_charm():
     if packages_removed and not is_unit_paused_set():
         log("Package purge detected, restarting services", "INFO")
         for s in services():
-            service_restart(s)
+            deferrable_svc_restart(s, 'Package purge detected')
     config_changed()
     update_legacy_ha_files(force=True)
 
@@ -288,7 +293,9 @@ def nm_changed():
             previous_nonce = db.get('restart_nonce')
             if previous_nonce != restart_nonce:
                 if not is_unit_paused_set():
-                    service_restart('nova-api-metadata')
+                    deferrable_svc_restart(
+                        'nova-api-metadata',
+                        'Restart trigger received')
                 db.set('restart_nonce', restart_nonce)
                 db.flush()
     # LP: #1812813

hooks/neutron_utils.py

@@ -803,6 +803,26 @@ def services():
     return list(set(_services))
 
 
+def deferrable_services():
+    """Services which should be stopped from restarting.
+
+    All services from services() are deferable. But the charm may
+    install a package which install a service that the charm does not add
+    to its restart_map. In that case it will be missing from
+    self.services. However one of the jobs of deferred events is to ensure
+    that packages updates outside of charms also do not restart services.
+    To ensure there is a complete list take the services from services{}
+    and also add in a known list of networking services.
+
+    NOTE: It does not matter if one of the services in the list is not
+    installed on the system.
+    """
+    _svcs = services()
+    _svcs.extend(['ovs-vswitchd', 'ovsdb-server',
+                  'openvswitch-switch'])
+    return list(set(_svcs))
+
+
 def do_openstack_upgrade(configs):
     """
     Perform an upgrade.  Takes care of upgrading packages, rewriting

tests/tests.yaml

@@ -60,12 +60,14 @@ configure_options:
   configure_gateway_ext_port_use_juju_wait: false
 
 tests:
+  - zaza.openstack.charm_tests.neutron.tests.NeutronGatewayDeferredRestartTest
   - zaza.openstack.charm_tests.neutron.tests.NeutronGatewayTest
   - zaza.openstack.charm_tests.neutron.tests.SecurityTest
   - zaza.openstack.charm_tests.neutron.tests.NeutronNetworkingTest
   - zaza.openstack.charm_tests.neutron.tests.NeutronOvsVsctlTest
   - zaza.openstack.charm_tests.neutron.tests.NeutronBridgePortMappingTest
   - migrate-ovn:
+    - zaza.openstack.charm_tests.neutron.tests.NeutronGatewayDeferredRestartTest
     - zaza.openstack.charm_tests.neutron.tests.NeutronGatewayTest
     - zaza.openstack.charm_tests.neutron.tests.SecurityTest
     - zaza.openstack.charm_tests.neutron.tests.NeutronOvsVsctlTest

unit_tests/test_neutron_hooks.py

@@ -50,7 +50,7 @@ TO_PATCH = [
     'stop_neutron_ha_monitor_daemon',
     'use_l3ha',
     'kv',
-    'service_restart',
+    'deferrable_svc_restart',
     'install_systemd_override',
     'configure_apparmor',
     'disable_nova_metadata',
@@ -60,6 +60,8 @@ TO_PATCH = [
     'services',
     'remove_old_packages',
     'is_container',
+    'configure_deferred_restarts',
+    'deferrable_services',
     'charmhelpers.contrib.openstack.utils.is_unit_paused_set',
 ]
 
@@ -236,7 +238,9 @@ class TestQuantumHooks(CharmTestCase):
         self.assertTrue(_install.called)
         self.assertTrue(_config_changed.called)
         self.assertTrue(self.install_systemd_override.called)
-        self.service_restart.assert_called_once_with('neutron-metadata-agent')
+        self.deferrable_svc_restart.assert_called_once_with(
+            'neutron-metadata-agent',
+            'Package purge detected')
 
     def test_amqp_joined(self):
         self._call_hook('amqp-relation-joined')
@@ -326,7 +330,9 @@ class TestQuantumHooks(CharmTestCase):
         self._call_hook('quantum-network-service-relation-changed')
         self.assertTrue(self.CONFIGS.write_all.called)
         self.install_ca_cert.assert_called_with('cert')
-        self.service_restart.assert_called_with('nova-api-metadata')
+        self.deferrable_svc_restart.assert_called_with(
+            'nova-api-metadata',
+            'Restart trigger received')
         kv_mock.get.assert_called_with('restart_nonce')
         kv_mock.set.assert_called_with('restart_nonce',
                                        '1111111222222333333')
@@ -365,7 +371,9 @@ class TestQuantumHooks(CharmTestCase):
         self._call_hook('quantum-network-service-relation-changed')
         self.assertTrue(self.CONFIGS.write_all.called)
         self.install_ca_cert.assert_called_with('cert')
-        self.service_restart.assert_called_with('nova-api-metadata')
+        self.deferrable_svc_restart.assert_called_with(
+            'nova-api-metadata',
+            'Restart trigger received')
         kv_mock.get.assert_called_with('restart_nonce')
         kv_mock.set.assert_called_with('restart_nonce',
                                        '1111111222222333333')
@@ -405,7 +413,7 @@ class TestQuantumHooks(CharmTestCase):
         self._call_hook('quantum-network-service-relation-changed')
         self.assertTrue(self.CONFIGS.write_all.called)
         self.install_ca_cert.assert_called_with('cert')
-        self.assertFalse(self.service_restart.called)
+        self.assertFalse(self.deferrable_svc_restart.called)
         kv_mock.get.assert_called_with('restart_nonce')
         self.assertFalse(kv_mock.set.called)
         self.assertFalse(kv_mock.flush.called)