Prune unused Galaxy roles during upgrade

The 'kayobe control host upgrade' command updates the installed Ansible Galaxy
roles based on requirements.yml. Sometimes roles are removed from that file,
but there is currently no way of removing them from the local system.

Normally this causes no problems, but due to the upstream role containing
symlinks with whitespace, we are switching out yatesr.timezone to a
stackhpc.timezone fork. In order to make upgrades work, we need to ensure the
old role is removed. It also makes sense to clean up old roles generally.

This change adds support for removing stale roles during control host upgrade,
currently including the following roles:

stackhpc.os-flavors
stackhpc.os-projects
stackhpc.parted-1.1
yatesr.timezone

Change-Id: I174c7e6f19cbefda56777229a2441bf6469c0982
Story: 2004252
Task: 29166

kayobe/ansible.py

@@ -254,3 +254,19 @@ def install_galaxy_roles(parsed_args, force=False):
 
     # Install roles from kayobe-config.
     utils.galaxy_install(kc_reqs_path, kc_roles_path, force=force)
+
+
+def prune_galaxy_roles(parsed_args):
+    """Prune galaxy roles that are no longer necessary.
+
+    :param parsed_args: Parsed command line arguments.
+    """
+    LOG.info("Removing unnecessary galaxy roles from kayobe")
+    roles_to_remove = [
+        'stackhpc.os-flavors',
+        'stackhpc.os-projects',
+        'stackhpc.parted-1-1',
+        'yatesr.timezone',
+    ]
+    LOG.debug("Removing roles: %s", ",".join(roles_to_remove))
+    utils.galaxy_remove(roles_to_remove, "ansible/roles")

kayobe/cli/commands.py

@@ -136,6 +136,9 @@ class ControlHostUpgrade(KayobeAnsibleMixin, VaultMixin, Command):
 
     def take_action(self, parsed_args):
         self.app.LOG.debug("Upgrading Kayobe Ansible control host")
+        # Remove roles that are no longer used. Do this before installing new
+        # ones, just in case a custom role dependency includes any.
+        ansible.prune_galaxy_roles(parsed_args)
         # Use force to upgrade roles.
         ansible.install_galaxy_roles(parsed_args, force=True)
         playbooks = _build_playbook_list("bootstrap")

kayobe/tests/unit/cli/test_commands.py

@@ -51,15 +51,17 @@ class TestCase(unittest.TestCase):
         self.assertEqual(expected_calls, mock_run.call_args_list)
 
     @mock.patch.object(ansible, "install_galaxy_roles", autospec=True)
+    @mock.patch.object(ansible, "prune_galaxy_roles", autospec=True)
     @mock.patch.object(commands.KayobeAnsibleMixin,
                        "run_kayobe_playbooks")
-    def test_control_host_upgrade(self, mock_run, mock_install):
+    def test_control_host_upgrade(self, mock_run, mock_prune, mock_install):
         command = commands.ControlHostUpgrade(TestApp(), [])
         parser = command.get_parser("test")
         parsed_args = parser.parse_args([])
         result = command.run(parsed_args)
         self.assertEqual(0, result)
         mock_install.assert_called_once_with(parsed_args, force=True)
+        mock_prune.assert_called_once_with(parsed_args)
         expected_calls = [
             mock.call(mock.ANY, ["ansible/bootstrap.yml"]),
             mock.call(mock.ANY, ["ansible/kolla-ansible.yml"],

kayobe/tests/unit/test_ansible.py

@@ -389,3 +389,20 @@ class TestCase(unittest.TestCase):
         mock_is_readable.assert_called_once_with(
             "/etc/kayobe/ansible/requirements.yml")
         mock_mkdirs.assert_called_once_with("/etc/kayobe/ansible/roles")
+
+    @mock.patch.object(utils, 'galaxy_remove', autospec=True)
+    def test_prune_galaxy_roles(self, mock_remove):
+        parser = argparse.ArgumentParser()
+        ansible.add_args(parser)
+        parsed_args = parser.parse_args([])
+
+        ansible.prune_galaxy_roles(parsed_args)
+
+        expected_roles = [
+            'stackhpc.os-flavors',
+            'stackhpc.os-projects',
+            'stackhpc.parted-1-1',
+            'yatesr.timezone',
+        ]
+        mock_remove.assert_called_once_with(expected_roles,
+                                            "ansible/roles")

kayobe/tests/unit/test_utils.py

@@ -48,6 +48,20 @@ class TestCase(unittest.TestCase):
                           utils.galaxy_install, "/path/to/role/file",
                           "/path/to/roles")
 
+    @mock.patch.object(utils, "run_command")
+    def test_galaxy_remove(self, mock_run):
+        utils.galaxy_remove(["role1", "role2"], "/path/to/roles")
+        mock_run.assert_called_once_with(["ansible-galaxy", "remove",
+                                          "--roles-path", "/path/to/roles",
+                                          "role1", "role2"])
+
+    @mock.patch.object(utils, "run_command")
+    def test_galaxy_remove_failure(self, mock_run):
+        mock_run.side_effect = subprocess.CalledProcessError(1, "command")
+        self.assertRaises(SystemExit,
+                          utils.galaxy_install, ["role1", "role2"],
+                          "/path/to/roles")
+
     @mock.patch.object(utils, "read_file")
     def test_read_yaml_file(self, mock_read):
         mock_read.return_value = """---

kayobe/utils.py

@@ -51,6 +51,21 @@ def galaxy_install(role_file, roles_path, force=False):
         sys.exit(e.returncode)
 
 
+def galaxy_remove(roles_to_remove, roles_path):
+
+    """Remove Ansible roles via Ansible Galaxy."""
+    cmd = ["ansible-galaxy", "remove"]
+    cmd += ["--roles-path", roles_path]
+    cmd += roles_to_remove
+    try:
+        run_command(cmd)
+    except subprocess.CalledProcessError as e:
+        LOG.error("Failed to remove Ansible roles %s via Ansible "
+                  "Galaxy: returncode %d",
+                  ",".join(roles_to_remove), e.returncode)
+        sys.exit(e.returncode)
+
+
 def read_file(path, mode="r"):
     """Read the content of a file."""
     with open(path, mode) as f: