Support installing galaxy roles from kayobe-config

Adds support for installing Ansible roles from Galaxy based on a
requirements.yml file in the kayobe configuration repository.

Roles are installed during 'kayobe control host bootstrap' and upgraded
during 'kayobe control host upgrade'. Custom roles are defined in a
requirements file at '$KAYOBE_CONFIG_PATH/ansible/requirements.yml'. The
roles will be installed to '$KAYOBE_CONFIG_PATH/ansible/roles/'.

This forms the basis for supporting customisable extensions to the
standard workflows.

Change-Id: I4cd732623fc26986d5814be487c7930501ac7b7c
Story: 2001663
Task: 12599

kayobe/ansible.py

@@ -12,6 +12,7 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+import errno
 import logging
 import os
 import os.path
@@ -20,6 +21,7 @@ import subprocess
 import sys
 import tempfile
 
+from kayobe import exception
 from kayobe import utils
 from kayobe import vault
 
@@ -224,3 +226,39 @@ def config_dump(parsed_args, host=None, hosts=None, var_name=None,
         return hostvars
     finally:
         shutil.rmtree(dump_dir)
+
+
+def install_galaxy_roles(parsed_args, force=False):
+    """Install Ansible Galaxy role dependencies.
+
+    Installs dependencies specified in kayobe, and if present, in kayobe
+    configuration.
+
+    :param parsed_args: Parsed command line arguments.
+    :param force: Whether to force reinstallation of roles.
+    """
+    LOG.info("Installing galaxy role dependencies from kayobe")
+    utils.galaxy_install("requirements.yml", "ansible/roles", force=force)
+
+    # Check for requirements in kayobe configuration.
+    kc_reqs_path = os.path.join(parsed_args.config_path,
+                                "ansible", "requirements.yml")
+    if not utils.is_readable_file(kc_reqs_path)["result"]:
+        LOG.info("Not installing galaxy role dependencies from kayobe config "
+                 "- requirements.yml not present")
+        return
+
+    LOG.info("Installing galaxy role dependencies from kayobe config")
+    # Ensure a roles directory exists in kayobe-config.
+    kc_roles_path = os.path.join(parsed_args.config_path,
+                                 "ansible", "roles")
+    try:
+        os.makedirs(kc_roles_path)
+    except OSError as e:
+        if e.errno != errno.EEXIST:
+            raise exception.Error("Failed to create directory ansible/roles/ "
+                                  "in kayobe configuration at %s: %s" %
+                                  (parsed_args.config_path, str(e)))
+
+    # Install roles from kayobe-config.
+    utils.galaxy_install(kc_reqs_path, kc_roles_path, force=force)

kayobe/cli/commands.py

@@ -19,7 +19,6 @@ from cliff.command import Command
 
 from kayobe import ansible
 from kayobe import kolla_ansible
-from kayobe import utils
 from kayobe import vault
 
 
@@ -120,7 +119,7 @@ class ControlHostBootstrap(KayobeAnsibleMixin, VaultMixin, Command):
 
     def take_action(self, parsed_args):
         self.app.LOG.debug("Bootstrapping Kayobe control host")
-        utils.galaxy_install("requirements.yml", "ansible/roles")
+        ansible.install_galaxy_roles(parsed_args)
         playbooks = _build_playbook_list("bootstrap")
         self.run_kayobe_playbooks(parsed_args, playbooks)
         playbooks = _build_playbook_list("kolla-ansible")
@@ -138,8 +137,7 @@ class ControlHostUpgrade(KayobeAnsibleMixin, VaultMixin, Command):
     def take_action(self, parsed_args):
         self.app.LOG.debug("Upgrading Kayobe control host")
         # Use force to upgrade roles.
-        utils.galaxy_install("requirements.yml", "ansible/roles",
+        ansible.install_galaxy_roles(parsed_args, force=True)
-                             force=True)
         playbooks = _build_playbook_list("bootstrap")
         self.run_kayobe_playbooks(parsed_args, playbooks)
         playbooks = _build_playbook_list("kolla-ansible")

kayobe/exception.py

@@ -0,0 +1,21 @@
+# Copyright (c) 2018 StackHPC Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+class KayobeException(Exception):
+    """Base class for kayobe exceptions."""
+
+
+class Error(KayobeException):
+    """Generic user error."""

kayobe/tests/unit/cli/test_commands.py

@@ -18,8 +18,8 @@ import cliff.app
 import cliff.commandmanager
 import mock
 
+from kayobe import ansible
 from kayobe.cli import commands
-from kayobe import utils
 
 
 class TestApp(cliff.app.App):
@@ -33,7 +33,7 @@ class TestApp(cliff.app.App):
 
 class TestCase(unittest.TestCase):
 
-    @mock.patch.object(utils, "galaxy_install", spec=True)
+    @mock.patch.object(ansible, "install_galaxy_roles", autospec=True)
     @mock.patch.object(commands.KayobeAnsibleMixin,
                        "run_kayobe_playbooks")
     def test_control_host_bootstrap(self, mock_run, mock_install):
@@ -42,8 +42,7 @@ class TestCase(unittest.TestCase):
         parsed_args = parser.parse_args([])
         result = command.run(parsed_args)
         self.assertEqual(0, result)
-        mock_install.assert_called_once_with("requirements.yml",
+        mock_install.assert_called_once_with(parsed_args)
-                                             "ansible/roles")
         expected_calls = [
             mock.call(mock.ANY, ["ansible/bootstrap.yml"]),
             mock.call(mock.ANY, ["ansible/kolla-ansible.yml"],
@@ -51,7 +50,7 @@ class TestCase(unittest.TestCase):
         ]
         self.assertEqual(expected_calls, mock_run.call_args_list)
 
-    @mock.patch.object(utils, "galaxy_install", spec=True)
+    @mock.patch.object(ansible, "install_galaxy_roles", autospec=True)
     @mock.patch.object(commands.KayobeAnsibleMixin,
                        "run_kayobe_playbooks")
     def test_control_host_upgrade(self, mock_run, mock_install):
@@ -60,8 +59,7 @@ class TestCase(unittest.TestCase):
         parsed_args = parser.parse_args([])
         result = command.run(parsed_args)
         self.assertEqual(0, result)
-        mock_install.assert_called_once_with("requirements.yml",
+        mock_install.assert_called_once_with(parsed_args, force=True)
-                                             "ansible/roles", force=True)
         expected_calls = [
             mock.call(mock.ANY, ["ansible/bootstrap.yml"]),
             mock.call(mock.ANY, ["ansible/kolla-ansible.yml"],

kayobe/tests/unit/test_ansible.py

@@ -13,6 +13,7 @@
 # under the License.
 
 import argparse
+import errno
 import os
 import shutil
 import subprocess
@@ -22,6 +23,7 @@ import unittest
 import mock
 
 from kayobe import ansible
+from kayobe import exception
 from kayobe import utils
 from kayobe import vault
 
@@ -306,6 +308,86 @@ class TestCase(unittest.TestCase):
             mock.call(os.path.join(dump_dir, "host2.yml")),
         ])
 
+    @mock.patch.object(utils, 'galaxy_install', autospec=True)
+    @mock.patch.object(utils, 'is_readable_file', autospec=True)
+    @mock.patch.object(os, 'makedirs', autospec=True)
+    def test_install_galaxy_roles(self, mock_mkdirs, mock_is_readable,
+                                  mock_install):
+        parser = argparse.ArgumentParser()
+        ansible.add_args(parser)
+        parsed_args = parser.parse_args([])
+        mock_is_readable.return_value = {"result": False}
+
+        ansible.install_galaxy_roles(parsed_args)
+
+        mock_install.assert_called_once_with("requirements.yml",
+                                             "ansible/roles", force=False)
+        mock_is_readable.assert_called_once_with(
+            "/etc/kayobe/ansible/requirements.yml")
+        self.assertFalse(mock_mkdirs.called)
+
+    @mock.patch.object(utils, 'galaxy_install', autospec=True)
+    @mock.patch.object(utils, 'is_readable_file', autospec=True)
+    @mock.patch.object(os, 'makedirs', autospec=True)
+    def test_install_galaxy_roles_with_kayobe_config(
+            self, mock_mkdirs, mock_is_readable, mock_install):
+        parser = argparse.ArgumentParser()
+        ansible.add_args(parser)
+        parsed_args = parser.parse_args([])
+        mock_is_readable.return_value = {"result": True}
+
+        ansible.install_galaxy_roles(parsed_args)
+
+        expected_calls = [
+            mock.call("requirements.yml", "ansible/roles", force=False),
+            mock.call("/etc/kayobe/ansible/requirements.yml",
+                      "/etc/kayobe/ansible/roles", force=False)]
+        self.assertEqual(expected_calls, mock_install.call_args_list)
+        mock_is_readable.assert_called_once_with(
+            "/etc/kayobe/ansible/requirements.yml")
+        mock_mkdirs.assert_called_once_with("/etc/kayobe/ansible/roles")
+
+    @mock.patch.object(utils, 'galaxy_install', autospec=True)
+    @mock.patch.object(utils, 'is_readable_file', autospec=True)
+    @mock.patch.object(os, 'makedirs', autospec=True)
+    def test_install_galaxy_roles_with_kayobe_config_forced(
+            self, mock_mkdirs, mock_is_readable, mock_install):
+        parser = argparse.ArgumentParser()
+        ansible.add_args(parser)
+        parsed_args = parser.parse_args([])
+        mock_is_readable.return_value = {"result": True}
+
+        ansible.install_galaxy_roles(parsed_args, force=True)
+
+        expected_calls = [
+            mock.call("requirements.yml", "ansible/roles", force=True),
+            mock.call("/etc/kayobe/ansible/requirements.yml",
+                      "/etc/kayobe/ansible/roles", force=True)]
+        self.assertEqual(expected_calls, mock_install.call_args_list)
+        mock_is_readable.assert_called_once_with(
+            "/etc/kayobe/ansible/requirements.yml")
+        mock_mkdirs.assert_called_once_with("/etc/kayobe/ansible/roles")
+
+    @mock.patch.object(utils, 'galaxy_install', autospec=True)
+    @mock.patch.object(utils, 'is_readable_file', autospec=True)
+    @mock.patch.object(os, 'makedirs', autospec=True)
+    def test_install_galaxy_roles_with_kayobe_config_mkdirs_failure(
+            self, mock_mkdirs, mock_is_readable, mock_install):
+        parser = argparse.ArgumentParser()
+        ansible.add_args(parser)
+        parsed_args = parser.parse_args([])
+        mock_is_readable.return_value = {"result": True}
+        mock_mkdirs.side_effect = OSError(errno.EPERM)
+
+        self.assertRaises(exception.Error,
+                          ansible.install_galaxy_roles, parsed_args)
+
+        mock_install.assert_called_once_with("requirements.yml",
+                                             "ansible/roles", force=False)
+        mock_is_readable.assert_called_once_with(
+            "/etc/kayobe/ansible/requirements.yml")
+        mock_mkdirs.assert_called_once_with("/etc/kayobe/ansible/roles")
+
     @mock.patch.object(utils, 'read_file')
     def test__read_vault_password_file(self, mock_read):
         mock_read.return_value = "test-pass\n"

releasenotes/notes/config-galaxy-roles-6bd129824436a983.yaml

@@ -0,0 +1,8 @@
+---
+features:
+  - |
+    Adds support for installing custom Ansible Galaxy roles during ``kayobe
+    control host bootstrap`` and ``kayobe control host upgrade``. Custom roles
+    are defined in a requirements file at
+    ``$KAYOBE_CONFIG_PATH/ansible/requirements.yml``. The roles will be
+    installed to ``$KAYOBE_CONFIG_PATH/ansible/roles/``.