From 30b85243cccffae75bb2aa23ce8b5ba0d02d7c2d Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Fri, 19 May 2017 19:15:12 +0100 Subject: [PATCH] Support vault encrypted config files with kolla-ansible --- kayobe/ansible.py | 31 +------------ kayobe/cli/commands.py | 59 ++++++++++++++++--------- kayobe/kolla_ansible.py | 3 ++ kayobe/tests/unit/test_ansible.py | 9 ++++ kayobe/tests/unit/test_kolla_ansible.py | 52 ++++++++++++++++++++++ kayobe/vault.py | 58 ++++++++++++++++++++++++ 6 files changed, 162 insertions(+), 50 deletions(-) create mode 100644 kayobe/vault.py diff --git a/kayobe/ansible.py b/kayobe/ansible.py index 8b243456d..c2d4654fd 100644 --- a/kayobe/ansible.py +++ b/kayobe/ansible.py @@ -21,40 +21,19 @@ import sys import tempfile from kayobe import utils +from kayobe import vault DEFAULT_CONFIG_PATH = "/etc/kayobe" CONFIG_PATH_ENV = "KAYOBE_CONFIG_PATH" -VAULT_PASSWORD_ENV = "KAYOBE_VAULT_PASSWORD" - LOG = logging.getLogger(__name__) -def _get_default_vault_password_file(): - """Return the default value for the vault password file argument. - - It is possible to use an environment variable to avoid typing the vault - password. - """ - if not os.getenv(VAULT_PASSWORD_ENV): - return None - cmd = ["which", "kayobe-vault-password-helper"] - try: - output = utils.run_command(cmd, check_output=True) - except subprocess.CalledProcessError: - return None - return output.strip() - - def add_args(parser): """Add arguments required for running Ansible playbooks to a parser.""" default_config_path = os.getenv(CONFIG_PATH_ENV, DEFAULT_CONFIG_PATH) - default_vault_password_file = _get_default_vault_password_file() - vault = parser.add_mutually_exclusive_group() - vault.add_argument("--ask-vault-pass", action="store_true", - help="ask for vault password") parser.add_argument("-b", "--become", action="store_true", help="run operations with become (nopasswd implied)") parser.add_argument("-C", "--check", action="store_true", @@ -79,9 +58,6 @@ def add_args(parser): parser.add_argument("-t", "--tags", metavar="TAGS", help="only run plays and tasks tagged with these " "values") - vault.add_argument("--vault-password-file", metavar="VAULT_PASSWORD_FILE", - default=default_vault_password_file, - help="vault password file") def _get_inventory_path(parsed_args): @@ -133,10 +109,7 @@ def build_args(parsed_args, playbooks, cmd = ["ansible-playbook"] if verbose_level: cmd += ["-" + "v" * verbose_level] - if parsed_args.ask_vault_pass: - cmd += ["--ask-vault-pass"] - elif parsed_args.vault_password_file: - cmd += ["--vault-password-file", parsed_args.vault_password_file] + cmd += vault.build_args(parsed_args) inventory = _get_inventory_path(parsed_args) cmd += ["--inventory", inventory] vars_files = _get_vars_files(parsed_args.config_path) diff --git a/kayobe/cli/commands.py b/kayobe/cli/commands.py index d2ed80ef8..bc5fd9317 100644 --- a/kayobe/cli/commands.py +++ b/kayobe/cli/commands.py @@ -21,6 +21,7 @@ from cliff.command import Command from kayobe import ansible from kayobe import kolla_ansible from kayobe import utils +from kayobe import vault def _build_playbook_list(*playbooks): @@ -28,6 +29,16 @@ def _build_playbook_list(*playbooks): return ["ansible/%s.yml" % playbook for playbook in playbooks] +class VaultMixin(object): + """Mixin class for commands requiring Ansible vault.""" + + def get_parser(self, prog_name): + parser = super(VaultMixin, self).get_parser(prog_name) + group = parser.add_argument_group("Ansible vault") + vault.add_args(group) + return parser + + class KayobeAnsibleMixin(object): """Mixin class for commands running Kayobe Ansible playbooks.""" @@ -100,7 +111,7 @@ class KollaAnsibleMixin(object): return kolla_ansible.run_seed(*args, **kwargs) -class ControlHostBootstrap(KayobeAnsibleMixin, Command): +class ControlHostBootstrap(KayobeAnsibleMixin, VaultMixin, Command): """Bootstrap the Kayobe control environment.""" def take_action(self, parsed_args): @@ -123,7 +134,7 @@ class ControlHostBootstrap(KayobeAnsibleMixin, Command): self.run_kayobe_playbooks(parsed_args, playbooks) -class ConfigurationDump(KayobeAnsibleMixin, Command): +class ConfigurationDump(KayobeAnsibleMixin, VaultMixin, Command): """Dump Kayobe configuration.""" def get_parser(self, prog_name): @@ -153,7 +164,7 @@ class ConfigurationDump(KayobeAnsibleMixin, Command): sys.exit(1) -class PlaybookRun(KayobeAnsibleMixin, Command): +class PlaybookRun(KayobeAnsibleMixin, VaultMixin, Command): """Run a Kayobe Ansible playbook.""" def add_kayobe_ansible_args(self, group): @@ -166,7 +177,7 @@ class PlaybookRun(KayobeAnsibleMixin, Command): self.run_kayobe_playbooks(parsed_args, parsed_args.playbook) -class KollaAnsibleRun(KollaAnsibleMixin, Command): +class KollaAnsibleRun(KollaAnsibleMixin, VaultMixin, Command): """Run a Kolla Ansible command.""" def add_kolla_ansible_args(self, group): @@ -185,7 +196,7 @@ class KollaAnsibleRun(KollaAnsibleMixin, Command): parsed_args.kolla_inventory_filename) -class PhysicalNetworkConfigure(KayobeAnsibleMixin, Command): +class PhysicalNetworkConfigure(KayobeAnsibleMixin, VaultMixin, Command): """Configure a set of physical network devices.""" def get_parser(self, prog_name): @@ -208,7 +219,8 @@ class PhysicalNetworkConfigure(KayobeAnsibleMixin, Command): extra_vars=extra_vars) -class SeedVMProvision(KollaAnsibleMixin, KayobeAnsibleMixin, Command): +class SeedVMProvision(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin, + Command): """Provision the seed VM.""" def take_action(self, parsed_args): @@ -221,7 +233,8 @@ class SeedVMProvision(KollaAnsibleMixin, KayobeAnsibleMixin, Command): tags="config") -class SeedHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin, Command): +class SeedHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin, + Command): """Configure the seed node host OS.""" def get_parser(self, prog_name): @@ -251,7 +264,8 @@ class SeedHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin, Command): self.run_kayobe_playbooks(parsed_args, playbooks, limit="seed") -class SeedServiceDeploy(KollaAnsibleMixin, KayobeAnsibleMixin, Command): +class SeedServiceDeploy(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin, + Command): """Deploy the seed services.""" def take_action(self, parsed_args): @@ -263,7 +277,7 @@ class SeedServiceDeploy(KollaAnsibleMixin, KayobeAnsibleMixin, Command): self.run_kayobe_playbooks(parsed_args, playbooks) -class SeedContainerImageBuild(KayobeAnsibleMixin, Command): +class SeedContainerImageBuild(KayobeAnsibleMixin, VaultMixin, Command): """Build the seed container images.""" def get_parser(self, prog_name): @@ -290,7 +304,7 @@ class SeedContainerImageBuild(KayobeAnsibleMixin, Command): extra_vars=extra_vars) -class OvercloudInventoryDiscover(KayobeAnsibleMixin, Command): +class OvercloudInventoryDiscover(KayobeAnsibleMixin, VaultMixin, Command): """Discover the overcloud inventory from the seed's Ironic service.""" def take_action(self, parsed_args): @@ -307,7 +321,7 @@ class OvercloudInventoryDiscover(KayobeAnsibleMixin, Command): tags="config") -class OvercloudBIOSRAIDConfigure(KayobeAnsibleMixin, Command): +class OvercloudBIOSRAIDConfigure(KayobeAnsibleMixin, VaultMixin, Command): """Configure BIOS and RAID for the overcloud hosts.""" def take_action(self, parsed_args): @@ -316,7 +330,7 @@ class OvercloudBIOSRAIDConfigure(KayobeAnsibleMixin, Command): self.run_kayobe_playbooks(parsed_args, playbooks) -class OvercloudHardwareInspect(KayobeAnsibleMixin, Command): +class OvercloudHardwareInspect(KayobeAnsibleMixin, VaultMixin, Command): """Inspect the overcloud hardware using ironic inspector.""" def take_action(self, parsed_args): @@ -325,7 +339,7 @@ class OvercloudHardwareInspect(KayobeAnsibleMixin, Command): self.run_kayobe_playbooks(parsed_args, playbooks) -class OvercloudProvision(KayobeAnsibleMixin, Command): +class OvercloudProvision(KayobeAnsibleMixin, VaultMixin, Command): """Provision the overcloud.""" def take_action(self, parsed_args): @@ -334,7 +348,7 @@ class OvercloudProvision(KayobeAnsibleMixin, Command): self.run_kayobe_playbooks(parsed_args, playbooks) -class OvercloudDeprovision(KayobeAnsibleMixin, Command): +class OvercloudDeprovision(KayobeAnsibleMixin, VaultMixin, Command): """Deprovision the overcloud.""" def take_action(self, parsed_args): @@ -343,7 +357,8 @@ class OvercloudDeprovision(KayobeAnsibleMixin, Command): self.run_kayobe_playbooks(parsed_args, playbooks) -class OvercloudHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin, Command): +class OvercloudHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin, + Command): """Configure the overcloud host OS.""" def get_parser(self, prog_name): @@ -373,7 +388,8 @@ class OvercloudHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin, Command): self.run_kayobe_playbooks(parsed_args, playbooks, limit="controllers") -class OvercloudServiceDeploy(KollaAnsibleMixin, KayobeAnsibleMixin, Command): +class OvercloudServiceDeploy(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin, + Command): """Deploy the overcloud services.""" def take_action(self, parsed_args): @@ -394,7 +410,7 @@ class OvercloudServiceDeploy(KollaAnsibleMixin, KayobeAnsibleMixin, Command): class OvercloudServiceReconfigure(KollaAnsibleMixin, KayobeAnsibleMixin, - Command): + VaultMixin, Command): """Reconfigure the overcloud services.""" def take_action(self, parsed_args): @@ -414,7 +430,8 @@ class OvercloudServiceReconfigure(KollaAnsibleMixin, KayobeAnsibleMixin, self.run_kayobe_playbooks(parsed_args, playbooks) -class OvercloudServiceUpgrade(KollaAnsibleMixin, KayobeAnsibleMixin, Command): +class OvercloudServiceUpgrade(KollaAnsibleMixin, KayobeAnsibleMixin, + VaultMixin, Command): """Upgrade the overcloud services.""" def take_action(self, parsed_args): @@ -425,7 +442,7 @@ class OvercloudServiceUpgrade(KollaAnsibleMixin, KayobeAnsibleMixin, Command): self.run_kolla_ansible_overcloud(parsed_args, command) -class OvercloudContainerImagePull(KollaAnsibleMixin, Command): +class OvercloudContainerImagePull(KollaAnsibleMixin, VaultMixin, Command): """Pull the overcloud container images from a registry.""" def take_action(self, parsed_args): @@ -433,7 +450,7 @@ class OvercloudContainerImagePull(KollaAnsibleMixin, Command): self.run_kolla_ansible_overcloud(parsed_args, "pull") -class OvercloudContainerImageBuild(KayobeAnsibleMixin, Command): +class OvercloudContainerImageBuild(KayobeAnsibleMixin, VaultMixin, Command): """Build the overcloud container images.""" def get_parser(self, prog_name): @@ -460,7 +477,7 @@ class OvercloudContainerImageBuild(KayobeAnsibleMixin, Command): extra_vars=extra_vars) -class OvercloudPostConfigure(KayobeAnsibleMixin, Command): +class OvercloudPostConfigure(KayobeAnsibleMixin, VaultMixin, Command): """Perform post-deployment configuration.""" def take_action(self, parsed_args): diff --git a/kayobe/kolla_ansible.py b/kayobe/kolla_ansible.py index 6042c8717..4600a6fdd 100644 --- a/kayobe/kolla_ansible.py +++ b/kayobe/kolla_ansible.py @@ -19,6 +19,7 @@ import subprocess import sys from kayobe import utils +from kayobe import vault DEFAULT_CONFIG_PATH = "/etc/kolla" @@ -96,6 +97,8 @@ def build_args(parsed_args, command, inventory_filename, extra_vars=None, cmd += ["kolla-ansible", command] if verbose_level: cmd += ["-" + "v" * verbose_level] + if parsed_args.vault_password_file: + cmd += ["--key", parsed_args.vault_password_file] inventory = _get_inventory_path(parsed_args, inventory_filename) cmd += ["--inventory", inventory] if parsed_args.kolla_config_path != DEFAULT_CONFIG_PATH: diff --git a/kayobe/tests/unit/test_ansible.py b/kayobe/tests/unit/test_ansible.py index 72bb96881..7ef1f8ed7 100644 --- a/kayobe/tests/unit/test_ansible.py +++ b/kayobe/tests/unit/test_ansible.py @@ -23,6 +23,7 @@ import mock from kayobe import ansible from kayobe import utils +from kayobe import vault class TestCase(unittest.TestCase): @@ -35,6 +36,7 @@ class TestCase(unittest.TestCase): "/etc/kayobe/vars-file2.yaml"] parser = argparse.ArgumentParser() ansible.add_args(parser) + vault.add_args(parser) parsed_args = parser.parse_args([]) ansible.run_playbooks(parsed_args, ["playbook1.yml", "playbook2.yml"]) expected_cmd = [ @@ -57,6 +59,7 @@ class TestCase(unittest.TestCase): "/path/to/config/vars-file2.yaml"] parser = argparse.ArgumentParser() ansible.add_args(parser) + vault.add_args(parser) args = [ "-b", "-C", @@ -93,6 +96,7 @@ class TestCase(unittest.TestCase): "/path/to/config/vars-file2.yaml"] parser = argparse.ArgumentParser() ansible.add_args(parser) + vault.add_args(parser) args = [ "--ask-vault-pass", "--become", @@ -130,6 +134,7 @@ class TestCase(unittest.TestCase): mock_vars.return_value = [] parser = argparse.ArgumentParser() ansible.add_args(parser) + vault.add_args(parser) args = [ "--vault-password-file", "/path/to/vault/pw", ] @@ -153,6 +158,7 @@ class TestCase(unittest.TestCase): parser = argparse.ArgumentParser() mock_run.return_value = "/path/to/kayobe-vault-password-helper" ansible.add_args(parser) + vault.add_args(parser) mock_run.assert_called_once_with( ["which", "kayobe-vault-password-helper"], check_output=True) mock_run.reset_mock() @@ -174,6 +180,7 @@ class TestCase(unittest.TestCase): mock_vars.return_value = [] parser = argparse.ArgumentParser() ansible.add_args(parser) + vault.add_args(parser) args = [ "--ask-vault-pass", "--vault-password-file", "/path/to/vault/pw", @@ -188,6 +195,7 @@ class TestCase(unittest.TestCase): "/etc/kayobe/vars-file2.yaml"] parser = argparse.ArgumentParser() ansible.add_args(parser) + vault.add_args(parser) args = [ "--extra-vars", "ev_name1=ev_value1", "--limit", "group1:host1", @@ -223,6 +231,7 @@ class TestCase(unittest.TestCase): def test_run_playbooks_failure(self, mock_validate, mock_vars, mock_run): parser = argparse.ArgumentParser() ansible.add_args(parser) + vault.add_args(parser) parsed_args = parser.parse_args([]) mock_run.side_effect = subprocess.CalledProcessError(1, "dummy") self.assertRaises(SystemExit, diff --git a/kayobe/tests/unit/test_kolla_ansible.py b/kayobe/tests/unit/test_kolla_ansible.py index 97eba5b3f..10d3e9dc6 100644 --- a/kayobe/tests/unit/test_kolla_ansible.py +++ b/kayobe/tests/unit/test_kolla_ansible.py @@ -13,6 +13,7 @@ # under the License. import argparse +import os import subprocess import unittest @@ -20,6 +21,7 @@ import mock from kayobe import kolla_ansible from kayobe import utils +from kayobe import vault class TestCase(unittest.TestCase): @@ -29,6 +31,7 @@ class TestCase(unittest.TestCase): def test_run(self, mock_validate, mock_run): parser = argparse.ArgumentParser() kolla_ansible.add_args(parser) + vault.add_args(parser) parsed_args = parser.parse_args([]) kolla_ansible.run(parsed_args, "command", "overcloud") expected_cmd = [ @@ -44,6 +47,7 @@ class TestCase(unittest.TestCase): def test_run_all_the_args(self, mock_validate, mock_run): parser = argparse.ArgumentParser() kolla_ansible.add_args(parser) + vault.add_args(parser) args = [ "--kolla-config-path", "/path/to/config", "-ke", "ev_name1=ev_value1", @@ -69,7 +73,9 @@ class TestCase(unittest.TestCase): def test_run_all_the_long_args(self, mock_validate, mock_run): parser = argparse.ArgumentParser() kolla_ansible.add_args(parser) + vault.add_args(parser) args = [ + "--ask-vault-pass", "--kolla-config-path", "/path/to/config", "--kolla-extra-vars", "ev_name1=ev_value1", "--kolla-inventory", "/path/to/inventory", @@ -80,6 +86,7 @@ class TestCase(unittest.TestCase): expected_cmd = [ "source", "ansible/kolla-venv/bin/activate", "&&", "kolla-ansible", "command", + "--ask-vault-pass", "--inventory", "/path/to/inventory", "--configdir", "/path/to/config", "--passwords", "/path/to/config/passwords.yml", @@ -89,11 +96,55 @@ class TestCase(unittest.TestCase): expected_cmd = " ".join(expected_cmd) mock_run.assert_called_once_with(expected_cmd, shell=True, quiet=False) + @mock.patch.object(utils, "run_command") + @mock.patch.object(kolla_ansible, "_validate_args") + def test_run_vault_password_file(self, mock_validate, mock_run): + parser = argparse.ArgumentParser() + kolla_ansible.add_args(parser) + vault.add_args(parser) + args = [ + "--vault-password-file", "/path/to/vault/pw", + ] + parsed_args = parser.parse_args(args) + kolla_ansible.run(parsed_args, "command", "overcloud") + expected_cmd = [ + "source", "ansible/kolla-venv/bin/activate", "&&", + "kolla-ansible", "command", + "--vault-password-file", "/path/to/vault/pw", + "--inventory", "/etc/kolla/inventory/overcloud", + ] + expected_cmd = " ".join(expected_cmd) + mock_run.assert_called_once_with(expected_cmd, shell=True, quiet=False) + + @mock.patch.dict(os.environ, {"KAYOBE_VAULT_PASSWORD": "test-pass"}) + @mock.patch.object(utils, "run_command") + @mock.patch.object(kolla_ansible, "_validate_args") + def test_run_vault_password_helper(self, mock_vars, mock_run): + mock_vars.return_value = [] + parser = argparse.ArgumentParser() + mock_run.return_value = "/path/to/kayobe-vault-password-helper" + kolla_ansible.add_args(parser) + vault.add_args(parser) + mock_run.assert_called_once_with( + ["which", "kayobe-vault-password-helper"], check_output=True) + mock_run.reset_mock() + parsed_args = parser.parse_args([]) + kolla_ansible.run(parsed_args, "command", "overcloud") + expected_cmd = [ + "source", "ansible/kolla-venv/bin/activate", "&&", + "kolla-ansible", "command", + "--vault-password-file", "/path/to/kayobe-vault-password-helper", + "--inventory", "/etc/kolla/inventory/overcloud", + ] + expected_cmd = " ".join(expected_cmd) + mock_run.assert_called_once_with(expected_cmd, shell=True, quiet=False) + @mock.patch.object(utils, "run_command") @mock.patch.object(kolla_ansible, "_validate_args") def test_run_func_args(self, mock_validate, mock_run): parser = argparse.ArgumentParser() kolla_ansible.add_args(parser) + vault.add_args(parser) args = [ "--kolla-extra-vars", "ev_name1=ev_value1", "--kolla-tags", "tag1,tag2", @@ -122,6 +173,7 @@ class TestCase(unittest.TestCase): def test_run_failure(self, mock_validate, mock_run): parser = argparse.ArgumentParser() kolla_ansible.add_args(parser) + vault.add_args(parser) parsed_args = parser.parse_args([]) mock_run.side_effect = subprocess.CalledProcessError(1, "dummy") self.assertRaises(SystemExit, diff --git a/kayobe/vault.py b/kayobe/vault.py new file mode 100644 index 000000000..10684ca87 --- /dev/null +++ b/kayobe/vault.py @@ -0,0 +1,58 @@ +# Copyright (c) 2017 StackHPC Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os +import subprocess + +from kayobe import utils + + +VAULT_PASSWORD_ENV = "KAYOBE_VAULT_PASSWORD" + + +def _get_default_vault_password_file(): + """Return the default value for the vault password file argument. + + It is possible to use an environment variable to avoid typing the vault + password. + """ + if not os.getenv(VAULT_PASSWORD_ENV): + return None + cmd = ["which", "kayobe-vault-password-helper"] + try: + output = utils.run_command(cmd, check_output=True) + except subprocess.CalledProcessError: + return None + return output.strip() + + +def add_args(parser): + """Add arguments required for running Ansible playbooks to a parser.""" + default_vault_password_file = _get_default_vault_password_file() + vault = parser.add_mutually_exclusive_group() + vault.add_argument("--ask-vault-pass", action="store_true", + help="ask for vault password") + vault.add_argument("--vault-password-file", metavar="VAULT_PASSWORD_FILE", + default=default_vault_password_file, + help="vault password file") + + +def build_args(parsed_args): + """Build a list of command line arguments for use with ansible-playbook.""" + cmd = [] + if parsed_args.ask_vault_pass: + cmd += ["--ask-vault-pass"] + elif parsed_args.vault_password_file: + cmd += ["--vault-password-file", parsed_args.vault_password_file] + return cmd