From 6bf74aa20d268f11f676a0e9affa92e3022b595d Mon Sep 17 00:00:00 2001 From: Scott Solkhon Date: Tue, 25 May 2021 11:20:26 +0100 Subject: [PATCH] Support storing passwords in Hashicorp Vault This commit adds two new cli commands to allow an operator to read and write passwords into a configured Hashicorp Vault KV. Change-Id: Icf0eaf7544fcbdf7b83f697cc711446f47118a4d --- doc/source/user/operating-kolla.rst | 27 ++++ kolla_ansible/cmd/readpwd.py | 118 +++++++++++++++++ kolla_ansible/cmd/writepwd.py | 120 ++++++++++++++++++ kolla_ansible/hashi_vault.py | 64 ++++++++++ ...-for-kolla-passwords-76a5b4ece6a4df07.yaml | 7 + requirements.txt | 3 + setup.cfg | 2 + tests/run-hashi-vault.yml | 75 +++++++++++ tests/test-hashicorp-vault-passwords.sh | 68 ++++++++++ tools/read_passwords.py | 1 + tools/write_passwords.py | 1 + zuul.d/base.yaml | 22 ++++ zuul.d/jobs.yaml | 7 + zuul.d/project.yaml | 1 + 14 files changed, 516 insertions(+) create mode 100755 kolla_ansible/cmd/readpwd.py create mode 100755 kolla_ansible/cmd/writepwd.py create mode 100644 kolla_ansible/hashi_vault.py create mode 100644 releasenotes/notes/support-hashicorp-vault-for-kolla-passwords-76a5b4ece6a4df07.yaml create mode 100644 tests/run-hashi-vault.yml create mode 100755 tests/test-hashicorp-vault-passwords.sh create mode 120000 tools/read_passwords.py create mode 120000 tools/write_passwords.py diff --git a/doc/source/user/operating-kolla.rst b/doc/source/user/operating-kolla.rst index 067d9b534a..65def70dc6 100644 --- a/doc/source/user/operating-kolla.rst +++ b/doc/source/user/operating-kolla.rst @@ -235,6 +235,33 @@ For example: To alter this behavior, and remove such entries, use the ``--clean`` argument when invoking ``kolla-mergepwd``. +Hashicorp Vault can be used as an alternative to Ansible Vault for storing +passwords generated by Kolla Ansible. To use Hashicorp Vault as the secrets +store you will first need to generate the passwords, and then you can +save them into an existing KV using the following command: + +.. code-block:: console + + kolla-writepwd \ + --passwords /etc/kolla/passwords.yml \ + --vault-addr \ + --vault-token + +.. note:: + + For a full list of ``kolla-writepwd`` arguments, use the ``--help`` + argument when invoking ``kolla-writepwd``. + +To read passwords from Hashicorp Vault and generate a passwords.yml: + +.. code-block:: console + + mv kolla-ansible/etc/kolla/passwords.yml /etc/kolla/passwords.yml + kolla-readpwd \ + --passwords /etc/kolla/passwords.yml \ + --vault-addr \ + --vault-token + Tools ----- diff --git a/kolla_ansible/cmd/readpwd.py b/kolla_ansible/cmd/readpwd.py new file mode 100755 index 0000000000..87bdf6ff5b --- /dev/null +++ b/kolla_ansible/cmd/readpwd.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import os +import sys + +import hvac +import yaml + +from kolla_ansible.hashi_vault import hashicorp_vault_client + + +def readpwd(passwords_file, vault_kv_path, vault_mount_point, vault_namespace, + vault_addr, vault_role_id, vault_secret_id, vault_token, + vault_cacert): + + with open(passwords_file, 'r') as f: + passwords = yaml.safe_load(f.read()) + + if not isinstance(passwords, dict): + print("ERROR: Passwords file not in expected key/value format") + sys.exit(1) + + client = hashicorp_vault_client(vault_namespace, vault_addr, vault_role_id, + vault_secret_id, vault_token, vault_cacert) + + vault_kv_passwords = dict() + for password_key in passwords: + try: + password_data = client.secrets.kv.v2.read_secret_version( + mount_point=vault_mount_point, + path="{}/{}".format(vault_kv_path, password_key)) + except hvac.exceptions.InvalidPath: + # Ignore passwords that are not found in Vault + print("WARNING: '%s' not found in Vault" % password_key) + vault_kv_passwords[password_key] = None + continue + try: + vault_kv_passwords[password_key] =\ + password_data['data']['data']['password'] + except KeyError: + vault_kv_passwords[password_key] = password_data['data']['data'] + + with open(passwords_file, 'w') as f: + yaml.safe_dump(vault_kv_passwords, f) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + '-p', '--passwords', type=str, + default=os.path.abspath('/etc/kolla/passwords.yml'), + help='Path to the passwords.yml file') + parser.add_argument( + '-kv', '--vault-mount-point', type=str, + default='kv', + help='Path to the KV mount point') + parser.add_argument( + '-kvp', '--vault-kv-path', type=str, + default='kolla_passwords', + help='Path to store passwords within your configured KV mount point') + parser.add_argument( + '-n', '--vault-namespace', type=str, + default='', + help='Vault namespace (enterprise only)') + parser.add_argument( + '-v', '--vault-addr', type=str, + required=True, + help='Address to connect to an existing Hashicorp Vault') + parser.add_argument( + '-r', '--vault-role-id', type=str, + default='', + help='Role-ID to authenticate to Vault. This must be used in ' + 'conjunction with --secret-id') + parser.add_argument( + '-s', '--vault-secret-id', type=str, + default='', + help='Secret-ID to authenticate to Vault. This must be used in ' + 'conjunction with --role-id') + parser.add_argument( + '-t', '--vault-token', type=str, + default='', + help='Vault token to authenticate to Vault') + parser.add_argument( + '-c', '--vault-cacert', type=str, + default='', + help='Path to CA certificate file') + + args = parser.parse_args() + passwords_file = os.path.expanduser(args.passwords) + vault_kv_path = args.vault_kv_path + vault_mount_point = args.vault_mount_point + vault_namespace = args.vault_namespace + vault_addr = args.vault_addr + vault_role_id = args.vault_role_id + vault_secret_id = args.vault_secret_id + vault_token = args.vault_token + vault_cacert = os.path.expanduser(args.vault_cacert) + + readpwd(passwords_file, vault_kv_path, vault_mount_point, vault_namespace, + vault_addr, vault_role_id, vault_secret_id, vault_token, + vault_cacert) + + +if __name__ == '__main__': + main() diff --git a/kolla_ansible/cmd/writepwd.py b/kolla_ansible/cmd/writepwd.py new file mode 100755 index 0000000000..9a3eb0d810 --- /dev/null +++ b/kolla_ansible/cmd/writepwd.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import os +import sys + +import hvac +import yaml + +from kolla_ansible.hashi_vault import hashicorp_vault_client + + +def writepwd(passwords_file, vault_kv_path, vault_mount_point, vault_namespace, + vault_addr, vault_role_id, vault_secret_id, vault_token, + vault_cacert): + with open(passwords_file, 'r') as f: + passwords = yaml.safe_load(f.read()) + + if not isinstance(passwords, dict): + print("ERROR: Passwords file not in expected key/value format") + sys.exit(1) + + client = hashicorp_vault_client(vault_namespace, vault_addr, vault_role_id, + vault_secret_id, vault_token, vault_cacert) + + for key, value in passwords.items(): + # Ignore empty values + if not value: + continue + + if isinstance(value, str): + value = dict(password=value) + + try: + remote_value = client.secrets.kv.v2.read_secret_version( + mount_point=vault_mount_point, + path="{}/{}".format(vault_kv_path, key)) + except hvac.exceptions.InvalidPath: + # Add to KV if value does not exists + remote_value = None + + # Update KV is value has changed or it does not exist + if not remote_value or remote_value['data']['data'] != value: + client.secrets.kv.v2.create_or_update_secret( + mount_point=vault_mount_point, + path="{}/{}".format(vault_kv_path, key), + secret=value) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + '-p', '--passwords', type=str, + default=os.path.abspath('/etc/kolla/passwords.yml'), + help='Path to the passwords.yml file') + parser.add_argument( + '-kv', '--vault-mount-point', type=str, + default='kv', + help='Path to the KV mount point') + parser.add_argument( + '-kvp', '--vault-kv-path', type=str, + default='kolla_passwords', + help='Path to store passwords within your configured KV mount point') + parser.add_argument( + '-n', '--vault-namespace', type=str, + default='', + help='Vault namespace (enterprise only)') + parser.add_argument( + '-v', '--vault-addr', type=str, + required=True, + help='Address to connect to an existing Hashicorp Vault') + parser.add_argument( + '-r', '--vault-role-id', type=str, + default='', + help='Role-ID to authenticate to Vault. This must be used in ' + 'conjunction with --secret-id') + parser.add_argument( + '-s', '--vault-secret-id', type=str, + default='', + help='Secret-ID to authenticate to Vault. This must be used in ' + 'conjunction with --role-id') + parser.add_argument( + '-t', '--vault-token', type=str, + default='', + help='Vault token to authenticate to Vault') + parser.add_argument( + '-c', '--vault-cacert', type=str, + default='', + help='Path to CA certificate file') + + args = parser.parse_args() + passwords_file = os.path.expanduser(args.passwords) + vault_kv_path = args.vault_kv_path + vault_mount_point = args.vault_mount_point + vault_namespace = args.vault_namespace + vault_addr = args.vault_addr + vault_role_id = args.vault_role_id + vault_secret_id = args.vault_secret_id + vault_token = args.vault_token + vault_cacert = os.path.expanduser(args.vault_cacert) + + writepwd(passwords_file, vault_kv_path, vault_mount_point, vault_namespace, + vault_addr, vault_role_id, vault_secret_id, vault_token, + vault_cacert) + + +if __name__ == '__main__': + main() diff --git a/kolla_ansible/hashi_vault.py b/kolla_ansible/hashi_vault.py new file mode 100644 index 0000000000..de04235507 --- /dev/null +++ b/kolla_ansible/hashi_vault.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys + +import hvac + + +def hashicorp_vault_client(vault_namespace, vault_addr, vault_role_id, + vault_secret_id, vault_token, vault_cacert): + """Connect to a Vault sever and create a client. + + :param vault_namespace: Vault namespace (enterprise only). + :param vault_addr: Address to connect to an existing Hashicorp Vault. + :param vault_role_id: Role-ID to authenticate to Vault. This must be used + in conjunction with --secret-id. + :param vault_secret_id: Secret-ID to authenticate to Vault. This must be + used in conjunction with --role-id. + :param vault_token: Vault token to authenticate to Vault. + :param vault_cacert: Path to CA certificate file. + :returns: Hashicorp Vault Client (hvac.Client). + """ + + if any([vault_role_id, vault_secret_id]): + if vault_token: + print("ERROR: Vault token cannot be used at the same time as " + "role-id and secret-id") + sys.exit(1) + if not all([vault_role_id, vault_secret_id]): + print("ERROR: role-id and secret-id must be provided together") + sys.exit(1) + elif not vault_token: + print("ERROR: You must provide either a Vault token or role-id and " + "secret-id") + sys.exit(1) + + # Authenticate to Hashicorp Vault + if vault_cacert != "": + os.environ['REQUESTS_CA_BUNDLE'] = vault_cacert + + if vault_token != "": # nosec + client = hvac.Client(url=vault_addr, token=vault_token, + namespace=vault_namespace) + else: + client = hvac.Client(url=vault_addr, namespace=vault_namespace) + client.auth_approle(vault_role_id, vault_secret_id) + + if not client.is_authenticated(): + print('Failed to authenticate to vault') + sys.exit(1) + + return client diff --git a/releasenotes/notes/support-hashicorp-vault-for-kolla-passwords-76a5b4ece6a4df07.yaml b/releasenotes/notes/support-hashicorp-vault-for-kolla-passwords-76a5b4ece6a4df07.yaml new file mode 100644 index 0000000000..dd1bd42ff9 --- /dev/null +++ b/releasenotes/notes/support-hashicorp-vault-for-kolla-passwords-76a5b4ece6a4df07.yaml @@ -0,0 +1,7 @@ +--- +features: + - | + Adds functionality to allow passwords that are generated for Kolla + Ansible to be stored in Hashicorp Vault. Use new CLI commands + `kolla-readpwd` and `kolla-writepwd` to read and write Kolla Ansible + passwords to a configured Hashicorp Vault kv secrets engine. diff --git a/requirements.txt b/requirements.txt index d38c44e6fb..e85f7744cb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,3 +15,6 @@ Jinja2>=2.10 # BSD License (3 clause) # Ansible's json_query jmespath>=0.9.3 # MIT + +# Hashicorp Vault +hvac>=0.10.1 diff --git a/setup.cfg b/setup.cfg index 5259a9d052..28249f24cd 100644 --- a/setup.cfg +++ b/setup.cfg @@ -45,3 +45,5 @@ scripts = console_scripts = kolla-genpwd = kolla_ansible.cmd.genpwd:main kolla-mergepwd = kolla_ansible.cmd.mergepwd:main + kolla-writepwd = kolla_ansible.cmd.writepwd:main + kolla-readpwd = kolla_ansible.cmd.readpwd:main diff --git a/tests/run-hashi-vault.yml b/tests/run-hashi-vault.yml new file mode 100644 index 0000000000..691cf59e1e --- /dev/null +++ b/tests/run-hashi-vault.yml @@ -0,0 +1,75 @@ +--- +- hosts: all + any_errors_fatal: true + tasks: + # NOTE(yoctozepto): setting vars as facts for all to have them around in all the plays + - name: set facts for commonly used variables + set_fact: + kolla_ansible_src_dir: "{{ ansible_env.PWD }}/src/{{ zuul.project.canonical_hostname }}/openstack/kolla-ansible" + upper_constraints_file: "{{ ansible_env.HOME }}/src/opendev.org/openstack/requirements/upper-constraints.txt" + pip_user_path_env: + PATH: "{{ ansible_env.HOME + '/.local/bin:' + ansible_env.PATH }}" + +- hosts: primary + any_errors_fatal: true + environment: "{{ pip_user_path_env }}" + tasks: + - name: ensure /etc/kolla exists + file: + path: "/etc/kolla" + state: "directory" + mode: 0777 + become: true + + # NOTE(mgoddard): We need a recent pip to install the latest cryptography + # library. See https://github.com/pyca/cryptography/issues/5753 + - name: install pip 19.1.1+ + pip: + name: "pip>=19.1.1" + executable: "pip3" + extra_args: "--user" + + - name: install kolla-ansible and dependencies + pip: + name: + - "{{ kolla_ansible_src_dir }}" + executable: "pip3" + extra_args: "-c {{ upper_constraints_file }} --user" + + - name: copy passwords.yml file + copy: + src: "{{ kolla_ansible_src_dir }}/etc/kolla/passwords.yml" + dest: /etc/kolla/passwords.yml + remote_src: true + + - name: generate passwords + command: kolla-genpwd + + # At this point we have generated all necessary configuration, and are + # ready to test Hashicorp Vault. + - name: Run test-hashicorp-vault-passwords.sh script + script: + cmd: test-hashicorp-vault-passwords.sh + executable: /bin/bash + chdir: "{{ kolla_ansible_src_dir }}" + environment: + BASE_DISTRO: "{{ base_distro }}" + + - name: Read template file + slurp: + src: "/etc/kolla/passwords.yml" + register: template_file + + - name: Read generated file + slurp: + src: "/tmp/passwords-hashicorp-vault.yml" + register: generated_file + + # This test will load in the original input file and the one that was + # generated by Vault and ensure that the keys are the same in both files. + # This ensures that we are not missing any passwords. + - name: Check passwords that were written to Vault are as expected + vars: + input_passwords: "{{ template_file['content'] | b64decode | from_yaml | sort }}" + output_passwords: "{{ generated_file['content'] | b64decode | from_yaml | sort }}" + assert: { that: "input_passwords == output_passwords" } diff --git a/tests/test-hashicorp-vault-passwords.sh b/tests/test-hashicorp-vault-passwords.sh new file mode 100755 index 0000000000..e4c79c5950 --- /dev/null +++ b/tests/test-hashicorp-vault-passwords.sh @@ -0,0 +1,68 @@ +#!/bin/bash + +set -o xtrace +set -o errexit + +export PYTHONUNBUFFERED=1 + +function install_vault { + if [[ "debian" == $BASE_DISTRO ]]; then + curl -fsSL https://apt.releases.hashicorp.com/gpg | sudo apt-key add - + sudo apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main" + sudo apt-get update -y && sudo apt-get install -y vault jq + else + sudo dnf install -y yum-utils + sudo dnf config-manager --add-repo https://rpm.releases.hashicorp.com/RHEL/hashicorp.repo + sudo dnf install -y vault jq + fi +} + +function start_vault { + nohup vault server --dev & + # Give Vault some time to warm up + sleep 10 +} + +function test_vault { + TOKEN=$(vault token create -address 'http://127.0.0.1:8200' -format json | jq '.auth.client_token' --raw-output) + echo "${TOKEN}" | vault login -address 'http://127.0.0.1:8200' - + vault kv put -address 'http://127.0.0.1:8200' secret/foo data=bar +} + +function test_writepwd { + TOKEN=$(vault token create -address 'http://127.0.0.1:8200' -format json | jq '.auth.client_token' --raw-output) + kolla-writepwd \ + --passwords /etc/kolla/passwords.yml \ + --vault-addr 'http://127.0.0.1:8200' \ + --vault-token ${TOKEN} \ + --vault-mount-point secret +} + +function test_readpwd { + TOKEN=$(vault token create -address 'http://127.0.0.1:8200' -format json | jq '.auth.client_token' --raw-output) + cp etc/kolla/passwords.yml /tmp/passwords-hashicorp-vault.yml + kolla-readpwd \ + --passwords /tmp/passwords-hashicorp-vault.yml \ + --vault-addr 'http://127.0.0.1:8200' \ + --vault-token ${TOKEN} \ + --vault-mount-point secret +} + +function teardown { + pkill vault +} + +function test_hashicorp_vault_passwords { + echo "Setting up development Vault server..." + install_vault + start_vault + test_vault + echo "Write passwords to Hashicorp Vault..." + test_writepwd + echo "Read passwords from Hashicorp Vault..." + test_readpwd + echo "Cleaning up..." + teardown +} + +test_hashicorp_vault_passwords diff --git a/tools/read_passwords.py b/tools/read_passwords.py new file mode 120000 index 0000000000..6d5c5cca64 --- /dev/null +++ b/tools/read_passwords.py @@ -0,0 +1 @@ +../kolla_ansible/cmd/readpwd.py \ No newline at end of file diff --git a/tools/write_passwords.py b/tools/write_passwords.py new file mode 120000 index 0000000000..1bec78aef3 --- /dev/null +++ b/tools/write_passwords.py @@ -0,0 +1 @@ +../kolla_ansible/cmd/writepwd.py \ No newline at end of file diff --git a/zuul.d/base.yaml b/zuul.d/base.yaml index 4ea767e334..7757d6a5d2 100644 --- a/zuul.d/base.yaml +++ b/zuul.d/base.yaml @@ -230,3 +230,25 @@ - ^tests/test-prometheus-efk.sh vars: scenario: prometheus-efk + +- job: + name: kolla-ansible-hashi-vault-base + run: tests/run-hashi-vault.yml + required-projects: + - openstack/kolla-ansible + - openstack/requirements + voting: false + irrelevant-files: + - ^.*\.rst$ + - ^doc/.* + - ^releasenotes/.*$ + - ^deploy-guide/.*$ + - ^test-requirements.txt$ + - ^etc/kolla/globals.yml$ + - ^tox.ini$ + - ^\..+ + - ^LICENSE$ + - ^contrib/ + - ^specs/ + - ^kolla_ansible/tests/ + - ^zuul\.d/ diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml index 81886c7ed6..4207a83a4e 100644 --- a/zuul.d/jobs.yaml +++ b/zuul.d/jobs.yaml @@ -378,3 +378,10 @@ vars: base_distro: ubuntu install_type: source + +- job: + name: kolla-ansible-centos8s-hashi-vault + parent: kolla-ansible-hashi-vault-base + nodeset: kolla-ansible-centos8s + vars: + base_distro: centos diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml index f0889e43c6..92b826d242 100644 --- a/zuul.d/project.yaml +++ b/zuul.d/project.yaml @@ -52,6 +52,7 @@ - kolla-ansible-ubuntu-source-cephadm - kolla-ansible-centos8s-source-upgrade-cephadm - kolla-ansible-ubuntu-source-upgrade-cephadm + - kolla-ansible-centos8s-hashi-vault check-arm64: jobs: - kolla-ansible-debian-source-aarch64