diff --git a/ansible/roles/keystone/handlers/main.yml b/ansible/roles/keystone/handlers/main.yml index b6ddfaf804..b738906b32 100644 --- a/ansible/roles/keystone/handlers/main.yml +++ b/ansible/roles/keystone/handlers/main.yml @@ -26,21 +26,6 @@ - kolla_action == "upgrade" - inventory_hostname == groups[service.group][0] -- name: Restart keystone container - vars: - service_name: "keystone" - service: "{{ keystone_services[service_name] }}" - become: true - kolla_docker: - action: "recreate_or_restart_container" - common_options: "{{ docker_common_options }}" - name: "{{ service.container_name }}" - image: "{{ service.image }}" - volumes: "{{ service.volumes|reject('equalto', '')|list }}" - dimensions: "{{ service.dimensions }}" - when: - - kolla_action != "config" - - name: Restart keystone-ssh container vars: service_name: "keystone-ssh" @@ -71,6 +56,21 @@ when: - kolla_action != "config" +- name: Restart keystone container + vars: + service_name: "keystone" + service: "{{ keystone_services[service_name] }}" + become: true + kolla_docker: + action: "recreate_or_restart_container" + common_options: "{{ docker_common_options }}" + name: "{{ service.container_name }}" + image: "{{ service.image }}" + volumes: "{{ service.volumes|reject('equalto', '')|list }}" + dimensions: "{{ service.dimensions }}" + when: + - kolla_action != "config" + - name: Finish keystone database upgrade vars: service_name: "keystone" diff --git a/ansible/roles/keystone/tasks/bootstrap_service.yml b/ansible/roles/keystone/tasks/bootstrap_service.yml index da27564019..af4a84b281 100644 --- a/ansible/roles/keystone/tasks/bootstrap_service.yml +++ b/ansible/roles/keystone/tasks/bootstrap_service.yml @@ -63,12 +63,12 @@ command: > bash -c 'sudo -E kolla_set_configs && keystone-manage --config-file /etc/keystone/keystone.conf - fernet_setup --keystone-user {{ keystone_username }} --keystone-group {{ keystone_groupname }} && ls -l /etc/keystone/fernet-keys/' + fernet_setup --keystone-user {{ keystone_username }} --keystone-group {{ keystone_groupname }}' name: "bootstrap_keystone_fernet" restart_policy: no volumes: "{{ keystone_fernet.volumes|reject('equalto', '')|list }}" run_once: True - delegate_to: "{{ groups['keystone_fernet_bootstrap'][0] }}" + delegate_to: "{{ groups['keystone'][0] }}" when: - keystone_token_provider == 'fernet' - groups['keystone_fernet_running'] is not defined diff --git a/ansible/roles/keystone/tasks/config.yml b/ansible/roles/keystone/tasks/config.yml index c0368364c8..f4532c6bbc 100644 --- a/ansible/roles/keystone/tasks/config.yml +++ b/ansible/roles/keystone/tasks/config.yml @@ -76,6 +76,20 @@ notify: - Restart {{ item.key }} container +- name: Copying keystone-startup script for keystone + vars: + keystone: "{{ keystone_services['keystone'] }}" + template: + src: "keystone-startup.sh.j2" + dest: "{{ node_config_directory }}/keystone/keystone-startup.sh" + mode: "0660" + become: true + when: + - inventory_hostname in groups[keystone.group] + - keystone.enabled | bool + notify: + - Restart keystone container + - name: Create Keystone domain-specific config directory vars: keystone: "{{ keystone_services.keystone }}" diff --git a/ansible/roles/keystone/templates/fernet-node-sync.sh.j2 b/ansible/roles/keystone/templates/fernet-node-sync.sh.j2 index ec43b7bd14..4293a807d8 100644 --- a/ansible/roles/keystone/templates/fernet-node-sync.sh.j2 +++ b/ansible/roles/keystone/templates/fernet-node-sync.sh.j2 @@ -3,13 +3,9 @@ set -o errexit set -o pipefail -# Get data on the fernet tokens -# NOTE(mnasiadka): Check for existence of at least two tokens (should exist after bootstrap) -TOKEN_CHECK=$(/usr/bin/python3 /usr/bin/fetch_fernet_tokens.py -t {{ fernet_token_expiry }} -n 2) - -# Ensure tokens are populated +# Ensure tokens are populated, check for 0 key which should always exist n=0 -while /usr/bin/python3 /usr/bin/fetch_fernet_tokens.py -t 86400 -n 1 | grep -q '"populated": false'; do +while [ ! -f /etc/keystone/fernet-keys/0 ]; do if [ $n -lt 10 ]; then n=$(( n + 1 )) echo "ERROR: Fernet tokens have not been populated, rechecking in 1 minute" @@ -21,15 +17,3 @@ while /usr/bin/python3 /usr/bin/fetch_fernet_tokens.py -t 86400 -n 1 | grep -q ' exit 1 fi done - -# Ensure the primary token exists and is not stale -if $(echo "$TOKEN_CHECK" | grep -q '"update_required": false'); then - exit 0; -fi - -# For each host node sync tokens -{% for host in groups['keystone'] %} -{% if inventory_hostname != host %} -/usr/bin/rsync -azu --delete -e 'ssh -i /var/lib/keystone/.ssh/id_rsa -p {{ hostvars[host]['keystone_ssh_port'] }} -F /var/lib/keystone/.ssh/config' keystone@{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:/etc/keystone/fernet-keys/ /etc/keystone/fernet-keys -{% endif %} -{% endfor %} diff --git a/ansible/roles/keystone/templates/keystone-startup.sh.j2 b/ansible/roles/keystone/templates/keystone-startup.sh.j2 new file mode 100644 index 0000000000..f5769b9488 --- /dev/null +++ b/ansible/roles/keystone/templates/keystone-startup.sh.j2 @@ -0,0 +1,35 @@ +#!/bin/bash -x +{% set keystone_cmd = 'apache2' if kolla_base_distro in ['ubuntu', 'debian'] else 'httpd' %} + +set -o errexit +set -o pipefail + +TOKEN_DIR="/etc/keystone/fernet-keys" + +# Ensure tokens are populated, check for 0 (staging) key +n=0 +while [ ! -f "${TOKEN_DIR}/0" ]; do + if [ $n -lt 36 ]; then + n=$(( n + 1 )) + echo "ERROR: Fernet tokens have not been populated, rechecking in 5 seconds" + echo "DEBUG: ${TOKEN_DIR} contents:" + ls -l ${TOKEN_DIR} + sleep 5 + else + echo "CRITICAL: Waited for 10 minutes - failing" + exit 1 + fi +done + +# Ensure tokens are not stale +# Get primary token (file with highest number) +TOKEN_PRIMARY=$(ls -1 ${TOKEN_DIR} | sort -hr | head -n 1) +# Check it's age in seconds +TOKEN_AGE=$(($(date +%s) - $(date +%s -r "${TOKEN_DIR}/${TOKEN_PRIMARY}"))) +# Compare if it's older than fernet_token_expiry and run key rotation if needed +if [ "${TOKEN_AGE}" -gt "{{ fernet_token_expiry }}" ]; then + echo "ERROR: Primary token ${TOKEN_PRIMARY} is stale." + exit 1 +fi + +exec /usr/sbin/{{ keystone_cmd }} $@ diff --git a/ansible/roles/keystone/templates/keystone.json.j2 b/ansible/roles/keystone/templates/keystone.json.j2 index 6637c5f559..e5c676190b 100644 --- a/ansible/roles/keystone/templates/keystone.json.j2 +++ b/ansible/roles/keystone/templates/keystone.json.j2 @@ -1,8 +1,13 @@ -{% set keystone_cmd = 'apache2' if kolla_base_distro in ['ubuntu', 'debian'] else 'httpd' %} {% set keystone_dir = 'apache2/conf-enabled' if kolla_base_distro in ['ubuntu', 'debian'] else 'httpd/conf.d' %} { - "command": "/usr/sbin/{{ keystone_cmd }}", + "command": "/usr/bin/keystone-startup.sh", "config_files": [ + { + "source": "{{ container_config_directory }}/keystone-startup.sh", + "dest": "/usr/bin/keystone-startup.sh", + "owner": "root", + "perm": "0755" + }, { "source": "{{ container_config_directory }}/keystone.conf", "dest": "/etc/keystone/keystone.conf",