Merge "Fix fernet bootstrap and key distribution - follow up"

ansible/roles/keystone/handlers/main.yml

@@ -26,21 +26,6 @@
     - kolla_action == "upgrade"
     - inventory_hostname == groups[service.group][0]
 
-- name: Restart keystone container
-  vars:
-    service_name: "keystone"
-    service: "{{ keystone_services[service_name] }}"
-  become: true
-  kolla_docker:
-    action: "recreate_or_restart_container"
-    common_options: "{{ docker_common_options }}"
-    name: "{{ service.container_name }}"
-    image: "{{ service.image }}"
-    volumes: "{{ service.volumes|reject('equalto', '')|list }}"
-    dimensions: "{{ service.dimensions }}"
-  when:
-    - kolla_action != "config"
-
 - name: Restart keystone-ssh container
   vars:
     service_name: "keystone-ssh"
@@ -71,6 +56,21 @@
   when:
     - kolla_action != "config"
 
+- name: Restart keystone container
+  vars:
+    service_name: "keystone"
+    service: "{{ keystone_services[service_name] }}"
+  become: true
+  kolla_docker:
+    action: "recreate_or_restart_container"
+    common_options: "{{ docker_common_options }}"
+    name: "{{ service.container_name }}"
+    image: "{{ service.image }}"
+    volumes: "{{ service.volumes|reject('equalto', '')|list }}"
+    dimensions: "{{ service.dimensions }}"
+  when:
+    - kolla_action != "config"
+
 - name: Finish keystone database upgrade
   vars:
     service_name: "keystone"

ansible/roles/keystone/tasks/bootstrap_service.yml

@@ -63,12 +63,12 @@
     command: >
       bash -c 'sudo -E kolla_set_configs &&
       keystone-manage --config-file /etc/keystone/keystone.conf
-      fernet_setup --keystone-user {{ keystone_username }} --keystone-group {{ keystone_groupname }} && ls -l /etc/keystone/fernet-keys/'
+      fernet_setup --keystone-user {{ keystone_username }} --keystone-group {{ keystone_groupname }}'
     name: "bootstrap_keystone_fernet"
     restart_policy: no
     volumes: "{{ keystone_fernet.volumes|reject('equalto', '')|list }}"
   run_once: True
-  delegate_to: "{{ groups['keystone_fernet_bootstrap'][0] }}"
+  delegate_to: "{{ groups['keystone'][0] }}"
   when:
     - keystone_token_provider == 'fernet'
     - groups['keystone_fernet_running'] is not defined

ansible/roles/keystone/tasks/config.yml

@@ -76,6 +76,20 @@
   notify:
     - Restart {{ item.key }} container
 
+- name: Copying keystone-startup script for keystone
+  vars:
+    keystone: "{{ keystone_services['keystone'] }}"
+  template:
+    src: "keystone-startup.sh.j2"
+    dest: "{{ node_config_directory }}/keystone/keystone-startup.sh"
+    mode: "0660"
+  become: true
+  when:
+    - inventory_hostname in groups[keystone.group]
+    - keystone.enabled | bool
+  notify:
+    - Restart keystone container
+
 - name: Create Keystone domain-specific config directory
   vars:
     keystone: "{{ keystone_services.keystone }}"

ansible/roles/keystone/templates/fernet-node-sync.sh.j2

@@ -3,13 +3,9 @@
 set -o errexit
 set -o pipefail
 
-# Get data on the fernet tokens
+# Ensure tokens are populated, check for 0 key which should always exist
-# NOTE(mnasiadka): Check for existence of at least two tokens (should exist after bootstrap)
-TOKEN_CHECK=$(/usr/bin/python3 /usr/bin/fetch_fernet_tokens.py -t {{ fernet_token_expiry }} -n 2)
-
-# Ensure tokens are populated
 n=0
-while /usr/bin/python3 /usr/bin/fetch_fernet_tokens.py -t 86400 -n 1 | grep -q '"populated": false'; do
+while [ ! -f /etc/keystone/fernet-keys/0 ]; do
     if [ $n -lt 10 ]; then
         n=$(( n + 1 ))
         echo "ERROR: Fernet tokens have not been populated, rechecking in 1 minute"
@@ -21,15 +17,3 @@ while /usr/bin/python3 /usr/bin/fetch_fernet_tokens.py -t 86400 -n 1 | grep -q '
         exit 1
     fi
 done
-
-# Ensure the primary token exists and is not stale
-if $(echo "$TOKEN_CHECK" | grep -q '"update_required": false'); then
-    exit 0;
-fi
-
-# For each host node sync tokens
-{% for host in groups['keystone'] %}
-{% if inventory_hostname != host %}
-/usr/bin/rsync -azu --delete -e 'ssh -i /var/lib/keystone/.ssh/id_rsa -p {{ hostvars[host]['keystone_ssh_port'] }} -F /var/lib/keystone/.ssh/config' keystone@{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:/etc/keystone/fernet-keys/ /etc/keystone/fernet-keys
-{% endif %}
-{% endfor %}

ansible/roles/keystone/templates/keystone-startup.sh.j2

@@ -0,0 +1,35 @@
+#!/bin/bash -x
+{% set keystone_cmd = 'apache2' if kolla_base_distro in ['ubuntu', 'debian'] else 'httpd' %}
+
+set -o errexit
+set -o pipefail
+
+TOKEN_DIR="/etc/keystone/fernet-keys"
+
+# Ensure tokens are populated, check for 0 (staging) key
+n=0
+while [ ! -f "${TOKEN_DIR}/0" ]; do
+    if [ $n -lt 36 ]; then
+        n=$(( n + 1 ))
+        echo "ERROR: Fernet tokens have not been populated, rechecking in 5 seconds"
+        echo "DEBUG: ${TOKEN_DIR} contents:"
+        ls -l ${TOKEN_DIR}
+        sleep 5
+    else
+        echo "CRITICAL: Waited for 10 minutes - failing"
+        exit 1
+    fi
+done
+
+# Ensure tokens are not stale
+# Get primary token (file with highest number)
+TOKEN_PRIMARY=$(ls -1 ${TOKEN_DIR} | sort -hr | head -n 1)
+# Check it's age in seconds
+TOKEN_AGE=$(($(date +%s) - $(date +%s -r "${TOKEN_DIR}/${TOKEN_PRIMARY}")))
+# Compare if it's older than fernet_token_expiry and run key rotation if needed
+if [ "${TOKEN_AGE}" -gt "{{ fernet_token_expiry }}" ]; then
+    echo "ERROR: Primary token ${TOKEN_PRIMARY} is stale."
+    exit 1
+fi
+
+exec /usr/sbin/{{ keystone_cmd }} $@

ansible/roles/keystone/templates/keystone.json.j2

@@ -1,8 +1,13 @@
-{% set keystone_cmd = 'apache2' if kolla_base_distro in ['ubuntu', 'debian'] else 'httpd' %}
 {% set keystone_dir = 'apache2/conf-enabled' if kolla_base_distro in ['ubuntu', 'debian'] else 'httpd/conf.d' %}
 {
-    "command": "/usr/sbin/{{ keystone_cmd }}",
+    "command": "/usr/bin/keystone-startup.sh",
     "config_files": [
+        {
+            "source": "{{ container_config_directory }}/keystone-startup.sh",
+            "dest": "/usr/bin/keystone-startup.sh",
+            "owner": "root",
+            "perm": "0755"
+        },
         {
             "source": "{{ container_config_directory }}/keystone.conf",
             "dest": "/etc/keystone/keystone.conf",