Rebuild credential-key repo during keystone[0] rebuild

When the first Keystone container is rebult in an existing environment,
the credential key repository is overwritten with new keys and the
existing keys are overwritten on the other infrastructure hosts without
any migration taking place. This results in an irrevocable loss of
the keys used to encrypt the credentials.

Now we will collect keys from any existing credential keys on the other
containers and use them to rebuild the credential-key repo on the primary
container before performing a key migration and rotation.

If no keys are found on the other containers, we will perform a
credential_setup on the primary container and sync the keys, just
as we would have before.

Closes-Bug: #1667960
Change-Id: Ic616d397574573629273838fbf68ea3f6bdb0468

releasenotes/notes/primary-container-rebuild-a2f4d7f33d66c843.yaml

@@ -0,0 +1,5 @@
+---
+critical:
+  - A bug that caused the Keystone credential keys to be lost when the playbook
+    is run during a rebuild of the first Keystone container has been fixed.
+    Please see launchpad bug 1667960 for more details.

tasks/keystone_credential_create.yml

@@ -18,6 +18,58 @@
     path: "{{ keystone_credential_key_repository }}/0"
   register: _credential_keys
 
+- name: Check for credential keys on all Keystone containers
+  find:
+    paths: "{{ keystone_credential_key_repository }}"
+    patterns: "^[0-9]+$"
+    use_regex: True
+  when: not _credential_keys.stat.exists
+  register: credential_key_list
+  delegate_to: "{{ item }}"
+  with_items: "{{ groups['keystone_all'] }}"
+
+- name: Aggregate the collected file lists
+  set_fact:
+    existing_credential_keys: >-
+      {% set _var = [] -%}
+      {% for result in credential_key_list.results -%}
+      {%   if result.files is defined -%}
+      {%     for file in result.files -%}
+      {%       if _var.append({'host': result.item, 'file': file.path}) -%}{% endif -%}
+      {%     endfor -%}
+      {%   endif -%}
+      {% endfor -%}
+      {{ _var }}
+  when: not credential_key_list | skipped
+
+- name: Collect the existing keys from containers
+  slurp:
+    src: "{{ item.file }}"
+  delegate_to: "{{ item.host }}"
+  with_items: "{{ existing_credential_keys }}"
+  register: collected_existing_credential_keys
+  when: existing_credential_keys is defined
+
+- name: Ensure the target directory exists on the master Keystone container
+  file:
+    path: "{{ keystone_credential_key_repository }}"
+    state: directory
+    owner: "{{ keystone_system_user_name }}"
+    group: "{{ keystone_system_group_name }}"
+    mode: "0700"
+  when: not collected_existing_credential_keys | skipped
+
+- name: Drop the existing credential keys in the master Keystone container
+  copy:
+    content: "{{ item.1 | b64decode }}"
+    dest: "{{ keystone_credential_key_repository }}/{{ item.0 }}"
+    owner: "{{ keystone_system_user_name }}"
+    group: "{{ keystone_system_group_name }}"
+    mode: "0600"
+  when: not collected_existing_credential_keys | skipped
+  register: drop_existing_credential_keys
+  with_indexed_items: "{{ collected_existing_credential_keys.results | map(attribute='content') | list | unique }}"
+
 - name: Create credential keys for Keystone
   command: >
     {{ keystone_bin }}/keystone-manage credential_setup
@@ -25,7 +77,10 @@
                                        --keystone-group "{{ keystone_system_group_name }}"
   become: yes
   become_user: "{{ keystone_system_user_name }}"
-  when: not _credential_keys.stat.exists
+  register: create_credential_keys
+  when:
+    - not _credential_keys.stat.exists
+    - not drop_existing_credential_keys | changed
 
 - name: Ensure newest key is used for credential in Keystone
   command: >
@@ -34,7 +89,7 @@
                                        --keystone-group "{{ keystone_system_group_name }}"
   become: yes
   become_user: "{{ keystone_system_user_name }}"
-  when: _credential_keys.stat.exists
+  when: create_credential_keys | skipped
 
 - name: Rotate credential keys for Keystone
   command: >
@@ -43,4 +98,4 @@
                                        --keystone-group "{{ keystone_system_group_name }}"
   become: yes
   become_user: "{{ keystone_system_user_name }}"
-  when: _credential_keys.stat.exists
+  when: create_credential_keys | skipped