Implement serialisable rolling upgrade

The current rolling upgrade implementation
relies on the role to orchestrate the rolling
upgrade. When the role is executed using
playbook serialisation, the db sync contract
is executed before all hosts are upgraded,
potentially resulting in data corruption.

This patch returns the role to the simpler,
best practice model of expecting that the role
is applied to a single host and that the
playbook handles orchestration. This method
can be used with any form of serialisation.

Depends-On: Ie90cdcbf9e73082a2074c8832b7490d188e178af
Change-Id: I5650f16b9a115bd392012b743788057a94d09226

handlers/main.yml

@@ -13,26 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-- name: Restart web server on first node
-  command: "/bin/true"
-  notify:
-    - Restart web server
-    - Wait for web server to complete starting
-  when:
-    - inventory_hostname == groups['keystone_all'][0]
-  tags:
-    - keystone-config
-
-- name: Restart web server on other nodes
-  command: "/bin/true"
-  notify:
-    - Restart web server
-    - Wait for web server to complete starting
-  when:
-    - inventory_hostname != groups['keystone_all'][0]
-  tags:
-    - keystone-config
-
 - name: Restart web server
   service:
     name: "{{ (keystone_apache_enabled | bool) | ternary(keystone_system_service_name, 'nginx') }}"
@@ -43,8 +23,6 @@
   until: _restart | success
   retries: 5
   delay: 2
-  tags:
-    - keystone-config
 
 - name: Wait for web server to complete starting
   wait_for:
@@ -57,32 +35,7 @@
   register: _wait_check
   until: _wait_check | success
   retries: 5
-  tags:
+  listen: "Restart web server"
-    - keystone-config
-
-- name: Restart uWSGI on first node
-  command: "/bin/true"
-  when:
-    - inventory_hostname == groups['keystone_all'][0]
-  notify:
-    - Stop uWSGI
-    - Copy new policy file into place
-    - Start uWSGI
-    - Wait for uWSGI socket to be ready
-  tags:
-    - keystone-config
-
-- name: Restart uWSGI on other nodes
-  command: "/bin/true"
-  when:
-    - inventory_hostname != groups['keystone_all'][0]
-  notify:
-    - Stop uWSGI
-    - Copy new policy file into place
-    - Start uWSGI
-    - Wait for uWSGI socket to be ready
-  tags:
-    - keystone-config
 
 - name: Stop uWSGI
   service:
@@ -96,8 +49,7 @@
   with_items: "{{ keystone_wsgi_program_names }}"
   when:
     - not keystone_mod_wsgi_enabled | bool
-  tags:
+  listen: "Restart uWSGI"
-    - keystone-config
 
 # Note (odyssey4me):
 # The policy.json file is currently read continually by the services
@@ -115,8 +67,7 @@
     group: "{{ keystone_system_group_name }}"
     mode: "0640"
     remote_src: yes
-  tags:
+  listen: "Restart uWSGI"
-    - keystone-config
 
 - name: Start uWSGI
   service:
@@ -131,8 +82,7 @@
   with_items: "{{ keystone_wsgi_program_names }}"
   when:
     - not keystone_mod_wsgi_enabled | bool
-  tags:
+  listen: "Restart uWSGI"
-    - keystone-config
 
 - name: Wait for uWSGI socket to be ready
   wait_for:
@@ -147,8 +97,7 @@
   register: _wait_check
   until: _wait_check | success
   retries: 5
-  tags:
+  listen: "Restart uWSGI"
-    - keystone-config
 
 - name: Restart Shibd
   service:
@@ -160,13 +109,3 @@
   until: _restart | success
   retries: 5
   delay: 2
-  tags:
-    - keystone-config
-
-- name: Perform a Keystone DB sync contract
-  command: "{{ keystone_bin }}/keystone-manage db_sync --contract"
-  become: yes
-  become_user: "{{ keystone_system_user_name }}"
-  tags:
-    - keystone-config
-

tasks/keystone_apache.yml

@@ -53,8 +53,7 @@
   when:
     - ansible_pkg_mgr == 'apt'
   notify:
-    - Restart web server on first node
+    - Restart web server
-    - Restart web server on other nodes
 
 ## NOTE(andymccr):
 ## We need to enable a module for httpd on RedHat/CentOS using LoadModule inside conf files
@@ -66,8 +65,7 @@
   when:
    - ansible_pkg_mgr == 'yum'
   notify:
-    - Restart web server on first node
+    - Restart web server
-    - Restart web server on other nodes
 
 - name: Drop apache2 config files
   template:
@@ -77,8 +75,7 @@
     group: "root"
   with_items: "{{ keystone_apache_configs }}"
   notify:
-    - Restart web server on first node
+    - Restart web server
-    - Restart web server on other nodes
 
 - name: Disable default apache site
   file:
@@ -86,8 +83,7 @@
     state: "absent"
   with_items: "{{ keystone_apache_default_sites }}"
   notify:
-    - Restart web server on first node
+    - Restart web server
-    - Restart web server on other nodes
 
 - name: Enabled keystone vhost
   file:
@@ -98,16 +94,14 @@
     - keystone_apache_site_available is defined
     - keystone_apache_site_enabled is defined
   notify:
-    - Restart web server on first node
+    - Restart web server
-    - Restart web server on other nodes
 
 - name: Ensure Apache ServerName
   lineinfile:
     dest: "{{ keystone_apache_conf }}"
     line: "ServerName {{ ansible_hostname }}"
   notify:
-    - Restart web server on first node
+    - Restart web server
-    - Restart web server on other nodes
 
 - name: Ensure Apache ServerTokens
   lineinfile:
@@ -115,8 +109,7 @@
     regexp: '^ServerTokens'
     line: "ServerTokens {{ keystone_apache_servertokens }}"
   notify:
-    - Restart web server on first node
+    - Restart web server
-    - Restart web server on other nodes
 
 - name: Ensure Apache ServerSignature
   lineinfile:
@@ -124,8 +117,7 @@
     regexp: '^ServerSignature'
     line: "ServerSignature {{ keystone_apache_serversignature }}"
   notify:
-    - Restart web server on first node
+    - Restart web server
-    - Restart web server on other nodes
 
 - name: Remove Listen from Apache config
   lineinfile:
@@ -134,5 +126,4 @@
     backrefs: yes
     line: '#\1'
   notify:
-    - Restart web server on first node
+    - Restart web server
-    - Restart web server on other nodes

tasks/keystone_cleanup_old_facts.yml

@@ -16,18 +16,18 @@
 # TODO(odyssey4me):
 # This file and the task calling it may be removed in Queens.
 
-- name: Convert the old fact format to the new fact format
-  ini_file:
-    dest: "/etc/ansible/facts.d/openstack_ansible.fact"
-    section: keystone
-    option: need_db_sync
-    value: "{{ ansible_local['keystone']['general']['need_db_sync'] | bool }}"
-
 - name: Remove the old fact file
   file:
     path: "/etc/ansible/facts.d/keystone.fact"
     state: absent
 
+- name: Remove the old db_sync fact
+  ini_file:
+    dest: "/etc/ansible/facts.d/openstack_ansible.fact"
+    section: keystone
+    option: "need_db_sync"
+    state: absent
+
 - name: refresh local facts
   setup:
     filter: ansible_local

tasks/keystone_db_setup.yml

@@ -13,43 +13,68 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-- name: Ensure keystone service stopped on first node
-  service:
-    name: "{{ item }}"
-    state: stopped
-  register: keystone_stop
-  failed_when:
-    - keystone_stop.msg is defined
-    - "'no service or tool' not in keystone_stop.msg"
-    - "'systemd could not find' not in keystone_stop.msg"
-    - "'Could not find the requested service' not in keystone_stop.msg"
-  with_items:
-    - "{{ keystone_wsgi_program_names }}"
-    - "{{ keystone_system_service_name }}"
-
 - name: Check current state of Keystone DB
   command: "{{ keystone_bin }}/keystone-manage db_sync --check"
   register: keystone_db_sync_check
   failed_when: "keystone_db_sync_check.rc == 1"
   changed_when: "keystone_db_sync_check.rc not in [2, 3, 4]"
 
+- name: Set the db sync local facts
+  ini_file:
+    dest: "/etc/ansible/facts.d/openstack_ansible.fact"
+    section: keystone
+    option: "{{ item.name }}"
+    value: "{{ item.state }}"
+  with_items:
+    - name: "need_db_expand"
+      state: "{{ (keystone_db_sync_check.rc | int == 2) | bool }}"
+    - name: "need_db_migrate"
+      state: "{{ (keystone_db_sync_check.rc | int in [2, 3] ) | bool }}"
+    - name: "need_db_contract"
+      state: "{{ (keystone_db_sync_check.rc | int in [2, 3, 4]) | bool }}"
+
+- name: Refresh local facts
+  setup:
+    filter: ansible_local
+    gather_subset: "!all"
+  tags:
+    - keystone-config
+
+- name: Ensure keystone service is stopped
+  service:
+    name: "{{ item }}"
+    state: stopped
+  register: keystone_stop
+  failed_when:
+    - "keystone_stop.msg is defined"
+    - "'no service or tool' not in keystone_stop.msg"
+    - "'systemd could not find' not in keystone_stop.msg"
+    - "'Could not find the requested service' not in keystone_stop.msg"
+  with_items:
+    - "{{ keystone_wsgi_program_names }}"
+    - "{{ keystone_system_service_name }}"
+  when:
+    - "(ansible_local['openstack_ansible']['keystone']['need_db_expand'] | bool) or
+       (ansible_local['openstack_ansible']['keystone']['need_db_migrate'] | bool)"
+
 - name: Perform a Keystone DB sync expand
   command: "{{ keystone_bin }}/keystone-manage db_sync --expand"
   become: yes
   become_user: "{{ keystone_system_user_name }}"
-  when: "keystone_db_sync_check.rc == 2"
+  when:
+    - "ansible_local['openstack_ansible']['keystone']['need_db_expand'] | bool"
 
 - name: Perform a Keystone DB sync migrate
   command: "{{ keystone_bin }}/keystone-manage db_sync --migrate"
   become: yes
   become_user: "{{ keystone_system_user_name }}"
-  when: "keystone_db_sync_check.rc in [2, 3]"
+  when:
-  notify:
+    - "ansible_local['openstack_ansible']['keystone']['need_db_migrate'] | bool"
-    - Perform a Keystone DB sync contract
 
-- name: Check if Keystone DB sync contract is required
+- name: Perform a Keystone DB sync contract
-  command: "true"
+  command: "{{ keystone_bin }}/keystone-manage db_sync --contract"
-  changed_when: "keystone_db_sync_check.rc == 4"
+  become: yes
-  register: dbsync
+  become_user: "{{ keystone_system_user_name }}"
-  notify:
+  when:
-    - Perform a Keystone DB sync contract
+    - "(keystone_all_software_updated | default('no')) | bool"
+    - "ansible_local['openstack_ansible']['keystone']['need_db_contract'] | bool"

tasks/keystone_federation_sp_setup.yml

@@ -33,8 +33,7 @@
   changed_when: false
   when: inventory_hostname == groups['keystone_all'][0]
   notify:
-    - Restart web server on first node
+    - Restart web server
-    - Restart web server on other nodes
     - Restart Shibd
 
 - name: Store sp cert
@@ -66,8 +65,7 @@
     mode: "0640"
   when: inventory_hostname != groups['keystone_all'][0]
   notify:
-    - Restart web server on first node
+    - Restart web server
-    - Restart web server on other nodes
     - Restart Shibd
 
 - name: Distribute sp cert
@@ -79,8 +77,7 @@
     mode: "0640"
   when: inventory_hostname != groups['keystone_all'][0]
   notify:
-    - Restart web server on first node
+    - Restart web server
-    - Restart web server on other nodes
     - Restart Shibd
 
 - name: Set appropriate file ownership on the Shibboleth SP key-pair
@@ -93,6 +90,5 @@
     - "/etc/shibboleth/sp-key.pem"
   when: inventory_hostname != groups['keystone_all'][0]
   notify:
-    - Restart web server on first node
+    - Restart web server
-    - Restart web server on other nodes
     - Restart Shibd

tasks/keystone_idp_metadata.yml

@@ -20,7 +20,4 @@
   become_user: "{{ keystone_system_user_name }}"
   when: keystone_idp != {}
   notify:
-    - Restart uWSGI on first node
+    - Restart uWSGI
-    - Restart uWSGI on other nodes
-    - Restart web server on first node
-    - Restart web server on other nodes

tasks/keystone_idp_self_signed_create.yml

@@ -33,8 +33,7 @@
   when: >
     inventory_hostname == groups['keystone_all'][0]
   notify:
-    - Restart web server on first node
+    - Restart web server
-    - Restart web server on other nodes
 
 - name: Set appropriate file ownership on the IdP self-signed cert
   file:

tasks/keystone_idp_self_signed_distribute.yml

@@ -30,8 +30,7 @@
   retries: 5
   delay: 2
   notify:
-    - Restart web server on first node
+    - Restart web server
-    - Restart web server on other nodes
 
 - name: Set appropriate file ownership on the IdP self-signed cert
   file:

tasks/keystone_init_systemd.yml

@@ -32,10 +32,8 @@
     owner: "root"
     group: "root"
   notify:
-    - Restart uWSGI on first node
+    - Restart uWSGI
-    - Restart uWSGI on other nodes
+    - Restart web server
-    - Restart web server on first node
-    - Restart web server on other nodes
 
 - name: Place the systemd init script
   config_template:
@@ -47,7 +45,5 @@
     config_overrides: "{{ keystone_uwsgi_init_overrides }}"
     config_type: "ini"
   notify:
-    - Restart uWSGI on first node
+    - Restart uWSGI
-    - Restart uWSGI on other nodes
+    - Restart web server
-    - Restart web server on first node
-    - Restart web server on other nodes

tasks/keystone_install.yml

@@ -60,8 +60,7 @@
   retries: 5
   delay: 2
   notify:
-    - Restart web server on first node
+    - Restart web server
-    - Restart web server on other nodes
 
 - name: Create developer mode constraint file
   copy:
@@ -120,10 +119,8 @@
     copy: "no"
   when: keystone_get_venv | changed
   notify:
-    - Restart uWSGI on first node
+    - Restart uWSGI
-    - Restart uWSGI on other nodes
+    - Restart web server
-    - Restart web server on first node
-    - Restart web server on other nodes
 
 - name: Install pip packages
   pip:
@@ -141,10 +138,8 @@
   delay: 2
   when: keystone_get_venv | failed or keystone_get_venv | skipped
   notify:
-    - Restart uWSGI on first node
+    - Restart uWSGI
-    - Restart uWSGI on other nodes
+    - Restart web server
-    - Restart web server on first node
-    - Restart web server on other nodes
 
 - name: CentOS remove python from path first
   file:
@@ -171,15 +166,21 @@
     - src: "{{ keystone_bin }}/keystone-wsgi-public"
       dest: main
   notify:
-    - Restart web server on first node
+    - Restart web server
-    - Restart web server on other nodes
 
-- name: Record the need for a db sync
+- name: Initialise the db sync local facts
   ini_file:
     dest: "/etc/ansible/facts.d/openstack_ansible.fact"
     section: keystone
-    option: need_db_sync
+    option: "{{ item.name }}"
-    value: True
+    value: "{{ item.state }}"
+  with_items:
+    - name: "need_db_expand"
+      state: "True"
+    - name: "need_db_migrate"
+      state: "True"
+    - name: "need_db_contract"
+      state: "True"
   when: keystone_get_venv | changed or
         keystone_venv_dir | changed or
         install_packages | changed

tasks/keystone_ldap_setup.yml

@@ -35,10 +35,8 @@
     mode: "0640"
   with_dict: "{{ keystone_ldap }}"
   notify:
-    - Restart uWSGI on first node
+    - Restart uWSGI
-    - Restart uWSGI on other nodes
+    - Restart web server
-    - Restart web server on first node
-    - Restart web server on other nodes
 
 # Bug 1547542 - Older versions of the keystone role would deploy a blank
 # keystone.Default.conf and this will cause errors when adding LDAP-backed
@@ -49,7 +47,5 @@
     state: absent
   when: keystone_ldap.Default is not defined
   notify:
-    - Restart uWSGI on first node
+    - Restart uWSGI
-    - Restart uWSGI on other nodes
+    - Restart web server
-    - Restart web server on first node
-    - Restart web server on other nodes

tasks/keystone_nginx.yml

@@ -24,8 +24,7 @@
     path: /etc/nginx/sites-enabled/default
     state: absent
   notify:
-    - Restart web server on first node
+    - Restart web server
-    - Restart web server on other nodes
 
 - name: Configure custom nginx log format
   lineinfile:
@@ -33,8 +32,7 @@
     dest: "/etc/nginx/nginx.conf"
     line: "log_format custom '{{ keystone_nginx_access_log_format_combined }} {{ keystone_nginx_access_log_format_extras }}';"
   notify:
-    - Restart web server on first node
+    - Restart web server
-    - Restart web server on other nodes
 
 # Configure app
 - name: Configure virtual hosts
@@ -43,8 +41,7 @@
     dest: "/etc/nginx/{{ keystone_nginx_conf_path }}/{{ item }}.conf"
   with_items: "{{ keystone_wsgi_program_names }}"
   notify:
-    - Restart web server on first node
+    - Restart web server
-    - Restart web server on other nodes
 
 - name: Link to enable virtual hosts
   file:
@@ -54,5 +51,4 @@
   with_items: "{{ keystone_wsgi_program_names }}"
   when: ansible_os_family == "Debian"
   notify:
-    - Restart web server on first node
+    - Restart web server
-    - Restart web server on other nodes

tasks/keystone_post_install.yml

@@ -46,10 +46,8 @@
       config_type: "json"
       content: "{{ keystone_policy_user_content | default('{}', true) }}"
   notify:
-    - Restart uWSGI on first node
+    - Restart uWSGI
-    - Restart uWSGI on other nodes
+    - Restart web server
-    - Restart web server on first node
-    - Restart web server on other nodes
 
 - name: Copy Keystone Federation SP SSO callback template
   copy:
@@ -61,10 +59,8 @@
   when:
     - keystone_idp != {}
   notify:
-    - Restart uWSGI on first node
+    - Restart uWSGI
-    - Restart uWSGI on other nodes
+    - Restart web server
-    - Restart web server on first node
-    - Restart web server on other nodes
 
 - name: Clean up Keystone Federation SP SSO callback template
   file:
@@ -73,7 +69,5 @@
   when:
     - keystone_idp == {}
   notify:
-    - Restart uWSGI on first node
+    - Restart uWSGI
-    - Restart uWSGI on other nodes
+    - Restart web server
-    - Restart web server on first node
-    - Restart web server on other nodes

tasks/keystone_ssl_key_create.yml

@@ -29,8 +29,7 @@
     -extensions v3_ca
     creates={{ keystone_ssl_cert }}
   notify:
-    - Restart web server on first node
+    - Restart web server
-    - Restart web server on other nodes
 
 - name: Ensure keystone user owns the self-signed key and certificate
   file:
@@ -42,5 +41,4 @@
     - "{{ keystone_ssl_key }}"
     - "{{ keystone_ssl_cert }}"
   notify:
-    - Restart web server on first node
+    - Restart web server
-    - Restart web server on other nodes

tasks/keystone_ssl_key_distribute.yml

@@ -21,8 +21,7 @@
     group: "{{ keystone_system_group_name }}"
     mode: "0640"
   notify:
-    - Restart web server on first node
+    - Restart web server
-    - Restart web server on other nodes
 
 - name: Distribute self signed ssl cert
   copy:
@@ -32,8 +31,7 @@
     group: "{{ keystone_system_group_name }}"
     mode: "0640"
   notify:
-    - Restart web server on first node
+    - Restart web server
-    - Restart web server on other nodes
 
 - name: Ensure keystone user owns the self-signed key and certificate
   file:
@@ -44,5 +42,4 @@
     - "{{ keystone_ssl_key }}"
     - "{{ keystone_ssl_cert }}"
   notify:
-    - Restart web server on first node
+    - Restart web server
-    - Restart web server on other nodes

tasks/keystone_ssl_user_provided.yml

@@ -22,8 +22,7 @@
     mode: "0644"
   when: keystone_user_ssl_cert is defined
   notify:
-    - Restart web server on first node
+    - Restart web server
-    - Restart web server on other nodes
 
 - name: Drop user provided ssl key
   copy:
@@ -34,8 +33,7 @@
     mode: "0640"
   when: keystone_user_ssl_key is defined
   notify:
-    - Restart web server on first node
+    - Restart web server
-    - Restart web server on other nodes
 
 - name: Drop user provided ssl CA cert
   copy:
@@ -46,5 +44,4 @@
     mode: "0644"
   when: keystone_user_ssl_ca_cert is defined
   notify:
-    - Restart web server on first node
+    - Restart web server
-    - Restart web server on other nodes

tasks/keystone_uwsgi.yml

@@ -27,8 +27,7 @@
     config_type: ini
   with_items: "{{ keystone_wsgi_program_names }}"
   notify:
-    - Restart uWSGI on first node
+    - Restart uWSGI
-    - Restart uWSGI on other nodes
 
 - include: "keystone_init_{{ ansible_service_mgr }}.yml"
   vars:

tasks/main.yml

@@ -53,12 +53,14 @@
 - name: refresh local facts
   setup:
     filter: ansible_local
+    gather_subset: "!all"
   tags:
     - keystone-config
 
 - include: keystone_cleanup_old_facts.yml
   when:
-    - "ansible_local['keystone'] is defined"
+    - "('keystone' in ansible_local) or
+       ('need_db_sync' in ansible_local['openstack_ansible']['keystone'])"
   tags:
     - keystone-config
 
@@ -92,19 +94,8 @@
 
 - include: keystone_db_setup.yml
   when:
-    - keystone_database_enabled | bool
+    - "keystone_database_enabled | bool"
-    - inventory_hostname == ansible_play_hosts[0]
+    - "inventory_hostname == ansible_play_hosts[0]"
-    - ansible_local['openstack_ansible']['keystone']['need_db_sync'] | bool
-  tags:
-    - keystone-config
-
-- name: Disable the need of a db sync on all nodes
-  ini_file:
-    dest: "/etc/ansible/facts.d/openstack_ansible.fact"
-    section: keystone
-    option: need_db_sync
-    value: False
-  when: dbsync | succeeded
   tags:
     - keystone-config