Implement zero downtime upgrades

This patch implements upgrading keystone with zero downtime as the
default installation process. Handlers have been modified to ensure that
the first keystone node is stopped, facilitates the database migrations,
and that it is started and available before restarting any other keystone
nodes. Migrations also now only occur when there is a change within the
installed keystone venv.

This process is documented at
http://docs.openstack.org/developer/keystone/upgrading.html#upgrading-without-downtime

A new test scenario has been added for testing basic upgradability
between releases.

Implements: blueprint upgrade-testing
Change-Id: I0d3cfcb80b64d005d60f4c8445f991855f844796

handlers/main.yml

@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-- name: Restart service
+- name: Restart service on first node
   service:
     name: "{{ keystone_system_service_name }}"
     state: restarted
@@ -22,9 +22,13 @@
   until: _restart|success
   retries: 5
   delay: 2
-  when: (keystone_apache_enabled | bool) or (keystone_mod_wsgi_enabled | bool)
+  when:
+    - inventory_hostname == groups['keystone_all'][0]
+    - (keystone_apache_enabled | bool) or (keystone_mod_wsgi_enabled | bool)
+  notify:
+    - Wait for keystone service port
 
-- name: Restart Nginx
+- name: Restart Nginx on first node
   service:
     name: nginx
     state: restarted
@@ -32,9 +36,11 @@
   until: keystone_restart | success
   retries: 5
   delay: 2
-  when: not keystone_apache_enabled | bool
+  when:
+    - inventory_hostname == groups['keystone_all'][0]
+    - not keystone_apache_enabled | bool
 
-- name: Restart Keystone APIs
+- name: Restart Keystone APIs on first node
   service:
     name: "{{ item }}"
     state: "restarted"
@@ -43,7 +49,58 @@
   retries: 5
   delay: 2
   with_items: "{{ keystone_wsgi_program_names }}"
-  when: not keystone_mod_wsgi_enabled | bool
+  when:
+    - inventory_hostname == groups['keystone_all'][0]
+    - not keystone_mod_wsgi_enabled | bool
+  notify:
+    - Wait for keystone service port
+
+- name: Wait for keystone service port
+  wait_for:
+    port: "{{ keystone_service_port }}"
+    timeout: 25
+    delay: 10
+  register: keystone_wait_check
+  until: keystone_wait_check | success
+  retries: 5
+
+- name: Restart service on other nodes
+  service:
+    name: "{{ keystone_system_service_name }}"
+    state: restarted
+    pattern: "{{ keystone_system_service_name }}"
+  register: _restart
+  until: _restart|success
+  retries: 5
+  delay: 2
+  when:
+    - inventory_hostname != groups['keystone_all'][0]
+    - (keystone_apache_enabled | bool) or (keystone_mod_wsgi_enabled | bool)
+
+- name: Restart Nginx on other nodes
+  service:
+    name: nginx
+    state: restarted
+  register: keystone_restart
+  until: keystone_restart | success
+  retries: 5
+  delay: 2
+  when:
+    - inventory_hostname != groups['keystone_all'][0]
+    - not keystone_apache_enabled | bool
+
+- name: Restart Keystone APIs on other nodes
+  service:
+    name: "{{ item }}"
+    state: "restarted"
+  register: keystone_restart
+  until: keystone_restart | success
+  retries: 5
+  delay: 2
+  with_items: "{{ keystone_wsgi_program_names }}"
+  when:
+    - inventory_hostname != groups['keystone_all'][0]
+    - not keystone_mod_wsgi_enabled | bool
 
 - name: Restart Shibd
   service:
@@ -54,3 +111,8 @@
   until: shibd_restart|success
   retries: 5
   delay: 2
+
+- name: Perform a Keystone DB sync contract
+  command: "{{ keystone_bin }}/keystone-manage db_sync --contract"
+  become: yes
+  become_user: "{{ keystone_system_user_name }}"

releasenotes/notes/os-keystone-zero-downtime-upgrade-5f19ab84183490b9.yaml

@@ -0,0 +1,5 @@
+---
+features:
+  - The os_keystone role now performs a rolling upgrade
+    without downtime during installation. This process
+    is documented `here <http://docs.openstack.org/developer/keystone/upgrading.html#upgrading-without-downtime>`_.

tasks/keystone_apache.yml

@@ -43,7 +43,8 @@
   when:
     - ansible_pkg_mgr == 'apt'
   notify:
-    - Restart service
+    - Restart service on first node
+    - Restart service on other nodes
 
 ## NOTE(andymccr):
 ## We need to enable a module for httpd on RedHat/CentOS using LoadModule inside conf files
@@ -55,7 +56,8 @@
   when:
    - ansible_pkg_mgr == 'yum'
   notify:
-    - Restart service
+    - Restart service on first node
+    - Restart service on other nodes
 
 - name: Drop apache2 config files
   template:
@@ -65,7 +67,8 @@
     group: "root"
   with_items: "{{ keystone_apache_configs }}"
   notify:
-    - Restart service
+    - Restart service on first node
+    - Restart service on other nodes
 
 - name: Disable default apache site
   file:
@@ -73,7 +76,8 @@
     state: "absent"
   with_items: "{{ keystone_apache_default_sites }}"
   notify:
-    - Restart service
+    - Restart service on first node
+    - Restart service on other nodes
 
 - name: Enabled keystone vhost
   file:
@@ -84,14 +88,16 @@
     - keystone_apache_site_available is defined
     - keystone_apache_site_enabled is defined
   notify:
-    - Restart service
+    - Restart service on first node
+    - Restart service on other nodes
 
 - name: Ensure Apache ServerName
   lineinfile:
     dest: "{{ keystone_apache_conf }}"
     line: "ServerName {{ ansible_hostname }}"
   notify:
-    - Restart service
+    - Restart service on first node
+    - Restart service on other nodes
 
 - name: Ensure Apache ServerTokens
   lineinfile:
@@ -99,7 +105,8 @@
     regexp: '^ServerTokens'
     line: "ServerTokens {{ keystone_apache_servertokens }}"
   notify:
-    - Restart service
+    - Restart service on first node
+    - Restart service on other nodes
 
 - name: Ensure Apache ServerSignature
   lineinfile:
@@ -107,7 +114,8 @@
     regexp: '^ServerSignature'
     line: "ServerSignature {{ keystone_apache_serversignature }}"
   notify:
-    - Restart service
+    - Restart service on first node
+    - Restart service on other nodes
 
 ## NOTE(mgariepy):
 ## We need to enable httpd on CentOS if not it won't start when the container is restarted.
@@ -116,4 +124,5 @@
     name: "{{ keystone_system_service_name }}"
     enabled: "yes"
   notify:
-    - Restart service
+    - Restart service on first node
+    - Restart service on other nodes

tasks/keystone_db_setup.yml

@@ -13,8 +13,30 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-- name: Perform a Keystone DB sync
+- name: Ensure keystone service stopped on first node
-  command: "{{ keystone_bin }}/keystone-manage db_sync"
+  service:
-  changed_when: false
+    name: "{{ item }}"
+    state: stopped
+  register: keystone_stop
+  failed_when:
+    - keystone_stop.msg is defined
+    - "'no service or tool' not in keystone_stop.msg"
+    - "'systemd could not find' not in keystone_stop.msg"
+    - "'Could not find the requested service' not in keystone_stop.msg"
+  with_items:
+    - "{{ keystone_wsgi_program_names }}"
+    - "{{ keystone_system_service_name }}"
+
+- name: Perform a Keystone DB sync expand
+  command: "{{ keystone_bin }}/keystone-manage db_sync --expand"
+  changed_when: true
   become: yes
   become_user: "{{ keystone_system_user_name }}"
+
+- name: Perform a Keystone DB sync migrate
+  command: "{{ keystone_bin }}/keystone-manage db_sync --migrate"
+  changed_when: true
+  become: yes
+  become_user: "{{ keystone_system_user_name }}"
+  notify:
+    - Perform a Keystone DB sync contract

tasks/keystone_federation_sp_setup.yml

@@ -33,7 +33,8 @@
   changed_when: false
   when: inventory_hostname == groups['keystone_all'][0]
   notify:
-    - Restart service
+    - Restart service on first node
+    - Restart service on other nodes
     - Restart Shibd
 
 - name: Store Shibboleth SP key-pair
@@ -70,7 +71,8 @@
   delay: 2
   when: inventory_hostname != groups['keystone_all'][0]
   notify:
-    - Restart service
+    - Restart service on first node
+    - Restart service on other nodes
     - Restart Shibd
 
 - name: Set appropriate file ownership on the Shibboleth SP key-pair
@@ -83,5 +85,6 @@
     - "/etc/shibboleth/sp-key.pem"
   when: inventory_hostname != groups['keystone_all'][0]
   notify:
-    - Restart service
+    - Restart service on first node
+    - Restart service on other nodes
     - Restart Shibd

tasks/keystone_idp_metadata.yml

@@ -20,5 +20,7 @@
   become_user: "{{ keystone_system_user_name }}"
   when: keystone_idp != {}
   notify:
-    - Restart Keystone APIs
+    - Restart Keystone APIs on first node
-    - Restart service
+    - Restart Keystone APIs on other nodes
+    - Restart service on first node
+    - Restart service on other nodes

tasks/keystone_idp_self_signed_create.yml

@@ -33,7 +33,8 @@
   when: >
     inventory_hostname == groups['keystone_all'][0]
   notify:
-    - Restart service
+    - Restart service on first node
+    - Restart service on other nodes
 
 - name: Set appropriate file ownership on the IdP self-signed cert
   file:

tasks/keystone_idp_self_signed_distribute.yml

@@ -30,7 +30,8 @@
   retries: 5
   delay: 2
   notify:
-    - Restart service
+    - Restart service on first node
+    - Restart service on other nodes
 
 - name: Set appropriate file ownership on the IdP self-signed cert
   file:

tasks/keystone_init_systemd.yml

@@ -45,4 +45,7 @@
   command: "systemctl daemon-reload"
   when: systemd_init | changed
   notify:
-    - Restart Keystone APIs
+    - Restart Keystone APIs on first node
+    - Restart Keystone APIs on other nodes
+    - Restart service on first node
+    - Restart service on other nodes

tasks/keystone_init_upstart.yml

@@ -21,11 +21,18 @@
     owner: "root"
     group: "root"
   register: upstart_init
-  notify: Restart Keystone APIs
+  notify:
+    - Restart Keystone APIs on first node
+    - Restart Keystone APIs on other nodes
+    - Restart service on first node
+    - Restart service on other nodes
 
 - name: Reload init scripts
   command: initctl reload-configuration
   changed_when: false
   when: upstart_init | changed
   notify:
-    - Restart Keystone APIs
+    - Restart Keystone APIs on first node
+    - Restart Keystone APIs on other nodes
+    - Restart service on first node
+    - Restart service on other nodes

tasks/keystone_install.yml

@@ -84,8 +84,10 @@
     - not keystone_developer_mode | bool
     - keystone_get_venv | changed or keystone_venv_dir | changed
   notify:
-    - Restart Keystone APIs
+    - Restart Keystone APIs on first node
-    - Restart service
+    - Restart Keystone APIs on other nodes
+    - Restart service on first node
+    - Restart service on other nodes
 
 - name: Install pip packages
   pip:
@@ -103,8 +105,10 @@
   delay: 2
   when: keystone_developer_mode | bool
   notify:
-    - Restart Keystone APIs
+    - Restart Keystone APIs on first node
-    - Restart service
+    - Restart Keystone APIs on other nodes
+    - Restart service on first node
+    - Restart service on other nodes
 
 - name: Update virtualenv path
   command: >

tasks/keystone_ldap_setup.yml

@@ -35,8 +35,10 @@
     mode: "0644"
   with_dict: "{{ keystone_ldap }}"
   notify:
-    - Restart Keystone APIs
+    - Restart Keystone APIs on first node
-    - Restart service
+    - Restart Keystone APIs on other nodes
+    - Restart service on first node
+    - Restart service on other nodes
 
 # Bug 1547542 - Older versions of the keystone role would deploy a blank
 # keystone.Default.conf and this will cause errors when adding LDAP-backed
@@ -47,5 +49,7 @@
     state: absent
   when: keystone_ldap.Default is not defined
   notify:
-    - Restart Keystone APIs
+    - Restart Keystone APIs on first node
-    - Restart service
+    - Restart Keystone APIs on other nodes
+    - Restart service on first node
+    - Restart service on other nodes

tasks/keystone_nginx.yml

@@ -24,14 +24,18 @@
   file:
     path: /etc/nginx/sites-enabled/default
     state: absent
-  notify: Restart Nginx
+  notify:
+    - Restart Nginx on first node
+    - Restart Nginx on other nodes
 
 - name: Configure custom nginx log format
   lineinfile:
     insertbefore: access_log
     dest: "/etc/nginx/nginx.conf"
     line: "log_format custom '{{ keystone_nginx_access_log_format_combined }} {{ keystone_nginx_access_log_format_extras }}';"
-  notify: Restart Nginx
+  notify:
+    - Restart Nginx on first node
+    - Restart Nginx on other nodes
 
 # Configure app
 - name: Configure virtual hosts
@@ -39,7 +43,9 @@
     src: keystone_nginx.conf.j2
     dest: "/etc/nginx/{{ keystone_nginx_conf_path }}/{{ item }}.conf"
   with_items: "{{ keystone_wsgi_program_names }}"
-  notify: Restart Nginx
+  notify:
+    - Restart Nginx on first node
+    - Restart Nginx on other nodes
 
 - name: Link to enable virtual hosts
   file:
@@ -48,4 +54,6 @@
     state: link
   with_items: "{{ keystone_wsgi_program_names }}"
   when: ansible_os_family == "Debian"
-  notify: Restart Nginx
+  notify:
+    - Restart Nginx on first node
+    - Restart Nginx on other nodes

tasks/keystone_post_install.yml

@@ -36,8 +36,10 @@
       config_overrides: "{{ keystone_policy_overrides }}"
       config_type: "json"
   notify:
-    - Restart Keystone APIs
+    - Restart Keystone APIs on first node
-    - Restart service
+    - Restart Keystone APIs on other nodes
+    - Restart service on first node
+    - Restart service on other nodes
 
 - name: Copy Keystone Federation SP SSO callback template
   copy:
@@ -49,8 +51,10 @@
   when:
     - keystone_idp != {}
   notify:
-    - Restart Keystone APIs
+    - Restart Keystone APIs on first node
-    - Restart service
+    - Restart Keystone APIs on other nodes
+    - Restart service on first node
+    - Restart service on other nodes
 
 - name: Clean up Keystone Federation SP SSO callback template
   file:
@@ -59,5 +63,7 @@
   when:
     - keystone_idp == {}
   notify:
-    - Restart Keystone APIs
+    - Restart Keystone APIs on first node
-    - Restart service
+    - Restart Keystone APIs on other nodes
+    - Restart service on first node
+    - Restart service on other nodes

tasks/keystone_ssl_key_create.yml

@@ -29,7 +29,8 @@
     -extensions v3_ca
     creates={{ keystone_ssl_cert }}
   notify:
-    - Restart service
+    - Restart service on first node
+    - Restart service on other nodes
 
 - name: Ensure keystone user owns the self-signed key and certificate
   file:
@@ -41,4 +42,5 @@
     - "{{ keystone_ssl_key }}"
     - "{{ keystone_ssl_cert }}"
   notify:
-    - Restart service
+    - Restart service on first node
+    - Restart service on other nodes

tasks/keystone_ssl_user_provided.yml

@@ -22,7 +22,8 @@
     mode: "0644"
   when: keystone_user_ssl_cert is defined
   notify:
-    - Restart service
+    - Restart service on first node
+    - Restart service on other nodes
 
 - name: Drop user provided ssl key
   copy:
@@ -33,7 +34,8 @@
     mode: "0640"
   when: keystone_user_ssl_key is defined
   notify:
-    - Restart service
+    - Restart service on first node
+    - Restart service on other nodes
 
 - name: Drop user provided ssl CA cert
   copy:
@@ -44,4 +46,5 @@
     mode: "0644"
   when: keystone_user_ssl_ca_cert is defined
   notify:
-    - Restart service
+    - Restart service on first node
+    - Restart service on other nodes

tasks/keystone_uwsgi.yml

@@ -27,7 +27,9 @@
     config_overrides: "{{ keystone_uwsgi_ini_overrides }}"
     config_type: ini
   with_items: "{{ keystone_wsgi_program_names }}"
-  notify: Restart Keystone APIs
+  notify:
+    - Restart Keystone APIs on first node
+    - Restart Keystone APIs on other nodes
 
 - include: keystone_init_common.yml
   vars:
@@ -36,7 +38,9 @@
     system_user: "{{ keystone_system_user_name }}"
     system_group: "{{ keystone_system_group_name }}"
     service_home: "{{ keystone_system_user_home }}"
-  notify: Restart Keystone APIs
+  notify:
+    - Restart Keystone APIs on first node
+    - Restart Keystone APIs on other nodes
 
 - include: keystone_init_common.yml
   vars:
@@ -45,7 +49,9 @@
     system_user: "{{ keystone_system_user_name }}"
     system_group: "{{ keystone_system_group_name }}"
     service_home: "{{ keystone_system_user_home }}"
-  notify: Restart Keystone APIs
+  notify:
+    - Restart Keystone APIs on first node
+    - Restart Keystone APIs on other nodes
 
 - name: Ensure uwsgi service started
   service:

tasks/main.yml

@@ -85,6 +85,7 @@
   when:
     - keystone_database_enabled | bool
     - inventory_hostname == groups['keystone_all'][0]
+    - keystone_get_venv | changed or keystone_venv_dir | changed or install_packages | changed
   tags:
     - keystone-install
 

tests/ansible-role-requirements.yml

@@ -38,6 +38,10 @@
   src: https://git.openstack.org/openstack/openstack-ansible-openstack_openrc
   scm: git
   version: master
+- name: os_previous_keystone
+  src: https://git.openstack.org/openstack/openstack-ansible-os_keystone
+  scm: git
+  version: stable/newton
 - name: os_tempest
   src: https://git.openstack.org/openstack/openstack-ansible-os_tempest
   scm: git

tests/test-install-previous-keystone.yml

@@ -0,0 +1,35 @@
+---
+# Copyright 2015, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: Playbook for deploying previous keystone
+  hosts: keystone_all
+  user: root
+  gather_facts: true
+  pre_tasks:
+    - name: Set keystone_messaging fact
+      set_fact:
+        keystone_messaging_enabled: "{{ groups['rabbitmq_all'] is defined }}"
+    - include: common/ensure-rabbitmq.yml
+      vhost_name: "{{ keystone_rabbitmq_vhost }}"
+      user_name: "{{ keystone_rabbitmq_userid }}"
+      user_password: "{{ keystone_rabbitmq_password }}"
+      when: "{{ groups['rabbitmq_all'] is defined }}"
+    - include: common/create-grant-db.yml
+      db_name: "{{ keystone_galera_database }}"
+      db_password: "{{ keystone_container_mysql_password }}"
+  roles:
+    - role: "os_previous_keystone"
+  vars_files:
+    - common/previous/test-vars.yml

tests/test.yml

@@ -25,6 +25,12 @@
 # Install RabbitMQ/MariaDB
 - include: common/test-install-infra.yml
 
+# Install previous Keystone
+- include: test-install-previous-keystone.yml
+  when:
+    - keystone_upgrade is defined
+    - keystone_upgrade | bool
+
 # Install Keystone
 - include: common/test-install-keystone.yml
 

tox.ini

@@ -110,6 +110,21 @@ commands =
     bash -c "{toxinidir}/tests/common/test-ansible-functional.sh"
 
 
+[testenv:upgrade]
+deps =
+    {[testenv:ansible]deps}
+setenv =
+    {[testenv]setenv}
+    ANSIBLE_PARAMETERS=-vvv -e keystone_upgrade=True
+commands =
+    {[testenv:tests_clone]commands}
+    bash -c "if [ ! -d "{toxinidir}/tests/common/previous" ]; then \
+               git clone https://git.openstack.org/openstack/openstack-ansible-tests -b stable/newton \
+                 {toxinidir}/tests/common/previous; \
+             fi"
+    bash -c "{toxinidir}/tests/common/test-ansible-functional.sh"
+
+
 [testenv:uwsgi_apache]
 deps =
     {[testenv:ansible]deps}