Implement Keystone zero-downtime upgrade

1- Expand and migrate database in first keystone node
  2- Upgrade all nodes sequentially along with updation of each node's
     configuration file with latest release version
  3- Last keystone node, contract database

With this patch, there is small downtime when all containers are
restarted. It will be fixed in other patch.

[1] http://docs.openstack.org/developer/keystone/upgrading.html#upgrading-without-downtime

Co-Authored-By: Surya Prakash Singh <surya.singh@nectechnologies.in>
Co-Authored-By: Eduardo Gonzalez <dabarren@gmail.com>
Co-Authored-By: Duong Ha-Quang <duonghq@vn.fujitsu.com>

Partially-Implements: blueprint ks-rolling-upgrade-role

Change-Id: I2159af567c40848840ff5e483e7d1f6de760b435

ansible/roles/keystone/handlers/main.yml

@@ -1,4 +1,30 @@
 ---
+# NOTE: Handler order matter, init database upgrade must be before other handlers
+# finish database upgrade must be the last handler.
+
+- name: Init keystone database upgrade
+  vars:
+    service_name: "keystone"
+    service: "{{ keystone_services[service_name] }}"
+  kolla_docker:
+    action: "start_container"
+    common_options: "{{ docker_common_options }}"
+    detach: False
+    environment:
+      KOLLA_UPGRADE:
+      KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}"
+    image: "{{ service.image }}"
+    labels:
+      KOLLA_UPGRADE:
+    name: "init_upgrade_database"
+    restart_policy: "never"
+    volumes: "{{ service.volumes|reject('equalto', '')|list }}"
+  run_once: True
+  when:
+    - action == "upgrade"
+    - service.enabled | bool
+    - inventory_hostname == groups[service.group][0]
+
 - name: Restart keystone container
   vars:
     service_name: "keystone"
@@ -68,3 +94,26 @@
     - config_json.changed | bool
       or keystone_ssh_confs.changed | bool
       or keystone_ssh_container.changed | bool
+
+- name: Finish keystone database upgrade
+  vars:
+    service_name: "keystone"
+    service: "{{ keystone_services[service_name] }}"
+  kolla_docker:
+    action: "start_container"
+    common_options: "{{ docker_common_options }}"
+    detach: False
+    environment:
+      KOLLA_FINISH_UPGRADE:
+      KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}"
+    image: "{{ service.image }}"
+    labels:
+      KOLLA_UPGRADE:
+    name: "finish_upgrade_database"
+    restart_policy: "never"
+    volumes: "{{ service.volumes|reject('equalto', '')|list }}"
+  run_once: True
+  when:
+    - action == "upgrade"
+    - service.enabled | bool
+    - inventory_hostname == groups[service.group][-1]

ansible/roles/keystone/tasks/upgrade.yml

@@ -1,7 +1,42 @@
 ---
 - include: config.yml
 
-- include: bootstrap_service.yml
+- name: Enable log_bin_trust_function_creators function
+  kolla_toolbox:
+    module_name: mysql_variables
+    module_args:
+      login_host: "{{ database_address }}"
+      login_port: "{{ database_port }}"
+      login_user: "{{ database_user }}"
+      login_password: "{{ database_password }}"
+      variable: log_bin_trust_function_creators
+      value: 1
+  run_once: True
+  when: inventory_hostname == groups['keystone'][0]
+
+- name: Init keystone database upgrade
+  command: /bin/true
+  notify: Init keystone database upgrade
+  when: inventory_hostname == groups['keystone'][0]
+
+- name: Finish keystone database upgrade
+  command: /bin/true
+  notify: Finish keystone database upgrade
+  when: inventory_hostname == groups['keystone'][-1]
 
 - name: Flush handlers
   meta: flush_handlers
+
+- name: Disable log_bin_trust_function_creators function
+  kolla_toolbox:
+    module_name: mysql_variables
+    module_args:
+      login_host: "{{ database_address }}"
+      login_port: "{{ database_port }}"
+      login_user: "{{ database_user }}"
+      login_password: "{{ database_password }}"
+      variable: log_bin_trust_function_creators
+      value: 0
+  run_once: True
+  when: inventory_hostname == groups['keystone'][-1]
+

releasenotes/notes/implement-keystone-zero-downtime-upgrade-8e11d080bcda0cf6.yaml

@@ -0,0 +1,3 @@
+---
+features:
+  - Implement keystone zero-downtime upgrade procedure