Apply Ironic rolling upgrade logic

This patchset apply Ironic rolling upgrade logic [1][2]
[1] https://docs.openstack.org/ironic/latest/contributor/rolling-upgrades.html
[2] https://docs.openstack.org/ironic/latest/admin/upgrade-guide.html#rolling-upgrades

Depends-On: https://review.openstack.org/#/c/575594/

Co-author: Ha Manh Dong <donghm@vn.fujitsu.com>
Change-Id: Id68244951dc66d5c3423ef44324bd72058f4ba67
Implements: blueprint apply-service-upgrade-procedure

ansible/group_vars/all.yml

@@ -352,6 +352,9 @@ openstack_logging_debug: "False"
 
 openstack_region_name: "RegionOne"
 
+# Variable defined the pin_release_version, apply for rolling upgrade process
+openstack_previous_release_name: "queens"
+
 # A list of policy file formats that are supported by Oslo.policy
 supported_policy_format_list:
   - policy.yaml

ansible/roles/ironic/defaults/main.yml

@@ -143,7 +143,7 @@ ironic_dnsmasq_boot_file: "{% if enable_ironic_ipxe | bool %}undionly.kpxe{% els
 ironic_cleaning_network:
 ironic_console_serial_speed: "115200n8"
 ironic_ipxe_url: http://{{ api_interface_address }}:{{ ironic_ipxe_port }}
-
+ironic_enable_rolling_upgrade: "yes"
 
 ####################
 ## Kolla

ansible/roles/ironic/handlers/main.yml

@@ -1,28 +1,4 @@
 ---
-- name: Restart ironic-api container
-  vars:
-    service_name: "ironic-api"
-    service: "{{ ironic_services[service_name] }}"
-    config_json: "{{ ironic_config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}"
-    ironic_conf: "{{ ironic_confs.results|selectattr('item.key', 'equalto', service_name)|first }}"
-    policy_json: "{{ ironic_policy_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}"
-    ironic_api_container: "{{ check_ironic_containers.results|selectattr('item.key', 'equalto', service_name)|first }}"
-  become: true
-  kolla_docker:
-    action: "recreate_or_restart_container"
-    common_options: "{{ docker_common_options }}"
-    name: "{{ service.container_name }}"
-    image: "{{ service.image }}"
-    volumes: "{{ service.volumes|reject('equalto', '')|list }}"
-  when:
-    - kolla_action != "config"
-    - inventory_hostname in groups[service.group]
-    - service.enabled | bool
-    - config_json.changed | bool
-      or ironic_conf.changed | bool
-      or policy_json.changed | bool
-      or ironic_api_container.changed | bool
-
 - name: Restart ironic-conductor container
   vars:
     service_name: "ironic-conductor"
@@ -48,6 +24,30 @@
       or policy_json.changed | bool
       or ironic_conductor_container.changed | bool
 
+- name: Restart ironic-api container
+  vars:
+    service_name: "ironic-api"
+    service: "{{ ironic_services[service_name] }}"
+    config_json: "{{ ironic_config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}"
+    ironic_conf: "{{ ironic_confs.results|selectattr('item.key', 'equalto', service_name)|first }}"
+    policy_json: "{{ ironic_policy_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}"
+    ironic_api_container: "{{ check_ironic_containers.results|selectattr('item.key', 'equalto', service_name)|first }}"
+  become: true
+  kolla_docker:
+    action: "recreate_or_restart_container"
+    common_options: "{{ docker_common_options }}"
+    name: "{{ service.container_name }}"
+    image: "{{ service.image }}"
+    volumes: "{{ service.volumes|reject('equalto', '')|list }}"
+  when:
+    - kolla_action != "config"
+    - inventory_hostname in groups[service.group]
+    - service.enabled | bool
+    - config_json.changed | bool
+      or ironic_conf.changed | bool
+      or policy_json.changed | bool
+      or ironic_api_container.changed | bool
+
 - name: Restart ironic-inspector container
   vars:
     service_name: "ironic-inspector"

ansible/roles/ironic/tasks/bootstrap_service.yml

@@ -2,14 +2,18 @@
 - name: Running Ironic bootstrap container
   vars:
     ironic_api: "{{ ironic_services['ironic-api'] }}"
+    bootstrap_environment:
+      KOLLA_BOOTSTRAP:
+      KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}"
+    upgrade_environment:
+      KOLLA_UPGRADE:
+      KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}"
   become: true
   kolla_docker:
     action: "start_container"
     common_options: "{{ docker_common_options }}"
     detach: False
-    environment:
-      KOLLA_BOOTSTRAP:
-      KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}"
+    environment: "{{ upgrade_environment if ironic_enable_rolling_upgrade|bool else bootstrap_environment }}"
     image: "{{ ironic_api.image }}"
     labels:
       BOOTSTRAP:

ansible/roles/ironic/tasks/legacy_upgrade.yml

@@ -0,0 +1,7 @@
+---
+- include: config.yml
+
+- include: bootstrap_service.yml
+
+- name: Flush handlers
+  meta: flush_handlers

ansible/roles/ironic/tasks/rolling_upgrade.yml

@@ -0,0 +1,44 @@
+---
+- include: pull.yml
+
+# Pin release version
+- include: config.yml
+  vars:
+    pin_release_version: "{{ openstack_previous_release_name }}"
+
+- include: bootstrap_service.yml
+
+# TODO(donghm): Flush_handlers to restart ironic services
+# should be run in serial nodes to decrease downtime. Update when
+# the module ansible strategy for rolling upgrade is finished.
+
+# Restart ironic services with pinned release version
+- name: Flush handlers
+  meta: flush_handlers
+
+# Unpin version
+- include: config.yml
+
+# Restart ironic services with unpinned release version
+- name: Flush handlers
+  meta: flush_handlers
+
+- name: Running Ironic online data migration
+  vars:
+    ironic_api: "{{ ironic_services['ironic-api'] }}"
+  kolla_docker:
+    action: "start_container"
+    common_options: "{{ docker_common_options }}"
+    detach: False
+    environment:
+      KOLLA_OSM:
+      KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}"
+    image: "{{ ironic_api.image }}"
+    labels:
+      BOOTSTRAP:
+    name: "bootstrap_ironic"
+    restart_policy: "never"
+    volumes: "{{ ironic_api.volumes }}"
+  run_once: True
+  delegate_to: "{{ groups[ironic_api.group][0] }}"
+  when: inventory_hostname in groups[ironic_api.group]

ansible/roles/ironic/tasks/upgrade.yml

@@ -1,12 +1,6 @@
 ---
-- include: register.yml
-  when: enable_keystone | bool and
-        (inventory_hostname in groups['ironic-api'] or
-        inventory_hostname in groups['ironic-inspector'])
+- include: rolling_upgrade.yml
+  when: ironic_enable_rolling_upgrade | bool
 
-- include: config.yml
-
-- include: bootstrap_service.yml
-
-- name: Flush handlers
-  meta: flush_handlers
+- include: legacy_upgrade.yml
+  when: not ironic_enable_rolling_upgrade | bool

ansible/roles/ironic/templates/ironic.conf.j2

@@ -15,6 +15,10 @@ log_dir = /var/log/kolla/ironic
 
 transport_url = {{ rpc_transport_url }}
 
+{% if pin_release_version is defined %}
+pin_release_version = {{ pin_release_version }}
+{% endif %}
+
 [oslo_messaging_notifications]
 transport_url = {{ notify_transport_url }}
 

etc/kolla/globals.yml

@@ -398,6 +398,13 @@ ironic_dnsmasq_dhcp_range:
 # PXE bootloader file for Ironic Inspector, relative to /tftpboot.
 #ironic_dnsmasq_boot_file: "pxelinux.0"
 
+# Configure ironic upgrade option, due to currently kolla support
+# two upgrade ways for ironic: legacy_upgrade and rolling_upgrade
+# The variable "ironic_enable_rolling_upgrade: yes" is meaning legacy_upgrade
+# were enabled and opposite
+# Rolling upgrade were enable by default
+#ironic_enable_rolling_upgrade: "yes"
+
 ######################################
 # Manila - Shared File Systems Options
 ######################################

releasenotes/notes/implement-ironic-rolling-upgrade-c45536fe4814212e.yaml

@@ -0,0 +1,5 @@
+---
+features:
+  - |
+    Implement Ironic rolling upgrade logic, enabled by default at
+    ironic_enable_rolling_upgrade: "yes" in etc/kolla/globals.yml file.