Apply Swift rolling upgrade

Apply Swift rolling upgrade based on recommendations from Swift PTL John
Dickinson at [1]

[1] https://www.swiftstack.com/blog/2013/12/20/upgrade-openstack-swift-no-downtime/

Co-Authored-By: Surya Prakash <singh.surya64mnnit@gmail.com>
Change-Id: I99f505438916be2f89b24df20506339604e5bd6e
Implements: blueprint apply-service-upgrade-procedure

ansible/roles/swift/defaults/main.yml

@@ -70,3 +70,5 @@ openstack_swift_auth: "{{ openstack_auth }}"
 
 syslog_server: "{{ api_interface_address }}"
 syslog_swift_facility: "local0"
+
+swift_enable_rolling_upgrade: "yes"

ansible/roles/swift/tasks/deploy.yml

@@ -4,5 +4,7 @@
 - include_tasks: config.yml
 
 - include_tasks: start.yml
+  vars:
+    run_start_swift_proxy_server: "yes"
 
 - include_tasks: check.yml

ansible/roles/swift/tasks/legacy_upgrade.yml

@@ -1,4 +1,6 @@
 ---
+- include_tasks: config.yml
+
 # NOTE(pbourke): because the images that are used to start Swift containers
 # were refactored for Mitaka, we need to completely remove the existing
 # containers or we get a conflict when attempting to start the new ones.
@@ -6,9 +8,7 @@
   become: true
   kolla_docker:
     name: "{{ item }}"
-    # TODO(pbourke): Swift recommend using a SIGHUP to gracefully stop the
-    # services. Update once kolla_docker supports this.
-    action: remove_container
+    action: "stop_container"
   with_items:
     - "swift_rsyncd"
     - "swift_account_server"
@@ -27,3 +27,5 @@
     - "swift_proxy_server"
 
 - include_tasks: start.yml
+  vars:
+    run_start_swift_proxy_server: "yes"

ansible/roles/swift/tasks/reconfigure.yml

@@ -114,6 +114,8 @@
     - "{{ check_results.results }}"
 
 - include_tasks: start.yml
+  vars:
+    run_start_swift_proxy_server: "yes"
   when: remove_containers.changed
 
 - name: Restart containers

ansible/roles/swift/tasks/rolling_upgrade.yml

@@ -0,0 +1,63 @@
+---
+- include_tasks: pull.yml
+
+# Upgrade swift storage nodes first
+- include_tasks: config.yml
+  when: inventory_hostname in groups['swift-account-server'] or
+        inventory_hostname in groups['swift-container-server'] or
+        inventory_hostname in groups['swift-object-server']
+
+# TODO(donghm): Below tasks should be run in serial nodes to
+# decrease downtime. Update once the module ansible strategy for
+# rolling upgrade will be finished.
+- name: Gracefully shutdown swift services in storage nodes
+  become: true
+  kolla_docker:
+    action: "stop_container"
+    name: "{{ item }}"
+  with_items:
+    - "swift_rsyncd"
+    - "swift_account_server"
+    - "swift_account_auditor"
+    - "swift_account_replicator"
+    - "swift_account_reaper"
+    - "swift_container_server"
+    - "swift_container_auditor"
+    - "swift_container_replicator"
+    - "swift_container_updater"
+    - "swift_object_server"
+    - "swift_object_auditor"
+    - "swift_object_replicator"
+    - "swift_object_updater"
+    - "swift_object_expirer"
+  when: inventory_hostname in groups['swift-account-server'] or
+        inventory_hostname in groups['swift-container-server'] or
+        inventory_hostname in groups['swift-object-server']
+
+- include_tasks: start.yml
+  vars:
+    run_start_swift_proxy_server: "no"
+
+# Upgrade swift proxy nodes
+- include_tasks: config.yml
+  when: inventory_hostname in groups['swift-proxy-server']
+
+- name: Gracefully shutdown swift proxy services in proxy nodes
+  become: true
+  kolla_docker:
+    action: "stop_container"
+    name: "swift_proxy_server"
+  when: inventory_hostname in groups['swift-proxy-server']
+
+- name: Start new swift proxy server container
+  become: true
+  kolla_docker:
+    action: "start_container"
+    common_options: "{{ docker_common_options }}"
+    image: "{{ swift_proxy_server_image_full }}"
+    name: "swift_proxy_server"
+    volumes:
+      - "{{ node_config_directory }}/swift/:/var/lib/kolla/swift/:ro"
+      - "{{ node_config_directory }}/swift-proxy-server/:{{ container_config_directory }}/:ro"
+      - "/etc/localtime:/etc/localtime:ro"
+  when: inventory_hostname in groups['swift-proxy-server']

ansible/roles/swift/tasks/start.yml

@@ -246,4 +246,5 @@
       - "{{ node_config_directory }}/swift/:/var/lib/kolla/swift/:ro"
       - "{{ node_config_directory }}/swift-proxy-server/:{{ container_config_directory }}/:ro"
       - "/etc/localtime:/etc/localtime:ro"
-  when: inventory_hostname in groups['swift-proxy-server']
+  when: inventory_hostname in groups['swift-proxy-server'] and
+        run_start_swift_proxy_server | bool

ansible/roles/swift/tasks/upgrade.yml

@@ -1,4 +1,6 @@
 ---
-- include_tasks: config.yml
+- include_tasks: rolling_upgrade.yml
+  when: swift_enable_rolling_upgrade | bool
 
-- include_tasks: stop_and_start.yml
+- include_tasks: legacy_upgrade.yml
+  when: not swift_enable_rolling_upgrade | bool

etc/kolla/globals.yml

@@ -481,6 +481,13 @@ ironic_dnsmasq_dhcp_range:
 # filesystems' labels prepared for swift.
 #swift_devices_name: "KOLLA_SWIFT_DATA"
 
+# Configure swift upgrade option, due to currently kolla support
+# two upgrade ways for swift: legacy_upgrade and rolling_upgrade
+# The variable "swift_enable_rolling_upgrade: yes" is meaning rolling_upgrade
+# were enabled and opposite
+# Rolling upgrade were enable by default
+#swift_enable_rolling_upgrade: "yes"
+
 
 ################################################
 # Tempest - The OpenStack Integration Test Suite

releasenotes/notes/implement-swift-rolling-upgrade-bb715b173263f844.yaml

@@ -0,0 +1,5 @@
+---
+features:
+  - |
+    Implement Swift rolling upgrade logic, enabled by default at
+    swift_enable_rolling_upgrade: "yes" in etc/kolla/globals.yml file.