Ensure that services restart in a particular order

Currently when multiple services share a host, the
restart order is random. This is due to an unordered
dict being used to facilitate the mapping of services
to their groups, names and other options.

Based on [1], this patch implements changes to the role
to ensure that services on the same host are restarted
in the correct order when the software/config changes.

[1] https://docs.openstack.org/developer/cinder/upgrade.html#minimal-downtime-upgrade-procedure

Change-Id: If5729671cb69f928df660ec2d9ba83fe3f567946

defaults/main.yml

@@ -269,19 +269,23 @@ cinder_services:
     group: cinder_api
     service_name: cinder-api
     init_config_overrides: "{{ cinder_api_init_overrides }}"
+    start_order: 1
   cinder-scheduler:
     group: cinder_scheduler
     service_name: cinder-scheduler
     init_config_overrides: "{{ cinder_scheduler_init_overrides }}"
+    start_order: 2
   cinder-volume:
     group: cinder_volume
     service_name: cinder-volume
     init_config_overrides: "{{ cinder_volume_init_overrides }}"
+    start_order: 3
   cinder-backup:
     group: cinder_backup
     service_name: cinder-backup
     condition: "{{ cinder_service_backup_program_enabled | bool }}"
     init_config_overrides: "{{ cinder_backup_init_overrides }}"
+    start_order: 4
 
 # This variable is used by the repo_build process to determine
 # which host group to check for members of before building the

handlers/main.yml

@@ -22,11 +22,11 @@
 
 - name: Stop services
   service:
-    name: "{{ item.value.service_name }}"
+    name: "{{ item.service_name }}"
     enabled: yes
     state: stopped
     daemon_reload: "{{ (ansible_service_mgr == 'systemd') | ternary('yes', omit) }}"
-  with_dict: "{{ filtered_cinder_services }}"
+  with_items: "{{ filtered_cinder_services }}"
   register: _stop
   until: _stop | success
   retries: 5
@@ -51,11 +51,11 @@
 
 - name: Start services
   service:
-    name: "{{ item.value.service_name }}"
+    name: "{{ item.service_name }}"
     enabled: yes
     state: "started"
     daemon_reload: "{{ (ansible_service_mgr == 'systemd') | ternary('yes', omit) }}"
-  with_dict: "{{ filtered_cinder_services }}"
+  with_items: "{{ filtered_cinder_services }}"
   register: _start
   until: _start | success
   retries: 5

tasks/cinder_init_systemd.yml

@@ -16,51 +16,51 @@
 
 - name: Create TEMP run dir
   file:
-    path: "/var/run/{{ item.value.service_name }}"
+    path: "/var/run/{{ item.service_name }}"
     state: directory
     owner: "{{ cinder_system_user_name }}"
     group: "{{ cinder_system_group_name }}"
     mode: "02755"
-  with_dict: "{{ filtered_cinder_services }}"
+  with_items: "{{ filtered_cinder_services }}"
 
 - name: Create TEMP lock dir
   file:
-    path: "/var/lock/{{ item.value.service_name }}"
+    path: "/var/lock/{{ item.service_name }}"
     state: directory
     owner: "{{ cinder_system_user_name }}"
     group: "{{ cinder_system_group_name }}"
     mode: "02755"
-  with_dict: "{{ filtered_cinder_services }}"
+  with_items: "{{ filtered_cinder_services }}"
 
 # TODO(mgariepy):
 # Remove this in Pike as it only needed to handle upgrades
 # from Newton->Newton and Newton->Ocata
 - name: Cleanup old tmpfiles.d entry
   file:
-    path: "/etc/tmpfiles.d/{{ item.value.service_name }}.conf"
+    path: "/etc/tmpfiles.d/{{ item.service_name }}.conf"
     state: absent
-  with_dict: "{{ filtered_cinder_services }}"
+  with_items: "{{ filtered_cinder_services }}"
 
 - name: Create tmpfiles.d entry
   template:
     src: "cinder-systemd-tmpfiles.j2"
-    dest: "/etc/tmpfiles.d/openstack-{{ item.value.service_name }}.conf"
+    dest: "/etc/tmpfiles.d/openstack-{{ item.service_name }}.conf"
     mode: "0644"
     owner: "root"
     group: "root"
-  with_dict: "{{ filtered_cinder_services }}"
+  with_items: "{{ filtered_cinder_services }}"
   notify:
     - Restart cinder services
 
 - name: Place the systemd init script
   config_template:
     src: "cinder-systemd-init.j2"
-    dest: "/etc/systemd/system/{{ item.value.service_name }}.service"
+    dest: "/etc/systemd/system/{{ item.service_name }}.service"
     mode: "0644"
     owner: "root"
     group: "root"
-    config_overrides: "{{ item.value.init_config_overrides }}"
+    config_overrides: "{{ item.init_config_overrides }}"
     config_type: "ini"
-  with_dict: "{{ filtered_cinder_services }}"
+  with_items: "{{ filtered_cinder_services }}"
   notify:
     - Restart cinder services

templates/cinder-systemd-init.j2

@@ -11,9 +11,9 @@ User={{ cinder_system_user_name }}
 Group={{ cinder_system_group_name }}
 
 {% if program_override is defined %}
-ExecStart={{ program_override }} {{ program_config_options|default('') }} --log-file=/var/log/cinder/{{ item.value.service_name }}.log
+ExecStart={{ program_override }} {{ program_config_options|default('') }} --log-file=/var/log/cinder/{{ item.service_name }}.log
 {% else %}
-ExecStart={{ cinder_bin }}/{{ item.value.service_name }} {{ program_config_options|default('') }} --log-file=/var/log/cinder/{{ item.value.service_name }}.log
+ExecStart={{ cinder_bin }}/{{ item.service_name }} {{ program_config_options|default('') }} --log-file=/var/log/cinder/{{ item.service_name }}.log
 {% endif %}
 
 # Give a reasonable amount of time for the server to start up/shut down

templates/cinder-systemd-tmpfiles.j2

@@ -1,5 +1,5 @@
 # {{ ansible_managed }}
 
-D /var/lock/{{ item.value.service_name }} 2755 {{ cinder_system_user_name }} {{ cinder_system_group_name }}
-D /var/run/{{ item.value.service_name }} 2755 {{ cinder_system_user_name }} {{ cinder_system_group_name }}
+D /var/lock/{{ item.service_name }} 2755 {{ cinder_system_user_name }} {{ cinder_system_group_name }}
+D /var/run/{{ item.service_name }} 2755 {{ cinder_system_user_name }} {{ cinder_system_group_name }}
 D {{ cinder_lock_path }} 2755 {{ cinder_system_user_name }} {{ cinder_system_group_name }}

vars/main.yml

@@ -34,13 +34,16 @@ cinder_package_list: |-
 #
 # Compile a list of the services on a host based on whether
 # the host is in the host group and the service is enabled.
+# The service list is provided in the defined start order.
 #
-filtered_cinder_services: >
-  {%- set services = cinder_services.copy() %}
-  {%- for key,value in cinder_services.items() %}
-  {%- if value.group not in group_names or
-  (value.condition is defined and not value.condition) %}
-  {%- set _ = services.pop(key) %}
-  {%- endif %}
-  {%- endfor %}
-  {{- services -}}
+filtered_cinder_services: |-
+  {% set services = [] %}
+  {% for key, value in cinder_services.items() %}
+  {%   if (value['group'] in group_names) and
+          (('condition' not in value) or
+           ('condition' in value and value['condition'])) %}
+  {%     set _ = value.update({'service_key': key}) %}
+  {%     set _ = services.append(value) %}
+  {%   endif %}
+  {% endfor %}
+  {{ services | sort(attribute='start_order') }}