Ensure that services restart in a particular order

Currently when multiple services share a host, the
restart order is random. This is due to an unordered
dict being used to facilitate the mapping of services
to their groups, names and other options.

Based on [1], this patch implements changes to the role
to ensure that services on the same host are restarted
in the correct order when the software/config changes.

[1] https://docs.openstack.org/developer/neutron/devref/upgrade.html

Change-Id: I368b51ef37763f4163ead591d6743c4d56962ef9

handlers/main.yml

@@ -13,26 +13,18 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-- name: Restart neutron services
-  command: "/bin/true"
-  notify:
-    - Stop services
-    - Copy new policy file into place
-    - Start services
-
 - name: Stop services
   service:
-    name: "{{ item.value.service_name }}"
+    name: "{{ item.service_name }}"
     enabled: yes
     state: "stopped"
     daemon_reload: "{{ (ansible_service_mgr == 'systemd') | ternary('yes', omit) }}"
-  with_dict: "{{ filtered_neutron_services }}"
+  with_items: "{{ filtered_neutron_services }}"
   register: _stop
   until: _stop | success
   retries: 5
   delay: 2
-  notify:
+  listen: "Restart neutron services"
-    - Run ns-metadata-proxy process cleanup
 
 # NOTE(cloudnull):
 # When installing or upgrading it is possible that an old metadata proxy process will not
@@ -50,7 +42,8 @@
         fi
       fi
     done
-  when: '"neutron-metadata-agent" in filtered_neutron_services'
+  when: "'neutron-metadata-agent' in (filtered_neutron_services | map(attribute='service_key') | list)"
+  listen: "Restart neutron services"
 
 # Note (odyssey4me):
 # The policy.json file is currently read continually by the services
@@ -68,15 +61,17 @@
     group: "{{ neutron_system_group_name }}"
     mode: "0640"
     remote_src: yes
+  listen: "Restart neutron services"
 
 - name: Start services
   service:
-    name: "{{ item.value.service_name }}"
+    name: "{{ item.service_name }}"
     enabled: yes
     state: "started"
     daemon_reload: "{{ (ansible_service_mgr == 'systemd') | ternary('yes', omit) }}"
-  with_dict: "{{ filtered_neutron_services }}"
+  with_items: "{{ filtered_neutron_services }}"
   register: _start
   until: _start | success
   retries: 5
   delay: 2
+  listen: "Restart neutron services"

tasks/neutron_db_setup.yml

@@ -56,13 +56,7 @@
   when:
     - "ansible_local['openstack_ansible']['neutron']['need_db_contract'] | bool"
   run_once: yes
-
+  notify: Restart neutron services
-- name: Start Neutron server
-  service:
-    name: "neutron-server"
-    state: started
-  when:
-    - "ansible_local['openstack_ansible']['neutron']['need_db_contract'] | bool"
 
 - name: Disable the db sync local facts
   ini_file:

tasks/neutron_init_systemd.yml

@@ -15,51 +15,51 @@
 
 - name: Create TEMP run dir
   file:
-    path: "/var/run/{{ item.value.service_name }}"
+    path: "/var/run/{{ item.service_name }}"
     state: directory
     owner: "{{ neutron_system_user_name }}"
     group: "{{ neutron_system_group_name }}"
     mode: "02755"
-  with_dict: "{{ filtered_neutron_services }}"
+  with_items: "{{ filtered_neutron_services }}"
 
 - name: Create TEMP lock dir
   file:
-    path: "/var/lock/{{ item.value.service_name }}"
+    path: "/var/lock/{{ item.service_name }}"
     state: directory
     owner: "{{ neutron_system_user_name }}"
     group: "{{ neutron_system_group_name }}"
     mode: "02755"
-  with_dict: "{{ filtered_neutron_services }}"
+  with_items: "{{ filtered_neutron_services }}"
 
 # TODO(mgariepy):
 # Remove this in Pike as it only needed to handle upgrades
 # from Newton->Newton and Newton->Ocata
 - name: Cleanup old tmpfiles.d entry
   file:
-    path: "/etc/tmpfiles.d/{{ item.value.service_name }}.conf"
+    path: "/etc/tmpfiles.d/{{ item.service_name }}.conf"
     state: absent
-  with_dict: "{{ filtered_neutron_services }}"
+  with_items: "{{ filtered_neutron_services }}"
 
 - name: Create tmpfiles.d entry
   template:
     src: "neutron-systemd-tmpfiles.j2"
-    dest: "/etc/tmpfiles.d/openstack-{{ item.value.service_name }}.conf"
+    dest: "/etc/tmpfiles.d/openstack-{{ item.service_name }}.conf"
     mode: "0644"
     owner: "root"
     group: "root"
-  with_dict: "{{ filtered_neutron_services }}"
+  with_items: "{{ filtered_neutron_services }}"
   notify:
     - Restart neutron services
 
 - name: Place the systemd init script
   config_template:
     src: "neutron-systemd-init.j2"
-    dest: "/etc/systemd/system/{{ item.value.service_name }}.service"
+    dest: "/etc/systemd/system/{{ item.service_name }}.service"
     mode: "0644"
     owner: "root"
     group: "root"
-    config_overrides: "{{ item.value.init_config_overrides }}"
+    config_overrides: "{{ item.init_config_overrides }}"
     config_type: "ini"
-  with_dict: "{{ filtered_neutron_services }}"
+  with_items: "{{ filtered_neutron_services }}"
   notify:
     - Restart neutron services

tasks/neutron_post_install.yml

@@ -73,19 +73,19 @@
 
 - name: Generate neutron agent only Config
   config_template:
-    src: "{{ item.value.service_conf }}.j2"
+    src: "{{ item.service_conf }}.j2"
-    dest: "{{ item.value.service_conf_path }}/{{ item.value.service_conf }}"
+    dest: "{{ item.service_conf_path }}/{{ item.service_conf }}"
     owner: "root"
     group: "{{ neutron_system_group_name }}"
     mode: "0640"
-    config_overrides: "{{ item.value.config_overrides }}"
+    config_overrides: "{{ item.config_overrides }}"
-    config_type: "{{ item.value.config_type }}"
+    config_type: "{{ item.config_type }}"
-  with_dict: "{{ filtered_neutron_services }}"
+  with_items: "{{ filtered_neutron_services }}"
   notify:
     - Restart neutron services
   when:
-    - item.value.service_conf_path is defined
+    - "'service_conf_path' in item"
-    - item.value.service_conf is defined
+    - "'service_conf' in item"
 
 - name: Copy neutron rootwrap filters
   copy:
@@ -100,13 +100,13 @@
 
 - name: Drop neutron agent rootwrap filters
   copy:
-    src: "{{ item.value.service_rootwrap }}"
+    src: "{{ item.service_rootwrap }}"
-    dest: "{{ neutron_conf_dir }}/{{ item.value.service_rootwrap }}"
+    dest: "{{ neutron_conf_dir }}/{{ item.service_rootwrap }}"
     owner: "root"
     group: "root"
-  with_dict: "{{ filtered_neutron_services }}"
+  with_items: "{{ filtered_neutron_services }}"
   when:
-      - item.value.service_rootwrap is defined
+      - "'service_rootwrap' in item"
   notify:
     - Restart neutron services
 

templates/neutron-systemd-init.j2

@@ -7,13 +7,13 @@ After=network.target
 
 [Service]
 Type=simple
-User={{ item.value.system_user | default(neutron_system_user_name) }}
+User={{ item.system_user | default(neutron_system_user_name) }}
-Group={{ item.value.system_group | default(neutron_system_group_name) }}
+Group={{ item.system_group | default(neutron_system_group_name) }}
 
-{% if item.value.program_override is defined %}
+{% if item.program_override is defined %}
-ExecStart={{ item.value.program_override }} {{ item.value.config_options|default('') }}
+ExecStart={{ item.program_override }} {{ item.config_options | default('') }}
 {% else %}
-ExecStart={{ neutron_bin }}/{{ item.value.program_binary | default(item.value.service_name) }} {{ item.value.config_options|default('') }}
+ExecStart={{ neutron_bin }}/{{ item.program_binary | default(item.service_name) }} {{ item.config_options | default('') }}
 {% endif %}
 
 # Give a reasonable amount of time for the server to start up/shut down

templates/neutron-systemd-tmpfiles.j2

@@ -1,5 +1,5 @@
 # {{ ansible_managed }}
 
-D /var/lock/{{ item.value.program_binary | default(item.value.service_name) }} 2755 {{ item.value.system_user | default(neutron_system_user_name) }} {{ item.value.system_group | default(neutron_system_group_name) }}
+D /var/lock/{{ item.program_binary | default(item.service_name) }} 2755 {{ item.system_user | default(neutron_system_user_name) }} {{ item.system_group | default(neutron_system_group_name) }}
-D /var/run/{{ item.value.program_binary | default(item.value.service_name) }} 2755 {{ item.value.system_user | default(neutron_system_user_name) }} {{ item.value.system_group | default(neutron_system_group_name) }}
+D /var/run/{{ item.program_binary | default(item.service_name) }} 2755 {{ item.system_user | default(neutron_system_user_name) }} {{ item.system_group | default(neutron_system_group_name) }}
-D {{ neutron_lock_path }} 2755 {{ item.value.system_user | default(neutron_system_user_name) }} {{ item.value.system_group | default(neutron_system_group_name) }}
+D {{ neutron_lock_path }} 2755 {{ item.system_user | default(neutron_system_user_name) }} {{ item.system_group | default(neutron_system_group_name) }}

vars/main.yml

@@ -303,14 +303,22 @@ dragonflow_pub_sub_multiproc_enabled: "{{ (dragonflow_pub_sub_multiproc_driver |
 ### Services info
 ###
 
-filtered_neutron_services: >
+#
-  {%- set services = neutron_services.copy() %}
+# Compile a list of the services on a host based on whether
-  {%- for key,value in neutron_services.items() %}
+# the host is in the host group and the service is enabled.
-  {%- if value.group not in group_names or not value.service_en %}
+# The service list is provided in the defined start order.
-  {%- set _ = services.pop(key) %}
+#
-  {%- endif %}
+filtered_neutron_services: |-
-  {%- endfor %}
+  {% set services = [] %}
-  {{- services -}}
+  {% for key, value in neutron_services.items() %}
+  {%   if (value['group'] in group_names) and
+          (('service_en' not in value) or
+           ('service_en' in value and value['service_en'])) %}
+  {%     set _ = value.update({'service_key': key}) %}
+  {%     set _ = services.append(value) %}
+  {%   endif %}
+  {% endfor %}
+  {{ services | sort(attribute='start_order') }}
 
 ###
 ### Internals: neutron_services mappings
@@ -328,6 +336,7 @@ neutron_services:
    config_overrides: "{{ neutron_dhcp_agent_ini_overrides }}"
    config_type: "ini"
    init_config_overrides: "{{ neutron_dhcp_agent_init_overrides }}"
+   start_order: 3
  neutron-openvswitch-agent:
    group: neutron_openvswitch_agent
    service_name: neutron-openvswitch-agent
@@ -339,6 +348,7 @@ neutron_services:
    config_overrides: "{{ neutron_openvswitch_agent_ini_overrides }}"
    config_type: "ini"
    init_config_overrides: "{{ neutron_openvswitch_agent_init_overrides }}"
+   start_order: 2
  neutron-linuxbridge-agent:
    group: neutron_linuxbridge_agent
    service_name: neutron-linuxbridge-agent
@@ -350,6 +360,7 @@ neutron_services:
    config_overrides: "{{ neutron_linuxbridge_agent_ini_overrides }}"
    config_type: "ini"
    init_config_overrides: "{{ neutron_linuxbridge_agent_init_overrides }}"
+   start_order: 2
  neutron-metadata-agent:
    group: neutron_metadata_agent
    service_name: neutron-metadata-agent
@@ -360,6 +371,7 @@ neutron_services:
    config_overrides: "{{ neutron_metadata_agent_ini_overrides }}"
    config_type: "ini"
    init_config_overrides: "{{ neutron_metadata_agent_init_overrides }}"
+   start_order: 3
  neutron-metering-agent:
    group: neutron_metering_agent
    service_name: neutron-metering-agent
@@ -378,6 +390,7 @@ neutron_services:
    # TODO(hughsaunders): switch back to stevedore when
    # https://review.openstack.org/#/c/419881/ merges and is backported.
    interface_driver: neutron.agent.linux.interface.BridgeInterfaceDriver
+   start_order: 3
  neutron-l3-agent:
    group: neutron_l3_agent
    service_name: neutron-l3-agent
@@ -389,6 +402,7 @@ neutron_services:
    config_overrides: "{{ neutron_l3_agent_ini_overrides }}"
    config_type: "ini"
    init_config_overrides: "{{ neutron_l3_agent_init_overrides }}"
+   start_order: 3
  neutron-lbaasv2-agent:
    group: neutron_lbaas_agent
    service_name: neutron-lbaasv2-agent
@@ -400,6 +414,7 @@ neutron_services:
    config_overrides: "{{ neutron_lbaas_agent_ini_overrides }}"
    config_type: "ini"
    init_config_overrides: "{{ neutron_lbaas_agent_init_overrides }}"
+   start_order: 3
  neutron-bgp-dragent:
    group: neutron_bgp_dragent
    service_name: neutron-bgp-dragent
@@ -410,6 +425,7 @@ neutron_services:
    config_overrides: "{{ neutron_bgp_dragent_ini_overrides }}"
    config_type: "ini"
    init_config_overrides: "{{ neutron_bgp_dragent_init_overrides }}"
+   start_order: 3
  neutron-vpnaas-agent:
    group: neutron_l3_agent
    service_name: neutron-vpn-agent
@@ -421,12 +437,14 @@ neutron_services:
    config_overrides: "{{ neutron_vpnaas_agent_ini_overrides }}"
    config_type: "ini"
    init_config_overrides: "{{ neutron_vpn_agent_init_overrides }}"
+   start_order: 3
  neutron-server:
    group: neutron_server
    service_name: neutron-server
    service_en: True
    config_options: "--config-file {{ neutron_conf_dir }}/neutron.conf --config-file {{ neutron_conf_dir }}/{{ neutron_plugins[neutron_plugin_type].plugin_ini }} --log-file=/var/log/neutron/neutron-server.log {% if neutron_plugin_type == 'ml2.dragonflow' %}--config-file {{ neutron_conf_dir }}/dragonflow.ini{% endif %}"
    init_config_overrides: "{{ neutron_server_init_overrides }}"
+   start_order: 1
  calico-felix:
    group: neutron_calico_dhcp_agent
    system_user: root
@@ -439,6 +457,7 @@ neutron_services:
    config_overrides: "{{ neutron_calico_felix_ini_overrides }}"
    config_type: "ini"
    init_config_overrides: "{{ neutron_calico_felix_init_overrides }}"
+   start_order: 3
  calico-dhcp-agent:
    group: neutron_calico_dhcp_agent
    system_user: root
@@ -450,6 +469,7 @@ neutron_services:
    config_overrides: "{{ neutron_calico_dhcp_agent_ini_overrides }}"
    config_type: "ini"
    init_config_overrides: "{{ neutron_calico_dhcp_agent_init_overrides }}"
+   start_order: 3
  neutron-sriov-nic-agent:
    group: neutron_sriov_nic_agent
    service_name: neutron-sriov-nic-agent
@@ -460,6 +480,7 @@ neutron_services:
    config_overrides: "{{ neutron_sriov_nic_agent_ini_overrides }}"
    config_type: "ini"
    init_config_overrides: "{{ neutron_sriov_nic_agent_init_overrides }}"
+   start_order: 3
  dragonflow-controller-agent:
    group: dragonflow_controller_agent
    system_user: root
@@ -470,6 +491,7 @@ neutron_services:
    config_overrides: "{{ neutron_dragonflow_ini_overrides }}"
    config_type: "ini"
    init_config_overrides: "{{ neutron_dragonflow_controller_agent_init_overrides }}"
+   start_order: 3
  dragonflow-l3-agent:
    group: dragonflow_l3_agent
    system_user: root
@@ -483,6 +505,7 @@ neutron_services:
    config_overrides: "{{ neutron_dragonflow_ini_overrides }}"
    config_type: "ini"
    init_config_overrides: "{{ neutron_dragonflow_l3_agent_init_overrides }}"
+   start_order: 3
  dragonflow-pubsub-agent:
    group: dragonflow_pubsub_agent
    service_name: df-publisher-service
@@ -491,6 +514,7 @@ neutron_services:
    config_overrides: "{{ neutron_dragonflow_ini_overrides }}"
    config_type: "ini"
    init_config_overrides: "{{ neutron_dragonflow_pubsub_agent_init_overrides }}"
+   start_order: 3
 
 ###
 ### Internals: Drivers mappings