MNAIO: Ensure VM's are shut down before doing image save

When the virt module returns success for the shutdown state, it
has only sucessfully sent the signal to shut down. It may still
take a few more minutes for the VM's to actually complete their
shut down. If we try to change the image while the machine is
still busy shutting down, the image conversion/compression fails
and the resulting state is incomplete.

In this patch we do the following:

1. Find and shut down the running VM's without needing to look
   at the inventory. This reduces complexity in the play.
2. Makes sure that the VM's are all in the 'shut off' state,
   before continuing on to saving the images.

Change-Id: Icf337447f7a9b4033af261910f77216a170937ed

multi-node-aio/playbooks/save-vms.yml

@@ -20,61 +20,57 @@
   tags:
     - save-vms
   tasks:
-    - name: Gather variables for each operating system
+    - name: Get info about existing virt storage pools
-      include_vars: "{{ item }}"
-      with_first_found:
-        - "{{ playbook_dir }}/vars/{{ ansible_distribution | lower }}-{{ ansible_distribution_version | lower }}.yml"
-        - "{{ playbook_dir }}/vars/{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version | lower }}.yml"
-        - "{{ playbook_dir }}/vars/{{ ansible_os_family | lower }}-{{ ansible_distribution_major_version | lower }}.yml"
-        - "{{ playbook_dir }}/vars/{{ ansible_distribution | lower }}.yml"
-        - "{{ playbook_dir }}/vars/{{ ansible_os_family | lower }}.yml"
-      tags:
-        - always
-
-    - name: Get info about the virt storage pools
       virt_pool:
         command: info
       register: _virt_pools
-      tags:
-        - always
 
-    - name: Stop running VMs
+    - name: Get info about existing VM's
       virt:
-        name: "{{ hostvars[item]['server_hostname'] }}"
+        command: list_vms
-        state: shutdown
+      register: _virt_list
-      when:
+
-        - hostvars[item]['server_vm'] | default(false) | bool
+    - name: Shut down all running VM's
-      with_items: "{{ groups['pxe_servers'] }}"
+      virt:
+        name: "{{ item }}"
+        command: shutdown
+      failed_when: false
+      with_items: "{{ _virt_list.list_vms }}"
+
+    - name: Wait for shut down to complete
+      command: |
+        virsh domstate {{ item }}
+      register: _vm_shutdown
+      until: _vm_shutdown.stdout.find('shut off') != -1
+      retries: 5
+      delay: 60
+      with_items: "{{ _virt_list.list_vms }}"
 
     - name: Commit, compress and save VM Disk Image and prepare new copy-on-write image
       shell: |
-        if [[ -e {{ hostvars[item]['server_hostname'] }}.img ]]; then
+        if [[ -e {{ item }}.img ]]; then
-          if [[ -e {{ hostvars[item]['server_hostname'] }}-base.img ]]; then
+          if [[ -e {{ item }}-base.img ]]; then
-            qemu-img commit {{ hostvars[item]['server_hostname'] }}.img
+            qemu-img commit {{ item }}.img
           else
-            qemu-img convert -O qcow2 -c {{ hostvars[item]['server_hostname'] }}.img {{ hostvars[item]['server_hostname'] }}-base.img
+            qemu-img convert -O qcow2 -c {{ item }}.img {{ item }}-base.img
-            qemu-img create -f qcow2 -b {{ hostvars[item]['server_hostname'] }}-base.img {{ hostvars[item]['server_hostname'] }}.img
+            qemu-img create -f qcow2 -b {{ item }}-base.img {{ item }}.img
           fi
           exit 2
         fi
       args:
         executable: /bin/bash
         chdir: "{{ _virt_pools.pools.default.path | default('/data/images') }}"
-      when:
+      with_items: "{{ _virt_list.list_vms }}"
-        - hostvars[item]['server_vm'] | default(false) | bool
-      with_items: "{{ groups['pxe_servers'] }}"
       register: _save_disk_image
       changed_when: _save_disk_image.rc == 2
       failed_when: _save_disk_image.rc not in [0, 2]
 
     - name: Save VM definition
       copy:
-        src: "/etc/libvirt/qemu/{{ hostvars[item]['server_hostname'] }}.xml"
+        src: "/etc/libvirt/qemu/{{ item }}.xml"
         dest: "{{ _virt_pools.pools.default.path | default('/data/images') }}/"
         remote_src: yes
-      when:
+      with_items: "{{ _virt_list.list_vms }}"
-        - hostvars[item]['server_vm'] | default(false) | bool
-      with_items: "{{ groups['pxe_servers'] }}"
 
     - name: Get the current SHA1 for the manifest
       command: "git rev-parse HEAD"