Wait for container ssh after apparmor profile update

This patch adds a wait for the container's sshd to be available
after the container's apparmor profile is updated. When the
profile is updated the container is restarted, so this wait is
essential to the success of the playbook's completion.

It also includes 3 retries which has been found to improve the
rate of success.

Due to an upstream change in behaviour with netaddr 0.7.16 we
need to pin the package to a lower version until Neutron is
adjusted and we bump the Neutron SHA.

Change-Id: I30575ee31929b0c9af6353b7255cdfb6cebd2104
Closes-Bug: #1490142

playbooks/memcached-install.yml

@@ -27,6 +27,23 @@
       when: not is_metal | bool
       tags:
         - lxc-aa-profile
+    - name: Flush net cache
+      command: /usr/local/bin/lxc-system-manage flush-net-cache
+      delegate_to: "{{ physical_host }}"
+      tags:
+        - flush-net-cache
+    - name: Wait for container ssh
+      wait_for:
+        port: "22"
+        delay: "{{ ssh_delay }}"
+        search_regex: "OpenSSH"
+        host: "{{ ansible_ssh_host }}"
+      delegate_to: "{{ physical_host }}"
+      register: ssh_wait_check
+      until: ssh_wait_check|success
+      retries: 3
+      tags:
+        - ssh-wait
   roles:
     - { role: "memcached_server", tags: [ "memcached-server" ] }
     - role: "system_crontab_coordination"

playbooks/os-ceilometer-install.yml

@@ -40,7 +40,7 @@
         host: "{{ ansible_ssh_host }}"
       delegate_to: "{{ physical_host }}"
       tags:
-        - ceilometer-ssh-wait
+        - ssh-wait
     - name: Sort the rabbitmq servers
       dist_sort:
         value_to_lookup: "{{ container_name }}"

playbooks/os-cinder-install.yml

@@ -80,8 +80,11 @@
         search_regex: "OpenSSH"
         host: "{{ ansible_ssh_host }}"
       delegate_to: "{{ physical_host }}"
+      register: ssh_wait_check
+      until: ssh_wait_check|success
+      retries: 3
       tags:
-        - rabbit-ssh-wait
+        - ssh-wait
     - name: Sort the rabbitmq servers
       dist_sort:
         value_to_lookup: "{{ container_name }}"

playbooks/os-glance-install.yml

@@ -51,7 +51,7 @@
         host: "{{ ansible_ssh_host }}"
       delegate_to: "{{ physical_host }}"
       tags:
-        - glance-ssh-wait
+        - ssh-wait
     - name: Sort the rabbitmq servers
       dist_sort:
         value_to_lookup: "{{ container_name }}"

playbooks/os-heat-install.yml

@@ -27,6 +27,23 @@
       when: not is_metal | bool
       tags:
         - lxc-aa-profile
+    - name: Flush net cache
+      command: /usr/local/bin/lxc-system-manage flush-net-cache
+      delegate_to: "{{ physical_host }}"
+      tags:
+        - flush-net-cache
+    - name: Wait for container ssh
+      wait_for:
+        port: "22"
+        delay: "{{ ssh_delay }}"
+        search_regex: "OpenSSH"
+        host: "{{ ansible_ssh_host }}"
+      delegate_to: "{{ physical_host }}"
+      register: ssh_wait_check
+      until: ssh_wait_check|success
+      retries: 3
+      tags:
+        - ssh-wait
     - name: Sort the rabbitmq servers
       dist_sort:
         value_to_lookup: "{{ container_name }}"

playbooks/os-horizon-install.yml

@@ -27,6 +27,23 @@
       when: not is_metal | bool
       tags:
         - lxc-aa-profile
+    - name: Flush net cache
+      command: /usr/local/bin/lxc-system-manage flush-net-cache
+      delegate_to: "{{ physical_host }}"
+      tags:
+        - flush-net-cache
+    - name: Wait for container ssh
+      wait_for:
+        port: "22"
+        delay: "{{ ssh_delay }}"
+        search_regex: "OpenSSH"
+        host: "{{ ansible_ssh_host }}"
+      delegate_to: "{{ physical_host }}"
+      register: ssh_wait_check
+      until: ssh_wait_check|success
+      retries: 3
+      tags:
+        - ssh-wait
     - name: Sort the rabbitmq servers
       dist_sort:
         value_to_lookup: "{{ container_name }}"

playbooks/os-keystone-install.yml

@@ -27,6 +27,23 @@
       when: not is_metal | bool
       tags:
         - lxc-aa-profile
+    - name: Flush net cache
+      command: /usr/local/bin/lxc-system-manage flush-net-cache
+      delegate_to: "{{ physical_host }}"
+      tags:
+        - flush-net-cache
+    - name: Wait for container ssh
+      wait_for:
+        port: "22"
+        delay: "{{ ssh_delay }}"
+        search_regex: "OpenSSH"
+        host: "{{ ansible_ssh_host }}"
+      delegate_to: "{{ physical_host }}"
+      register: ssh_wait_check
+      until: ssh_wait_check|success
+      retries: 3
+      tags:
+        - ssh-wait
     - name: Sort the rabbitmq servers
       dist_sort:
         value_to_lookup: "{{ container_name }}"

playbooks/os-neutron-install.yml

@@ -56,7 +56,7 @@
         host: "{{ ansible_ssh_host }}"
       delegate_to: "{{ physical_host }}"
       tags:
-        - neutron-ssh-wait
+        - ssh-wait
     - name: Sort the rabbitmq servers
       dist_sort:
         value_to_lookup: "{{ container_name }}"

playbooks/os-nova-install.yml

@@ -27,6 +27,23 @@
       when: not is_metal | bool
       tags:
         - lxc-aa-profile
+    - name: Flush net cache
+      command: /usr/local/bin/lxc-system-manage flush-net-cache
+      delegate_to: "{{ physical_host }}"
+      tags:
+        - flush-net-cache
+    - name: Wait for container ssh
+      wait_for:
+        port: "22"
+        delay: "{{ ssh_delay }}"
+        search_regex: "OpenSSH"
+        host: "{{ ansible_ssh_host }}"
+      delegate_to: "{{ physical_host }}"
+      register: ssh_wait_check
+      until: ssh_wait_check|success
+      retries: 3
+      tags:
+        - ssh-wait
     - name: Sort the rabbitmq servers
       dist_sort:
         value_to_lookup: "{{ container_name }}"

playbooks/os-swift-setup.yml

@@ -29,6 +29,23 @@
       when: not is_metal | bool
       tags:
         - lxc-aa-profile
+    - name: Flush net cache
+      command: /usr/local/bin/lxc-system-manage flush-net-cache
+      delegate_to: "{{ physical_host }}"
+      tags:
+        - flush-net-cache
+    - name: Wait for container ssh
+      wait_for:
+        port: "22"
+        delay: "{{ ssh_delay }}"
+        search_regex: "OpenSSH"
+        host: "{{ ansible_ssh_host }}"
+      delegate_to: "{{ physical_host }}"
+      register: ssh_wait_check
+      until: ssh_wait_check|success
+      retries: 3
+      tags:
+        - ssh-wait
     - name: Create log dir
       file:
         path: "{{ item.path }}"

playbooks/rabbitmq-install.yml

@@ -27,6 +27,23 @@
       when: not is_metal | bool
       tags:
         - lxc-aa-profile
+    - name: Flush net cache
+      command: /usr/local/bin/lxc-system-manage flush-net-cache
+      delegate_to: "{{ physical_host }}"
+      tags:
+        - flush-net-cache
+    - name: Wait for container ssh
+      wait_for:
+        port: "22"
+        delay: "{{ ssh_delay }}"
+        search_regex: "OpenSSH"
+        host: "{{ ansible_ssh_host }}"
+      delegate_to: "{{ physical_host }}"
+      register: ssh_wait_check
+      until: ssh_wait_check|success
+      retries: 3
+      tags:
+        - ssh-wait
   roles:
     - role: "rabbitmq_server"
       tags:

playbooks/repo-server.yml

@@ -51,7 +51,7 @@
         host: "{{ ansible_ssh_host }}"
       delegate_to: "{{ physical_host }}"
       tags:
-        - repo-ssh-wait
+        - ssh-wait
   roles:
     - { role: "repo_server", tags: [ "repo-server" ] }
     - role: "rsyslog_client"

playbooks/rsyslog-install.yml

@@ -58,8 +58,11 @@
         search_regex: "OpenSSH"
         host: "{{ ansible_ssh_host }}"
       delegate_to: "{{ physical_host }}"
+      register: ssh_wait_check
+      until: ssh_wait_check|success
+      retries: 3
       tags:
-        - rsyslog-ssh-wait
+        - ssh-wait
 
   roles:
     - { role: "rsyslog_server", tags: [ "rsyslog-server" ] }

playbooks/utility-install.yml

@@ -27,6 +27,23 @@
       when: not is_metal | bool
       tags:
         - lxc-aa-profile
+    - name: Flush net cache
+      command: /usr/local/bin/lxc-system-manage flush-net-cache
+      delegate_to: "{{ physical_host }}"
+      tags:
+        - flush-net-cache
+    - name: Wait for container ssh
+      wait_for:
+        port: "22"
+        delay: "{{ ssh_delay }}"
+        search_regex: "OpenSSH"
+        host: "{{ ansible_ssh_host }}"
+      delegate_to: "{{ physical_host }}"
+      register: ssh_wait_check
+      until: ssh_wait_check|success
+      retries: 3
+      tags:
+        - ssh-wait
   roles:
     - { role: "galera_client", tags: [ "utility-galera-client" ] }
     - { role: "pip_lock_down", tags: [ "utility-pip-lock-down" ] }

requirements.txt

@@ -1,6 +1,6 @@
 cloudlib>=0.3.0         # scripts/os-ansible-role-requirements.py
 Jinja2>=2.6             # ansible
-netaddr>=0.7.12         # playbooks/inventory/dynamic_inventory.py
+netaddr>=0.7.12,<0.7.16 # playbooks/inventory/dynamic_inventory.py
 paramiko>=1.13.0        # ansible
 pip>=6.0
 PrettyTable>=0.7,<0.8   # scripts/inventory-manage.py