From c38dd767118164aae37613d557ed23691813f617 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Tue, 2 Jul 2019 08:30:02 +0100 Subject: [PATCH] Wait for all compute services before cell discovery There is a race condition during nova deploy since we wait for at least one compute service to register itself before performing cells v2 host discovery. It's quite possible that other compute nodes will not yet have registered and will therefore not be discovered. This leaves them not mapped into a cell, and results in the following error if the scheduler picks one when booting an instance: Host 'xyz' is not mapped to any cell The problem has been exacerbated by merging a fix [1][2] for a nova race condition, which disabled the dynamic periodic discovery mechanism in the nova scheduler. This change fixes the issue by waiting for all expected compute services to register themselves before performing host discovery. This includes both virtualised compute services and bare metal compute services. [1] https://bugs.launchpad.net/kolla-ansible/+bug/1832987 [2] https://review.opendev.org/665554 Change-Id: I2915e2610e5c0b8d67412e7ec77f7575b8fe9921 Closes-Bug: #1835002 --- .../roles/nova/tasks/discover_computes.yml | 45 +++++++++++++++++-- 1 file changed, 42 insertions(+), 3 deletions(-) diff --git a/ansible/roles/nova/tasks/discover_computes.yml b/ansible/roles/nova/tasks/discover_computes.yml index 647bd5d59c..37d4ed0c97 100644 --- a/ansible/roles/nova/tasks/discover_computes.yml +++ b/ansible/roles/nova/tasks/discover_computes.yml @@ -1,5 +1,44 @@ --- -- name: Waiting for nova-compute service up +# We need to wait for all expected compute services to register before running +# cells v2 host discovery. This includes virtualised compute services and +# ironic compute services. +# Work with --limit by including only hosts in ansible_play_batch. +- name: Build a list of expected compute service hosts + vars: + # For virt, use ansible_nodename rather than inventory_hostname, since this + # is similar to what nova uses internally as its default for the + # [DEFAULT] host config option. + virt_compute_service_hosts: >- + {{ groups['compute'] | + intersect(ansible_play_batch) | + map('extract', hostvars, 'ansible_nodename') | + list }} + # For ironic, use {{ansible_hostname}}-ironic since this is what we + # configure for [DEFAULT] host in nova.conf. + ironic_compute_service_hosts: >- + {{ (groups['nova-compute-ironic'] | + intersect(ansible_play_batch) | + map('extract', hostvars, 'ansible_hostname') | + map('regex_replace', '^(.*)$', '\1-ironic') | + list) + if enable_ironic | bool else [] }} + set_fact: + expected_compute_service_hosts: "{{ virt_compute_service_hosts + ironic_compute_service_hosts }}" + run_once: True + delegate_to: "{{ groups['nova-api'][0] }}" + +- name: Waiting for nova-compute services to register themselves + vars: + # A list containing the 'Host' field of compute services that have + # registered themselves. Don't exclude compute services that are disabled + # since these could have been explicitly disabled by the operator. While we + # could exclude services that are down, the nova-manage cell_v2 + # discover_hosts does not do this so let's not block on it here. + found_compute_service_hosts: >- + {{ nova_compute_services.stdout | + from_json | + map(attribute='Host') | + list }} become: true command: > docker exec kolla_toolbox openstack @@ -12,7 +51,7 @@ --os-password {{ keystone_admin_password }} --os-user-domain-name {{ openstack_auth.domain_name }} --os-region-name {{ openstack_region_name }} - compute service list -f json --service nova-compute + compute service list --format json --column Host --service nova-compute register: nova_compute_services changed_when: false run_once: True @@ -21,7 +60,7 @@ delay: 10 until: - nova_compute_services is success - - nova_compute_services.stdout | from_json | length != 0 + - found_compute_service_hosts is superset(expected_compute_service_hosts) # TODO(yoctozepto): no need to do --by-service if ironic not used - name: Discover nova hosts