Implement data migrations for rolling upgrades

In order to cater for artifact-based installed, and
rolling upgrades, this patch implements a set of local
facts to inform the online migrations task.

The 'nova_all_software_updated' variable will be
set by the playbook on each run to ensure that the
online migrations only happen once all venvs are
homogenous. This ensures that the playbook can be
executed in a serialised fashion and the data will
not be corrupted.

The ``upgrade_levels`` setting for ``compute`` is set
to ``auto`` to ensure that a mixed RPC version
deployment can operate properly when doing a rolling
upgrade as suggested by [1].

Additional changes are made to improve the role's
ability to be executed using serialised playbooks.

Finally, the nova-manage command references to the
config file location have been removed as they refer
to the default location.

[1] https://docs.openstack.org/developer/nova/upgrade.html
Change-Id: I08e5a7f0ce526b11aa52c35ee29c458954a5f22d

handlers/main.yml

@@ -86,4 +86,4 @@
   retries: 5
   delay: 2
   when:
-    - inventory_hostname in groups['nova_api_placement']
+    - "'nova_api_placement' in group_names"

releasenotes/notes/nova-data-migrations-f6c2bc0f9e7b1908.yaml

@@ -0,0 +1,9 @@
+---
+features:
+  - The ``os_nova`` role now provides for doing
+    online data migrations once the db sync has
+    been completed. The data migrations will not
+    be executed until the boolean variable
+    ``nova_all_software_updated`` is true. This
+    variable will need to be set by the playbook
+    consuming the role.

tasks/main.yml

@@ -49,13 +49,22 @@
   tags:
     - nova-install
 
+- name: refresh local facts
+  setup:
+    filter: ansible_local
+    gather_subset: "!all"
+  tags:
+    - nova-config
+
 - include: nova_post_install.yml
   tags:
     - nova-config
 
 - include: nova_db_setup.yml
+  static: no
   when:
-    - inventory_hostname == groups['nova_api_os_compute'][0]
+    - "'nova_conductor' in group_names"
+    - "inventory_hostname == ansible_play_hosts[0]"
   tags:
     - nova-config
 
@@ -64,29 +73,44 @@
     - nova-config
 
 - include: nova_service_setup.yml
+  static: no
   when:
-    - inventory_hostname == groups['nova_api_os_compute'][0]
+    - "'nova_conductor' in group_names"
+    - "inventory_hostname == ansible_play_hosts[0]"
   tags:
     - nova-config
 
 - include: nova_placement.yml
+  static: no
   when:
-    - nova_placement_service_enabled | bool
-    - inventory_hostname in groups['nova_api_placement']
+    - "nova_placement_service_enabled | bool"
+    - "'nova_api_placement' in group_names"
   tags:
     - nova-placement
 
 - include: nova_compute.yml
+  static: no
   when:
-    - inventory_hostname in groups['nova_compute']
+    - "'nova_compute' in group_names"
   tags:
     - nova-compute
 
 - name: Flush handlers
   meta: flush_handlers
 
+# We have to delegate this back to the conductor
+# because the compute hosts do not have access to
+# the database connection string and therefore
+# cannot run nova-manage.
+# Also, we delegate this to a random host in the
+# conductor group in order to spread the load of
+# multiple forks across the group members.
 - include: nova_db_post_setup.yml
+  delegate_to: "{{ conductor_host }}"
+  with_random_choice: "{{ groups['nova_conductor'] }}"
+  loop_control:
+    loop_var: conductor_host
   when:
-    - inventory_hostname == groups['nova_api_os_compute'][0]
+    - "'nova_compute' in group_names"
   tags:
     - nova-config

tasks/nova_db_post_setup.yml

@@ -19,40 +19,18 @@
   become: yes
   become_user: "{{ nova_system_user_name }}"
   changed_when: false
-  tags:
-    - nova-db-setup
-    - nova-setup
-    - nova-command-bin
 
 # When upgrading we need to map existing instances to the new cell1
 # To do this we need the cell UUID.
 - name: Get UUID of new Nova Cell
-  shell: "{{ nova_bin }}/nova-manage --config-file /etc/nova/nova.conf cell_v2 list_cells | awk '/{{ nova_cell1_name }}/ {print $4}'"
+  shell: "{{ nova_bin }}/nova-manage cell_v2 list_cells | grep ' {{ nova_cell1_name }} '"
   become: yes
   become_user: "{{ nova_system_user_name }}"
-  when: "nova_cell1_create.rc == 0"
   register: cell1_uuid
-  tags:
-    - nova-db-setup
-    - nova-setup
-    - nova-command-bin
+  changed_when: false
 
 - name: Map instances to new Cell1
-  command: "{{ nova_bin }}/nova-manage --config-file /etc/nova/nova.conf cell_v2 map_instances --cell_uuid {{ cell1_uuid.stdout }}"
-  become: yes
-  become_user: "{{ nova_system_user_name }}"
-  when: "nova_cell1_create.rc == 0"
-  tags:
-    - nova-db-setup
-    - nova-setup
-    - nova-command-bin
-
-- name: Perform Nova online data migrations
-  command: "{{ nova_bin }}/nova-manage --config-file /etc/nova/nova.conf db online_data_migrations"
+  command: "{{ nova_bin }}/nova-manage cell_v2 map_instances --cell_uuid {{ cell1_uuid['stdout'].split()[3] }}"
   become: yes
   become_user: "{{ nova_system_user_name }}"
   changed_when: false
-  tags:
-    - nova-db-setup
-    - nova-setup
-    - nova-command-bin

tasks/nova_db_setup.yml

@@ -19,32 +19,20 @@
   become_user: "{{ nova_system_user_name }}"
   register: nova_api_db_version
   changed_when: false
-  tags:
-    - nova-db-setup
-    - nova-setup
-    - nova-command-bin
 
 - name: Perform a Nova API DB sync
-  command: "{{ nova_bin }}/nova-manage --config-file /etc/nova/nova.conf api_db sync"
+  command: "{{ nova_bin }}/nova-manage api_db sync"
   become: yes
   become_user: "{{ nova_system_user_name }}"
   changed_when: false
   when:
     - nova_api_db_version.stdout == "0"
-  tags:
-    - nova-db-setup
-    - nova-setup
-    - nova-command-bin
 
 - name: Perform cell_v2 map cell0
   command: "{{ nova_bin }}/nova-manage cell_v2 map_cell0 --database_connection mysql+pymysql://{{ nova_api_galera_user }}:{{ nova_api_container_mysql_password }}@{{ nova_api_galera_address }}/{{ nova_cell0_database }}?charset=utf8"
   become: yes
   become_user: "{{ nova_system_user_name }}"
   changed_when: false
-  tags:
-    - nova-db-setup
-    - nova-setup
-    - nova-command-bin
 
 - name: Perform cell_v2 initial cell setup
   command: "{{ nova_bin }}/nova-manage cell_v2 create_cell --name {{ nova_cell1_name }} --database_connection mysql+pymysql://{{ nova_galera_user }}:{{ nova_container_mysql_password }}@{{ nova_galera_address }}/{{ nova_galera_database }}?charset=utf8 --transport-url rabbit://{% for host in nova_rabbitmq_servers.split(',') %}{{ nova_rabbitmq_userid }}:{{ nova_rabbitmq_password }}@{{ host }}:{{ nova_rabbitmq_port }}{% if not loop.last %},{% else %}/{{ nova_rabbitmq_vhost }}{% endif %}{% endfor %}"
@@ -53,27 +41,34 @@
   register: nova_cell1_create
   failed_when: "nova_cell1_create.rc not in [0, 2]"
   changed_when: "nova_cell1_create.rc == 0"
-  tags:
-    - nova-db-setup
-    - nova-setup
-    - nova-command-bin
 
 - name: Perform a Nova API DB sync
-  command: "{{ nova_bin }}/nova-manage --config-file /etc/nova/nova.conf api_db sync"
+  command: "{{ nova_bin }}/nova-manage api_db sync"
   become: yes
   become_user: "{{ nova_system_user_name }}"
   changed_when: false
-  tags:
-    - nova-db-setup
-    - nova-setup
-    - nova-command-bin
 
 - name: Perform a Nova DB sync
-  command: "{{ nova_bin }}/nova-manage --config-file /etc/nova/nova.conf db sync"
+  command: "{{ nova_bin }}/nova-manage db sync"
   become: yes
   become_user: "{{ nova_system_user_name }}"
   changed_when: false
-  tags:
-    - nova-db-setup
-    - nova-setup
-    - nova-command-bin
+
+- name: Perform online data migrations
+  command: "{{ nova_bin }}/nova-manage db online_data_migrations"
+  become: yes
+  become_user: "{{ nova_system_user_name }}"
+  when:
+    - "(nova_all_software_updated | default('no')) | bool"
+    - "ansible_local['openstack_ansible']['nova']['need_online_data_migrations'] | bool"
+  changed_when: false
+  register: data_migrations
+
+- name: Disable the online migrations requirement
+  ini_file:
+    dest: "/etc/ansible/facts.d/openstack_ansible.fact"
+    section: nova
+    option: need_online_data_migrations
+    value: False
+  when:
+    - data_migrations | succeeded

tasks/nova_install.yml

@@ -165,6 +165,25 @@
     - nova-novnc-console
     - nova-spice-console
 
+- name: Initialise the upgrade facts
+  ini_file:
+    dest: "/etc/ansible/facts.d/openstack_ansible.fact"
+    section: nova
+    option: "{{ item }}"
+    value: True
+  with_items:
+    - "need_service_restart"
+    - "need_online_data_migrations"
+  when:
+    - (nova_get_venv | changed) or
+      (nova_venv_dir | changed) or
+      (install_packages | changed) or
+      (ansible_local is not defined) or
+      ('openstack_ansible' not in ansible_local) or
+      ('nova' not in ansible_local['openstack_ansible']) or
+      ('need_online_data_migrations' not in ansible_local['openstack_ansible']['nova']) or
+      ('need_service_restart' not in ansible_local['openstack_ansible']['nova'])
+
 - name: Record the venv tag deployed
   ini_file:
     dest: "/etc/ansible/facts.d/openstack_ansible.fact"

templates/nova.conf.j2

@@ -324,3 +324,6 @@ metadata_items = {{ nova_quota_metadata_items }}
 ram = {{ nova_quota_ram }}
 server_group_members = {{ nova_quota_server_group_members }}
 server_groups = {{ nova_quota_server_groups }}
+
+[upgrade_levels]
+compute=auto