Simplify database migrations

Instead of using a module to check for migrations,
we now execute the expand every time the venv is
updated. We also implement a simple check for
offline migrations and do the proper server shut
down and execution as per [1].

The integrated playbook will target all
neutron-server hosts at once when executing the
role to make sure that the db contract is done
with them all offline.

[1] https://docs.openstack.org/developer/neutron/devref/upgrade.html

Change-Id: I7dfef45e1ed8a8dfa464f1b3d20c90ae3348ce2e

library/neutron_migrations_facts

@@ -1,289 +0,0 @@
-#!/usr/bin/env python
-# Copyright 2015, Rackspace US, Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import os
-import re
-import subprocess
-from ansible.module_utils.basic import *
-
-
-DOCUMENTATION = """
----
-module: neutron_migrations_facts
-short_description:
-    - A module for gathering neutron migrations facts.
-description:
-    - This module creates a fact called 'neutron_migrations', which is a dict
-      containing keys that represent each alembic migration branch.  The value
-      for each key is another dict, containing a key value for the current
-      neutron revision for that branch (revision), and whether or not the
-      branch is currently at the latest revision (head).  The
-      'neutron_migrations' fact can then be used to determine if migrations
-      for either branch are required, and allows us to only apply migrations
-      if necessary.
-options:
-    release:
-        description:
-            - This is the OpenStack release you're running, used when
-              searching for migration revisions in the neutron code.
-        default: liberty
-    library_path:
-        description:
-            - Local path to the location where the neutron python package
-              is installed.
-        default: /usr/local/lib/python2.7/dist-packages/neutron
-    bin_path:
-        description:
-            - Local path to the where the neutron binaries are.
-        default: /usr/local/bin
-author: Rcbops
-"""
-
-
-EXAMPLES = """
-- name: Gather neutron migration facts
-  neutron_migrations_facts:
-    release: ocata
-"""
-
-
-MIGRATIONS = {
-    'projects': {
-        'neutron': {
-            'branches': {
-                'expand': {
-                    'revision': None,
-                    'head': None
-                },
-                'contract': {
-                    'revision': None,
-                    'head': None
-                }
-            },
-            'installed': False
-        },
-        'neutron-fwaas': {
-            'branches': {
-                'expand': {
-                    'revision': None,
-                    'head': None
-                },
-                'contract': {
-                    'revision': None,
-                    'head': None
-                }
-            },
-            'installed': False
-        },
-        'neutron-lbaas': {
-            'branches': {
-                'expand': {
-                    'revision': None,
-                    'head': None
-                },
-                'contract': {
-                    'revision': None,
-                    'head': None
-                }
-            },
-            'installed': False
-        },
-        'neutron-vpnaas': {
-            'branches': {
-                'expand': {
-                    'revision': None,
-                    'head': None
-                },
-                'contract': {
-                    'revision': None,
-                    'head': None
-                }
-            },
-            'installed': False
-        },
-        'neutron-dynamic-routing': {
-            'branches': {
-                'expand': {
-                    'revision': None,
-                    'head': None
-                },
-                'contract': {
-                    'revision': None,
-                    'head': None
-                }
-            },
-            'installed': False
-        }
-    },
-    'run_contract': True,
-    'run_expand': True
-}
-
-RELEASES = ['pike', 'ocata', 'newton', 'mitaka', 'liberty']
-
-
-def get_branch(release, revision, library_path, project):
-    # Here we drop any releases that are newer than the specified release.
-    # We have to check previous releases for migrations as at time of writing,
-    # neutron-lbaas doesn't have mitaka migrations and therefore the plugin
-    # will fail unless we also check the previous release's migrations.
-    for release in RELEASES[RELEASES.index(release):]:
-        migrations_dir = (
-            '%s/%s/db/migration/alembic_migrations/versions/%s/' % (
-                library_path,
-                project.replace('-', '_'),
-                release,
-            )
-        )
-        if os.path.isdir(migrations_dir):
-            for branch in MIGRATIONS['projects'][project]['branches'].keys():
-                migration_dir = os.path.join(
-                    get_abs_path(migrations_dir), branch
-                )
-                # If a release has no migrations for a given branch, the branch
-                # directory will not exist.
-                if os.path.isdir(migration_dir):
-                    for file in os.listdir(migration_dir):
-                        if (file.endswith('.py') and
-                                file.split('_')[0] == revision):
-                            return branch
-
-
-def get_abs_path(path):
-    return os.path.abspath(
-        os.path.expanduser(
-            path
-        )
-    )
-
-
-def update_run_keys():
-    projects = MIGRATIONS['projects']
-
-    # If a probject or subproject is marked as installed but has no revision,
-    # we can assume the project has just been installed.  In this case we
-    # just return to main, leaving run_contract/run_expand as True so that
-    # migrations will run.
-    # TODO(mattt): We will want to try to figure out if migrations in this
-    #              situation are expand, contract, or both, so that we avoid
-    #              unnecessary neutron-server downtime.
-    for p in projects:
-        if (projects[p]['installed'] is True and
-                projects[p]['branches']['expand']['revision'] is None and
-                projects[p]['branches']['contract']['revision'] is None):
-            return
-
-    # Here we will determine if migrations should be skipped -- this will
-    # happen when all the projects or subprojects that have a revision
-    # are also at head.
-    for b in ('contract', 'expand'):
-        migrations_with_a_rev = [
-            v
-            for v in projects.values()
-            if v['branches'][b]['revision'] is not None
-        ]
-        if (len(migrations_with_a_rev) > 0 and
-                all(m['branches'][b]['head'] is True
-                    for m in migrations_with_a_rev)):
-            MIGRATIONS['run_%s' % b] = False
-
-
-def main():
-    module = AnsibleModule(
-        argument_spec=dict(
-            release=dict(
-                type='str',
-                default='liberty'
-            ),
-            library_path=dict(
-                type='str',
-                default='/usr/local/lib/python2.7/dist-packages/neutron'
-            ),
-            bin_path=dict(
-                type='str',
-                default='/usr/local/bin'
-            )
-        ),
-        supports_check_mode=False
-    )
-
-    if module.params['release'] not in RELEASES:
-        module.fail_json(
-            msg='neutron fact collection failed: release %s not found in'
-                ' %s' % (module.params['release'], RELEASES)
-        )
-
-    state_change = False
-    project = None
-
-    command = [
-        '%s/neutron-db-manage' % get_abs_path(module.params['bin_path']),
-        'current'
-    ]
-
-    try:
-        current = subprocess.check_output(command)
-    except subprocess.CalledProcessError as e:
-        module.fail_json(msg='neutron fact collection failed: "%s".' % e)
-
-    for line in current.splitlines():
-        head = False
-        project_match = re.search(
-            "^\s*Running current for (neutron(-[a-z-]+)?) ...$",
-            line
-        )
-        migration_match = re.search("^([0-9a-z]{4,12})(\s\(head\))?$", line)
-
-        if project_match:
-            project = project_match.group(1)
-            MIGRATIONS['projects'][project]['installed'] = True
-
-        if migration_match:
-            revision = migration_match.group(1)
-
-            if project is None:
-                module.fail_json(
-                    msg='neutron fact collection failed: unable to detect'
-                        ' project for revision %s' % revision
-                )
-
-            if migration_match.group(2):
-                head = True
-
-            branch = get_branch(
-                release=module.params['release'],
-                revision=revision,
-                library_path=get_abs_path(module.params['library_path']),
-                project=project
-            )
-            if branch is None:
-                module.fail_json(
-                    msg='neutron fact collection failed: unable to find'
-                        ' migration with revision %s' % revision
-                )
-
-            proj_migrations = MIGRATIONS['projects'][project]['branches']
-            proj_migrations[branch]['revision'] = revision
-            proj_migrations[branch]['head'] = head
-
-    update_run_keys()
-
-    module.exit_json(
-        changed=state_change,
-        ansible_facts={'neutron_migrations': MIGRATIONS}
-    )
-
-if __name__ == '__main__':
-    main()

tasks/main.yml

@@ -59,6 +59,13 @@
   tags:
     - neutron-install
 
+- name: refresh local facts
+  setup:
+    filter: ansible_local
+    gather_subset: "!all"
+  tags:
+    - neutron-config
+
 - include: neutron_post_install.yml
   tags:
     - neutron-config
@@ -80,21 +87,19 @@
 
 - include: neutron_db_setup.yml
   when:
-    - neutron_services['neutron-server']['group'] in group_names
+    - "neutron_services['neutron-server']['group'] in group_names"
-    - inventory_hostname == groups[neutron_services['neutron-server']['group']][0]
   tags:
     - neutron-config
 
 - include: neutron_service_setup.yml
   when:
-    - "'neutron_all' in group_names"
+    - "neutron_services['neutron-server']['group'] in group_names"
-    - inventory_hostname == groups['neutron_all'][0]
   tags:
     - neutron-config
 
 - include: neutron_l3_ha.yml
   when:
-    - neutron_services['neutron-l3-agent']['group'] in group_names
+    - "neutron_services['neutron-l3-agent']['group'] in group_names"
   tags:
     - neutron-config
 

tasks/neutron_db_setup.yml

@@ -13,68 +13,65 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-- name: Get neutron migrations facts
+- name: Perform a DB expand
-  neutron_migrations_facts:
+  command: "{{ neutron_bin }}/neutron-db-manage upgrade --expand"
-    release: pike
-    library_path: "{{ neutron_lib_dir }}"
-    bin_path: "{{ neutron_bin }}"
-  when: neutron_plugin_type.split('.')[0] == 'ml2'
-
-- name: Print neutron migrations facts
-  debug:
-    var: neutron_migrations
-  when: neutron_plugin_type.split('.')[0] == 'ml2'
-
-- name: Perform a Neutron DB online upgrade (expand)
-  command: |
-    {{ neutron_bin }}/neutron-db-manage
-                      --config-file {{ neutron_db_config }}
-                      --config-file {{ neutron_db_plugin }}
-                      upgrade --expand
   become: yes
   become_user: "{{ neutron_system_user_name }}"
   when:
-    - (neutron_migrations is defined and neutron_migrations['run_expand']|bool)
+    - "ansible_local['openstack_ansible']['neutron']['need_db_expand'] | bool"
-    - neutron_plugin_type.split('.')[0] == 'ml2'
+  run_once: yes
+
+- name: Check for available offline migrations
+  command: "{{ neutron_bin }}/neutron-db-manage has_offline_migrations"
+  register: _offline_migrations_check
+  changed_when: false
+  failed_when:
+    - "_offline_migrations_check.rc == 1"
+    - "'Need to apply migrations from neutron contract branch' not in _offline_migrations_check.stdout"
+  run_once: yes
+
+- name: Set the fact for the required offline migrations
+  ini_file:
+    dest: "/etc/ansible/facts.d/openstack_ansible.fact"
+    section: neutron
+    option: "need_db_contract"
+    value: "{{ ('Need to apply migrations from neutron contract branch' in _offline_migrations_check.stdout) | bool }}"
+
+- name: Refresh local facts
+  setup:
+    filter: ansible_local
+    gather_subset: "!all"
 
 - name: Stop Neutron server
   service:
     name: "neutron-server"
     state: stopped
-  delegate_to: "{{ item }}"
+  when:
-  with_items: "{{ groups[neutron_services['neutron-server']['group']] }}"
+    - "ansible_local['openstack_ansible']['neutron']['need_db_contract'] | bool"
-  when: (neutron_migrations is defined and neutron_migrations['run_contract']|bool) or neutron_plugin_type.split('.')[0] != 'ml2'
 
-- name: Perform a Neutron DB offline upgrade (contract)
+- name: Perform a DB contract
-  command: |
+  command: "{{ neutron_bin }}/neutron-db-manage upgrade --contract"
-    {{ neutron_bin }}/neutron-db-manage
-                      --config-file {{ neutron_db_config }}
-                      --config-file {{ neutron_db_plugin }}
-                      upgrade --contract
   become: yes
   become_user: "{{ neutron_system_user_name }}"
   when:
-    - (neutron_migrations is defined and neutron_migrations['run_contract']|bool)
+    - "ansible_local['openstack_ansible']['neutron']['need_db_contract'] | bool"
-    - neutron_plugin_type.split('.')[0] == 'ml2'
+  run_once: yes
 
-# NOTE: We have to handle neutron_plugin_type.split('.')[0] != 'ml2' because not all neutron
+- name: Start Neutron server
-#       plugins have contract/expand branches which breaks neutron-db-manage.
-#       This can be reverted once all plugins are conformant.
-- name: Perform a Neutron DB offline upgrade (heads)
-  command: |
-    {{ neutron_bin }}/neutron-db-manage
-                      --config-file {{ neutron_db_config }}
-                      --config-file {{ neutron_db_plugin }}
-                      upgrade heads
-  become: yes
-  become_user: "{{ neutron_system_user_name }}"
-  when:
-    - neutron_plugin_type.split('.')[0] != 'ml2'
-
-- name: Start neutron server
   service:
     name: "neutron-server"
     state: started
-  delegate_to: "{{ item }}"
+  when:
-  with_items: "{{ groups[neutron_services['neutron-server']['group']] }}"
+    - "ansible_local['openstack_ansible']['neutron']['need_db_contract'] | bool"
-  when: (neutron_migrations is defined and neutron_migrations['run_contract']|bool) or neutron_plugin_type.split('.')[0] != 'ml2'
+
+- name: Disable the db sync local facts
+  ini_file:
+    dest: "/etc/ansible/facts.d/openstack_ansible.fact"
+    section: neutron
+    option: "{{ item.name }}"
+    value: "{{ item.state }}"
+  with_items:
+    - name: "need_db_expand"
+      state: "False"
+    - name: "need_db_contract"
+      state: "False"

tasks/neutron_install.yml

@@ -77,6 +77,7 @@
   file:
     path: "{{ neutron_bin | dirname }}"
     state: directory
+  register: neutron_venv_dir
   when: neutron_get_venv | changed
 
 - name: Unarchive pre-built venv
@@ -98,7 +99,7 @@
       {{ (pip_install_upper_constraints is defined) | ternary('--constraint ' + pip_install_upper_constraints | default(''),'') }}
       {{ pip_install_options | default('') }}
   register: install_packages
-  until: install_packages|success
+  until: install_packages | success
   retries: 5
   delay: 2
   when: neutron_get_venv | failed or neutron_get_venv | skipped
@@ -135,12 +136,28 @@
         neutron_fwaas | bool or
         neutron_lbaasv2 | bool or
         neutron_vpnaas | bool
-  register: install_packages
+  register: install_optional_packages
-  until: install_packages|success
+  until: install_optional_packages | success
   retries: 5
   delay: 2
   notify: Restart neutron services
 
+- name: Initialise the db sync local facts
+  ini_file:
+    dest: "/etc/ansible/facts.d/openstack_ansible.fact"
+    section: neutron
+    option: "{{ item.name }}"
+    value: "{{ item.state }}"
+  with_items:
+    - name: "need_db_expand"
+      state: "True"
+    - name: "need_db_contract"
+      state: "True"
+  when: neutron_get_venv | changed or
+        neutron_venv_dir | changed or
+        install_packages | changed or
+        install_optional_packages | changed
+
 - name: Record the venv tag deployed
   ini_file:
     dest: "/etc/ansible/facts.d/openstack_ansible.fact"

tasks/neutron_service_setup.yml

@@ -29,6 +29,7 @@
   until: add_service|success
   retries: 5
   delay: 2
+  run_once: yes
 
 # Create an admin user
 - name: Ensure neutron user
@@ -47,6 +48,7 @@
   until: add_service|success
   retries: 5
   delay: 10
+  run_once: yes
 
 # Add a role to the user
 - name: Ensure neutron user to admin role
@@ -65,6 +67,7 @@
   until: add_service|success
   retries: 5
   delay: 10
+  run_once: yes
 
 # Create an endpoint
 - name: Ensure neutron endpoint
@@ -89,3 +92,4 @@
   until: add_service|success
   retries: 5
   delay: 10
+  run_once: yes

tests/test-neutron-upgrades.sh

@@ -22,10 +22,6 @@
 # PURPOSE:
 # This script executes test Ansible playbooks required for performing
 # an upgrade test of the os_neutron role.
-# Due to the way Ansible caches and handles modules, we need to run
-# separate Ansible runs to ensure the "upgrade" uses the new
-# "neutron_migrations_facts" module, instead of the cached version
-# used when deploying the previous Neutron version.
 
 ## Shell Opts ----------------------------------------------------------------