Update neutron_migrations_facts

This change does the following to neutron_migrations_facts:

- updates MIGRATIONS dict by adding entries for each neutron
  sub-project while also adding keys to track which sub-projects are
  installed and which migrations need to be run (contract or expand)
- takes subproject migrations into account (neutron-lbaas as an
  example)
- updates neutron_venv_lib_dir and neutron_non_venv_lib_dir by removing
  reference to neutron specifically since these vars need to be used
  for all neutron subprojects also

We also update neutron_db_setup.yml to use MIGRATIONS['run_contract']
and MIGRATIONS['run_expand'] to determine if neutron or a subproject
has migrations which need to be applied.

Closes-Bug: #1511756
Change-Id: I3a385f4530620718ba01a7b5e993ba08ce38f7c7

playbooks/roles/os_neutron/defaults/main.yml

@@ -37,9 +37,9 @@ neutron_venv_download_url: http://127.0.0.1/venvs/untagged/ubuntu/neutron.tgz
 # Set the lib dir path to that of the local python path where neutron is installed.
 #  This is used for role access to the db migrations.
 #  Example:
-#  neutron_lib_dir: "/usr/local/lib/python2.7/dist-packages/neutron"
+#  neutron_lib_dir: "/usr/local/lib/python2.7/dist-packages/"
-neutron_venv_lib_dir: "{{ neutron_bin | dirname }}/lib/python2.7/site-packages/neutron"
+neutron_venv_lib_dir: "{{ neutron_bin | dirname }}/lib/python2.7/site-packages/"
-neutron_non_venv_lib_dir: "/usr/local/lib/python2.7/dist-packages/neutron"
+neutron_non_venv_lib_dir: "/usr/local/lib/python2.7/dist-packages/"
 neutron_lib_dir: "{{ (neutron_venv_enabled | bool) | ternary(neutron_venv_lib_dir, neutron_non_venv_lib_dir) }}"
 
 neutron_fatal_deprecations: False

playbooks/roles/os_neutron/library/neutron_migrations_facts

@@ -60,6 +60,9 @@ EXAMPLES = """
 
 
 MIGRATIONS = {
+    'projects': {
+        'neutron': {
+            'branches': {
                 'expand': {
                     'revision': None,
                     'head': None
@@ -68,18 +71,74 @@ MIGRATIONS = {
                     'revision': None,
                     'head': None
                 }
+            },
+            'installed': False
+        },
+        'neutron-fwaas': {
+            'branches': {
+                'expand': {
+                    'revision': None,
+                    'head': None
+                },
+                'contract': {
+                    'revision': None,
+                    'head': None
+                }
+            },
+            'installed': False
+        },
+        'neutron-lbaas': {
+            'branches': {
+                'expand': {
+                    'revision': None,
+                    'head': None
+                },
+                'contract': {
+                    'revision': None,
+                    'head': None
+                }
+            },
+            'installed': False
+        },
+        'neutron-vpnaas': {
+            'branches': {
+                'expand': {
+                    'revision': None,
+                    'head': None
+                },
+                'contract': {
+                    'revision': None,
+                    'head': None
+                }
+            },
+            'installed': False
+        }
+    },
+    'run_contract': True,
+    'run_expand': True
 }
 
+RELEASES = ['mitaka', 'liberty']
 
-def get_branch(release, revision, library_path):
+
+def get_branch(release, revision, library_path, project):
+    # Here we drop any releases that are newer than the specified release.
+    # We have to check previous releases for migrations as at time of writing,
+    # neutron-lbaas doesn't have mitaka migrations and therefore the plugin
+    # will fail unless we also check the previous release's migrations.
+    for release in RELEASES[RELEASES.index(release):]:
         migrations_dir = (
-        '%s/db/migration/alembic_migrations/versions/%s/' % (
+            '%s/%s/db/migration/alembic_migrations/versions/%s/' % (
                 library_path,
+                project.replace('-', '_'),
                 release,
             )
         )
-    for branch in MIGRATIONS.keys():
+        if os.path.isdir(migrations_dir):
-        migration_dir = os.path.join(get_abs_path(migrations_dir), branch)
+            for branch in MIGRATIONS['projects'][project]['branches'].keys():
+                migration_dir = os.path.join(
+                    get_abs_path(migrations_dir), branch
+                )
                 for file in os.listdir(migration_dir):
                     if file.endswith('.py') and file.split('_')[0] == revision:
                         return branch
@@ -93,6 +152,37 @@ def get_abs_path(path):
     )
 
 
+def update_run_keys():
+    projects = MIGRATIONS['projects']
+
+    # If a probject or subproject is marked as installed but has no revision,
+    # we can assume the project has just been installed.  In this case we
+    # just return to main, leaving run_contract/run_expand as True so that
+    # migrations will run.
+    # TODO(mattt): We will want to try to figure out if migrations in this
+    #              situation are expand, contract, or both, so that we avoid
+    #              unnecessary neutron-server downtime.
+    for p in projects:
+        if (projects[p]['installed'] is True and
+                projects[p]['branches']['expand']['revision'] is None and
+                projects[p]['branches']['contract']['revision'] is None):
+            return
+
+    # Here we will determine if migrations should be skipped -- this will
+    # happen when all the projects or subprojects that have a revision
+    # are also at head.
+    for b in ('contract', 'expand'):
+        migrations_with_a_rev = [
+            v
+            for v in projects.values()
+            if v['branches'][b]['revision'] is not None
+        ]
+        if (len(migrations_with_a_rev) > 0 and
+                all(m['branches'][b]['head'] is True
+                    for m in migrations_with_a_rev)):
+            MIGRATIONS['run_%s' % b] = False
+
+
 def main():
     module = AnsibleModule(
         argument_spec=dict(
@@ -111,7 +201,15 @@ def main():
         ),
         supports_check_mode=False
     )
+
+    if module.params['release'] not in RELEASES:
+        module.fail_json(
+            msg='neutron fact collection failed: release %s not found in'
+                ' %s' % (module.params['release'], RELEASES)
+        )
+
     state_change = False
+    project = None
 
     command = [
         '%s/neutron-db-manage' % get_abs_path(module.params['bin_path']),
@@ -125,16 +223,33 @@ def main():
 
     for line in current.splitlines():
         head = False
-        match = re.search("^([0-9a-z]{4,12})(\s\(head\))?$", line)
+        project_match = re.search(
-        if match:
+            "^\s*Running current for (neutron(-[a-z]+)?) ...$",
-            revision = match.group(1)
+            line
-            if match.group(2):
+        )
+        migration_match = re.search("^([0-9a-z]{4,12})(\s\(head\))?$", line)
+
+        if project_match:
+            project = project_match.group(1)
+            MIGRATIONS['projects'][project]['installed'] = True
+
+        if migration_match:
+            revision = migration_match.group(1)
+
+            if project is None:
+                module.fail_json(
+                    msg='neutron fact collection failed: unable to detect'
+                        ' project for revision %s' % revision
+                )
+
+            if migration_match.group(2):
                 head = True
 
             branch = get_branch(
                 release=module.params['release'],
                 revision=revision,
-                library_path=get_abs_path(module.params['library_path'])
+                library_path=get_abs_path(module.params['library_path']),
+                project=project
             )
             if branch is None:
                 module.fail_json(
@@ -142,8 +257,11 @@ def main():
                         ' migration with revision %s' % revision
                 )
 
-            MIGRATIONS[branch]['revision'] = revision
+            proj_migrations = MIGRATIONS['projects'][project]['branches']
-            MIGRATIONS[branch]['head'] = head
+            proj_migrations[branch]['revision'] = revision
+            proj_migrations[branch]['head'] = head
+
+    update_run_keys()
 
     module.exit_json(
         changed=state_change,

playbooks/roles/os_neutron/tasks/neutron_db_setup.yml

@@ -41,7 +41,7 @@
 
 - name: Get neutron migrations facts
   neutron_migrations_facts:
-    release: liberty
+    release: mitaka
     library_path: "{{ neutron_lib_dir }}"
     bin_path: "{{ neutron_bin }}"
   tags:
@@ -62,7 +62,7 @@
                       upgrade --expand
   sudo: yes
   sudo_user: "{{ neutron_system_user_name }}"
-  when: not neutron_migrations['expand']['head']|bool
+  when: neutron_migrations['run_expand']|bool
   tags:
     - neutron-db-setup
     - neutron-upgrade
@@ -74,7 +74,7 @@
     pattern: "neutron-server"
   delegate_to: "{{ item }}"
   with_items: groups['neutron_server']
-  when: not neutron_migrations['contract']['head']|bool
+  when: neutron_migrations['run_contract']|bool
   tags:
     - neutron-db-setup
     - neutron-upgrade
@@ -87,7 +87,7 @@
                       upgrade --contract
   sudo: yes
   sudo_user: "{{ neutron_system_user_name }}"
-  when: not neutron_migrations['contract']['head']|bool
+  when: neutron_migrations['run_contract']|bool
   tags:
     - neutron-db-setup
     - neutron-upgrade
@@ -99,7 +99,7 @@
     pattern: "neutron-server"
   delegate_to: "{{ item }}"
   with_items: groups['neutron_server']
-  when: not neutron_migrations['contract']['head']|bool
+  when: neutron_migrations['run_contract']|bool
   tags:
     - neutron-db-setup
     - neutron-upgrade