From f422d55e8a084c89fd765db8e0f68472d426ba76 Mon Sep 17 00:00:00 2001 From: Matt Thompson Date: Thu, 12 Nov 2015 14:20:54 +0000 Subject: [PATCH] Update neutron_migrations_facts This change does the following to neutron_migrations_facts: - updates MIGRATIONS dict by adding entries for each neutron sub-project while also adding keys to track which sub-projects are installed and which migrations need to be run (contract or expand) - takes subproject migrations into account (neutron-lbaas as an example) - updates neutron_venv_lib_dir and neutron_non_venv_lib_dir by removing reference to neutron specifically since these vars need to be used for all neutron subprojects also We also update neutron_db_setup.yml to use MIGRATIONS['run_contract'] and MIGRATIONS['run_expand'] to determine if neutron or a subproject has migrations which need to be applied. Closes-Bug: #1511756 Change-Id: I3a385f4530620718ba01a7b5e993ba08ce38f7c7 --- defaults/main.yml | 6 +- library/neutron_migrations_facts | 168 ++++++++++++++++++++++++++----- tasks/neutron_db_setup.yml | 10 +- 3 files changed, 151 insertions(+), 33 deletions(-) diff --git a/defaults/main.yml b/defaults/main.yml index 4fa7f589..2cba5d4b 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -37,9 +37,9 @@ neutron_venv_download_url: http://127.0.0.1/venvs/untagged/ubuntu/neutron.tgz # Set the lib dir path to that of the local python path where neutron is installed. # This is used for role access to the db migrations. # Example: -# neutron_lib_dir: "/usr/local/lib/python2.7/dist-packages/neutron" -neutron_venv_lib_dir: "{{ neutron_bin | dirname }}/lib/python2.7/site-packages/neutron" -neutron_non_venv_lib_dir: "/usr/local/lib/python2.7/dist-packages/neutron" +# neutron_lib_dir: "/usr/local/lib/python2.7/dist-packages/" +neutron_venv_lib_dir: "{{ neutron_bin | dirname }}/lib/python2.7/site-packages/" +neutron_non_venv_lib_dir: "/usr/local/lib/python2.7/dist-packages/" neutron_lib_dir: "{{ (neutron_venv_enabled | bool) | ternary(neutron_venv_lib_dir, neutron_non_venv_lib_dir) }}" neutron_fatal_deprecations: False diff --git a/library/neutron_migrations_facts b/library/neutron_migrations_facts index 27322dc8..3f941de0 100644 --- a/library/neutron_migrations_facts +++ b/library/neutron_migrations_facts @@ -60,29 +60,88 @@ EXAMPLES = """ MIGRATIONS = { - 'expand': { - 'revision': None, - 'head': None + 'projects': { + 'neutron': { + 'branches': { + 'expand': { + 'revision': None, + 'head': None + }, + 'contract': { + 'revision': None, + 'head': None + } + }, + 'installed': False + }, + 'neutron-fwaas': { + 'branches': { + 'expand': { + 'revision': None, + 'head': None + }, + 'contract': { + 'revision': None, + 'head': None + } + }, + 'installed': False + }, + 'neutron-lbaas': { + 'branches': { + 'expand': { + 'revision': None, + 'head': None + }, + 'contract': { + 'revision': None, + 'head': None + } + }, + 'installed': False + }, + 'neutron-vpnaas': { + 'branches': { + 'expand': { + 'revision': None, + 'head': None + }, + 'contract': { + 'revision': None, + 'head': None + } + }, + 'installed': False + } }, - 'contract': { - 'revision': None, - 'head': None - } + 'run_contract': True, + 'run_expand': True } +RELEASES = ['mitaka', 'liberty'] -def get_branch(release, revision, library_path): - migrations_dir = ( - '%s/db/migration/alembic_migrations/versions/%s/' % ( - library_path, - release, + +def get_branch(release, revision, library_path, project): + # Here we drop any releases that are newer than the specified release. + # We have to check previous releases for migrations as at time of writing, + # neutron-lbaas doesn't have mitaka migrations and therefore the plugin + # will fail unless we also check the previous release's migrations. + for release in RELEASES[RELEASES.index(release):]: + migrations_dir = ( + '%s/%s/db/migration/alembic_migrations/versions/%s/' % ( + library_path, + project.replace('-', '_'), + release, + ) ) - ) - for branch in MIGRATIONS.keys(): - migration_dir = os.path.join(get_abs_path(migrations_dir), branch) - for file in os.listdir(migration_dir): - if file.endswith('.py') and file.split('_')[0] == revision: - return branch + if os.path.isdir(migrations_dir): + for branch in MIGRATIONS['projects'][project]['branches'].keys(): + migration_dir = os.path.join( + get_abs_path(migrations_dir), branch + ) + for file in os.listdir(migration_dir): + if file.endswith('.py') and file.split('_')[0] == revision: + return branch def get_abs_path(path): @@ -93,6 +152,37 @@ def get_abs_path(path): ) +def update_run_keys(): + projects = MIGRATIONS['projects'] + + # If a probject or subproject is marked as installed but has no revision, + # we can assume the project has just been installed. In this case we + # just return to main, leaving run_contract/run_expand as True so that + # migrations will run. + # TODO(mattt): We will want to try to figure out if migrations in this + # situation are expand, contract, or both, so that we avoid + # unnecessary neutron-server downtime. + for p in projects: + if (projects[p]['installed'] is True and + projects[p]['branches']['expand']['revision'] is None and + projects[p]['branches']['contract']['revision'] is None): + return + + # Here we will determine if migrations should be skipped -- this will + # happen when all the projects or subprojects that have a revision + # are also at head. + for b in ('contract', 'expand'): + migrations_with_a_rev = [ + v + for v in projects.values() + if v['branches'][b]['revision'] is not None + ] + if (len(migrations_with_a_rev) > 0 and + all(m['branches'][b]['head'] is True + for m in migrations_with_a_rev)): + MIGRATIONS['run_%s' % b] = False + + def main(): module = AnsibleModule( argument_spec=dict( @@ -111,7 +201,15 @@ def main(): ), supports_check_mode=False ) + + if module.params['release'] not in RELEASES: + module.fail_json( + msg='neutron fact collection failed: release %s not found in' + ' %s' % (module.params['release'], RELEASES) + ) + state_change = False + project = None command = [ '%s/neutron-db-manage' % get_abs_path(module.params['bin_path']), @@ -125,16 +223,33 @@ def main(): for line in current.splitlines(): head = False - match = re.search("^([0-9a-z]{4,12})(\s\(head\))?$", line) - if match: - revision = match.group(1) - if match.group(2): + project_match = re.search( + "^\s*Running current for (neutron(-[a-z]+)?) ...$", + line + ) + migration_match = re.search("^([0-9a-z]{4,12})(\s\(head\))?$", line) + + if project_match: + project = project_match.group(1) + MIGRATIONS['projects'][project]['installed'] = True + + if migration_match: + revision = migration_match.group(1) + + if project is None: + module.fail_json( + msg='neutron fact collection failed: unable to detect' + ' project for revision %s' % revision + ) + + if migration_match.group(2): head = True branch = get_branch( release=module.params['release'], revision=revision, - library_path=get_abs_path(module.params['library_path']) + library_path=get_abs_path(module.params['library_path']), + project=project ) if branch is None: module.fail_json( @@ -142,8 +257,11 @@ def main(): ' migration with revision %s' % revision ) - MIGRATIONS[branch]['revision'] = revision - MIGRATIONS[branch]['head'] = head + proj_migrations = MIGRATIONS['projects'][project]['branches'] + proj_migrations[branch]['revision'] = revision + proj_migrations[branch]['head'] = head + + update_run_keys() module.exit_json( changed=state_change, diff --git a/tasks/neutron_db_setup.yml b/tasks/neutron_db_setup.yml index 7d76ebec..29e739f0 100644 --- a/tasks/neutron_db_setup.yml +++ b/tasks/neutron_db_setup.yml @@ -41,7 +41,7 @@ - name: Get neutron migrations facts neutron_migrations_facts: - release: liberty + release: mitaka library_path: "{{ neutron_lib_dir }}" bin_path: "{{ neutron_bin }}" tags: @@ -62,7 +62,7 @@ upgrade --expand sudo: yes sudo_user: "{{ neutron_system_user_name }}" - when: not neutron_migrations['expand']['head']|bool + when: neutron_migrations['run_expand']|bool tags: - neutron-db-setup - neutron-upgrade @@ -74,7 +74,7 @@ pattern: "neutron-server" delegate_to: "{{ item }}" with_items: groups['neutron_server'] - when: not neutron_migrations['contract']['head']|bool + when: neutron_migrations['run_contract']|bool tags: - neutron-db-setup - neutron-upgrade @@ -87,7 +87,7 @@ upgrade --contract sudo: yes sudo_user: "{{ neutron_system_user_name }}" - when: not neutron_migrations['contract']['head']|bool + when: neutron_migrations['run_contract']|bool tags: - neutron-db-setup - neutron-upgrade @@ -99,7 +99,7 @@ pattern: "neutron-server" delegate_to: "{{ item }}" with_items: groups['neutron_server'] - when: not neutron_migrations['contract']['head']|bool + when: neutron_migrations['run_contract']|bool tags: - neutron-db-setup - neutron-upgrade