From a88778f2aedb9936a6f139d502ab3d8c81c9243a Mon Sep 17 00:00:00 2001 From: Kevin Carter Date: Tue, 3 May 2016 17:20:43 -0500 Subject: [PATCH] Initial commit to enable mitaka>newton upgrades This commit imports much of the documentation, scripts, and playbooks from the Liberty>Mitaka upgrade process and applies the appropriate series name changes. Implements: blueprint upgrade-mitaka-newton Change-Id: If8e43f1549e6fd121eae7b8d98d8cb16b01e2aab Signed-off-by: Kevin Carter (cherry picked from commit 5a09cb7c8dac51788686c5effbec50568293447d) --- doc/source/developer-docs/navigation.txt | 1 + doc/source/install-guide/navigation.txt | 1 + doc/source/upgrade-guide/index.rst | 14 ++ doc/source/upgrade-guide/manual-upgrade.rst | 160 ++++++++++++++++++ doc/source/upgrade-guide/navigation.txt | 4 + doc/source/upgrade-guide/overview.rst | 66 ++++++++ doc/source/upgrade-guide/scripts.rst | 61 +++++++ .../upgrade-guide/upgrade-playbooks.rst | 79 +++++++++ scripts/run-upgrade.sh | 108 +++++++++--- .../playbooks/ansible_fact_cleanup.yml | 25 +++ .../playbooks/deploy-config-changes.yml | 44 +++++ .../playbooks/memcached-flush.yml | 23 +++ .../repo-server-pip-conf-removal.yml | 24 +++ .../playbooks/user-secrets-adjustment.yml | 33 ++++ .../scripts/ansible_fact_cleanup.sh | 18 ++ .../scripts/make_rst_table.py | 45 +++++ .../scripts/migrate_openstack_vars.py | 70 ++++++++ .../scripts/test_migrate_openstack_vars.py | 86 ++++++++++ 18 files changed, 842 insertions(+), 20 deletions(-) create mode 100644 doc/source/upgrade-guide/index.rst create mode 100644 doc/source/upgrade-guide/manual-upgrade.rst create mode 100644 doc/source/upgrade-guide/navigation.txt create mode 100644 doc/source/upgrade-guide/overview.rst create mode 100644 doc/source/upgrade-guide/scripts.rst create mode 100644 doc/source/upgrade-guide/upgrade-playbooks.rst create mode 100644 scripts/upgrade-utilities/playbooks/ansible_fact_cleanup.yml create mode 100644 scripts/upgrade-utilities/playbooks/deploy-config-changes.yml create mode 100644 scripts/upgrade-utilities/playbooks/memcached-flush.yml create mode 100644 scripts/upgrade-utilities/playbooks/repo-server-pip-conf-removal.yml create mode 100644 scripts/upgrade-utilities/playbooks/user-secrets-adjustment.yml create mode 100755 scripts/upgrade-utilities/scripts/ansible_fact_cleanup.sh create mode 100755 scripts/upgrade-utilities/scripts/make_rst_table.py create mode 100755 scripts/upgrade-utilities/scripts/migrate_openstack_vars.py create mode 100755 scripts/upgrade-utilities/scripts/test_migrate_openstack_vars.py diff --git a/doc/source/developer-docs/navigation.txt b/doc/source/developer-docs/navigation.txt index 6e0cf79698..9d31296e4f 100644 --- a/doc/source/developer-docs/navigation.txt +++ b/doc/source/developer-docs/navigation.txt @@ -1,3 +1,4 @@ * `Documentation Home <../index.html>`_ * `Installation Guide <../install-guide/index.html>`_ +* `Upgrade Guide <../upgrade-guide/index.html>`_ * `Developer Documentation `_ diff --git a/doc/source/install-guide/navigation.txt b/doc/source/install-guide/navigation.txt index f6c86bd471..57b709815b 100644 --- a/doc/source/install-guide/navigation.txt +++ b/doc/source/install-guide/navigation.txt @@ -1,3 +1,4 @@ * `Documentation Home <../index.html>`_ * `Installation Guide `_ +* `Upgrade Guide <../upgrade-guide/index.html>`_ * `Developer Documentation <../developer-docs/index.html>`_ diff --git a/doc/source/upgrade-guide/index.rst b/doc/source/upgrade-guide/index.rst new file mode 100644 index 0000000000..809b7ee8c6 --- /dev/null +++ b/doc/source/upgrade-guide/index.rst @@ -0,0 +1,14 @@ +Upgrade Documentation +===================== + +In this section you will find information on the Mitaka to Newton upgrade process. + +Contents: + +.. toctree:: + :maxdepth: 2 + + overview + manual-upgrade + scripts + upgrade-playbooks diff --git a/doc/source/upgrade-guide/manual-upgrade.rst b/doc/source/upgrade-guide/manual-upgrade.rst new file mode 100644 index 0000000000..271043b308 --- /dev/null +++ b/doc/source/upgrade-guide/manual-upgrade.rst @@ -0,0 +1,160 @@ +.. _manual-upgrade: + +Manual Upgrade Steps +==================== + +The steps detailed here match those performed by the ``run-upgrade.sh`` +script. Any of these steps can safely be run multiple times. + +Checkout Newton version +----------------------- + +Ensure your OpenStack-Ansible code is on the latest Newton release tag (14.x.x). + +.. code-block:: console + + # git checkout stable/newton + # LATEST_TAG=$(git describe --abbrev=0 --tags) + # git checkout ${LATEST_TAG} + +Preparing the shell variables +----------------------------- + +.. note:: + + This step is optional, since these environment variables are simply + shortcuts. Files can be referenced directly. + +From the ``openstack-ansible`` root directory, run the following. + +.. code-block:: console + + # export MAIN_PATH="$(pwd)" + # export SCRIPTS_PATH="${MAIN_PATH}/scripts" + # export UPGRADE_PLAYBOOKS="${SCRIPTS_PATH}/upgrade-utilities/playbooks" + +These variables will reduce typing when running the remaining upgrade +tasks. + +Re-bootstrap Ansible for Newton +------------------------------- + +Bootstrapping Ansible again ensures that new external Ansible role +dependencies are in place before running playbooks from the Newton +release. + +.. code-block:: console + + # ${SCRIPTS_PATH}/bootstrap-ansible.sh + +Change to playbooks directory +----------------------------- + +Change to the playbooks directory so that the Ansible dynamic +inventory will be found automatically. + +.. code-block:: console + + # cd playbooks + +Update configuration and environment files +------------------------------------------ + +The user configuration files in ``/etc/openstack_deploy/`` and the environment +layout in ``/etc/openstack_deploy/env.d`` have had names changed and new +values added in Newton. This step updates the files as necessary. + +See :ref:`config-change-playbook` for more details. + +.. code-block:: console + + # openstack-ansible "${UPGRADE_PLAYBOOKS}/deploy-config-changes.yml" + +.. note:: + + The `-e pip_install_options=--force-reinstall` ensures that all pip + packages are reinstalled and running the correct version on hosts. + +Update user secrets file +------------------------ + +Newton introduces new user secrets to the stack. These are populated +automatically with the following playbook. + +See :ref:`user-secrets-playbook` for more details. + +.. code-block:: console + + # openstack-ansible "${UPGRADE_PLAYBOOKS}/user-secrets-adjustment.yml" + +Upgrade hosts +------------- + +Before installing the infrastructure and OpenStack, update the host machines. + +.. code-block:: console + + # openstack-ansible setup-hosts.yml --limit '!galera_all[0]' + +This command is the same as doing host setups on a new install. The first +member of the ``galera_all`` host group is excluded to prevent simultaneous +restarts of all galera containers. + +Update Galera LXC container configuration +----------------------------------------- + +Update the first galera container's configuration independently. + +.. code-block:: console + + # openstack-ansible lxc-containers-create.yml --limit galera_all[0] + +This command is a subset of the host setup playbook, limited to the first +member of the ``galera_all`` host group so that its container is restarted only +after other galera containers have been restarted in the previous step. + +Cleanup ``pip.conf`` file in the ``repo_servers`` +------------------------------------------------- + +It is possible that a ``pip.conf`` file may exist within the repository server +infrastructure. If this file exists, it will cause build failures when upgrading +to Newton. This play will remove the ``pip.conf`` file from the repository +servers if it is found. + +See :ref:`repo-server-pip-conf-removal` for more details. + +.. code-block:: console + + # openstack-ansible "${UPGRADE_PLAYBOOKS}/repo-server-pip-conf-removal.yml" + + +Upgrade infrastructure +---------------------- + +Running the standard OpenStack-Ansible infrastructure playbook applies the +relevant Newton settings and packages. This upgrade is required for the Newton +release of OpenStack-Ansible. + +RabbitMQ may need a minor version upgrade depending on what version of Mitaka +was previously installed. + +See :ref:`setup-infra-playbook` for details. + +.. code-block:: console + + # openstack-ansible setup-infrastructure.yml -e 'galera_upgrade=true' \ + -e 'rabbitmq_upgrade=true' + +Upgrade OpenStack +----------------- + +Upgrading the OpenStack components is done with the same playbook that +installs them, without any additional options. + +.. code-block:: console + + # openstack-ansible setup-openstack.yml + +-------------- + +.. include:: navigation.txt diff --git a/doc/source/upgrade-guide/navigation.txt b/doc/source/upgrade-guide/navigation.txt new file mode 100644 index 0000000000..e7f7212f19 --- /dev/null +++ b/doc/source/upgrade-guide/navigation.txt @@ -0,0 +1,4 @@ +* `Documentation Home <../index.html>`_ +* `Installation Guide <../installation-guide/index.html>`_ +* `Upgrade Guide `_ +* `Developer Documentation <../developer-docs/index.html>`_ diff --git a/doc/source/upgrade-guide/overview.rst b/doc/source/upgrade-guide/overview.rst new file mode 100644 index 0000000000..b5336b6b42 --- /dev/null +++ b/doc/source/upgrade-guide/overview.rst @@ -0,0 +1,66 @@ +Overview +======== + +The OpenStack-Ansible repository provides playbooks and scripts used +to upgrade an environment from Mitaka to Newton. The ``run-upgrade.sh`` +script runs each upgrade playbook in the correct order, or playbooks +can be run individually if necessary. + +Running the Upgrade script +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The Newton series releases of OpenStack-Ansible contain the code for +migrating from Mitaka to Newton. + +.. warning:: + + The upgrade script is still under active development and should not be run at this time. + +To upgrade from Mitaka to Newton using the upgrade script, perform the +following steps in the ``openstack-ansible`` directory: + +.. code-block:: console + + # git checkout stable/newton + # LATEST_TAG=$(git describe --abbrev=0 --tags) + # git checkout ${LATEST_TAG} + # ./scripts/run-upgrade.sh + +Upgrading Manually +~~~~~~~~~~~~~~~~~~ + +Deployers can run the upgrade steps manually. See :ref:`manual-upgrade`. +Manual execution is useful for scoping the changes in the upgrade process +(For example, in very large deployments with strict SLA requirements), or for +inclusion into other orchestration for upgrade automation beyond what +OpenStack-Ansible provides. + +Upgrade Actions +~~~~~~~~~~~~~~~ + +Both the upgrade script and manual upgrade steps perform the actions and +use the concepts introduced below. + +Configuration Changes +--------------------- + +The upgrade process will modify files residing in ``/etc/openstack_deploy`` in +order to reflect new Newton values. + +Flag Files +---------- + +Some flag files are created by the migration scripts in order to achieve +idempotency. These files are placed in the ``/etc/openstack_deploy.NEWTON`` +directory. + +RabbitMQ Upgrade +---------------- + +The RabbitMQ server can be upgraded during an OpenStack-Ansible upgrade. + +See :ref:`setup-infra-playbook` for details. + +-------------- + +.. include:: navigation.txt diff --git a/doc/source/upgrade-guide/scripts.rst b/doc/source/upgrade-guide/scripts.rst new file mode 100644 index 0000000000..7356453ebc --- /dev/null +++ b/doc/source/upgrade-guide/scripts.rst @@ -0,0 +1,61 @@ +Scripts +======= + +This section describes scripts that are used in the upgrade process in detail. + +Within the main :file:`scripts` directory there is a :file:`upgrade-utilities` +directory, which contains additional scripts that facilitate the initial +upgrade process. + +run-upgrade.sh +-------------- + +This script controls the overall upgrade process for deployers choosing not to +do so manually. + +It provides the following environment variables: + + * ``SCRIPTS_PATH`` - path to the top level scripts directory + * ``MAIN_PATH`` - openstack_ansible root directory. + * ``UPGRADE_PLAYBOOKS`` - path to the playbooks used in upgrading + +The upgrade script will also bootstrap ansible (using +``bootstrap-ansible.sh``) in order to provide the new role dependencies +introduced in the Mitaka series. + +.. _migrate-os-vars: + +migrate_openstack_vars.py +------------------------- + +In Mitaka, some variable names were changed to reflect upstream decisions. +This script will look for and replace any instances of these strings in the +variable override files matching the pattern +``/etc/openstack_deploy/user_*.yml``. +Comments in the file will be preserved, though the variable names within the +comments will be updated. + +This script will also create files of the form +``/etc/openstack_deploy.NEWTON/VARS_MIGRATED_file``. For example, once the script has +processed the file ``/etc/openstack_deploy/user_variables.yml``. it creates +``/etc/openstack_deploy.NEWTON/VARS_MIGRATED_user_variables`` to indicate to +ansible that the step can be skipped on successive runs. The script itself does +not check for this file. + +The variable changes are shown in the following table. + +.. This table was made with the output of + scripts/upgrade-utilities/scripts/make_rst_table.py. Insertion needs to be + done manually since the OpenStack publish jobs do not use `make` and there + isn't yet a sphinx extension that runs an abitrary script on build. + ++------------------------------------------+------------------------------------------+ +| Old Value | New Value | ++==========================================+==========================================+ ++------------------------------------------+------------------------------------------+ + +Called by :ref:`config-change-playbook` + +-------------- + +.. include:: navigation.txt diff --git a/doc/source/upgrade-guide/upgrade-playbooks.rst b/doc/source/upgrade-guide/upgrade-playbooks.rst new file mode 100644 index 0000000000..a950241983 --- /dev/null +++ b/doc/source/upgrade-guide/upgrade-playbooks.rst @@ -0,0 +1,79 @@ +Upgrade Playbooks +================= + +This section describes the playbooks that are used in the upgrade process in +further detail. + +Within the main :file:`scripts` directory there is an :file:`upgrade-utilities` +directory, which contains an additional playbooks directory. These playbooks +facilitate the upgrade process. + +.. _config-change-playbook: + +deploy-config-changes.yml +------------------------- + +This playbook will back up the ``/etc/openstack_deploy`` directory before +making the necessary changes to the configuration. + +``/etc/openstack_deploy`` is copied once to ``/etc/openstack_deploy.NEWTON``. +The copy happens only once, so repeated runs are safe. + +.. _user-secrets-playbook: + +user-secrets-adjustment.yml +--------------------------- + +This playbook ensures that the user secrets file is updated based on the example +file in the main repository. This makes it possible to guarantee all secrets are +carried into the upgraded environment and appropriately generated. Only new +secrets are added, such as those necessary for new services or new settings +added to existing services. Previously set values will not be changed. + +.. _setup-infra-playbook: + +repo-server-pip-conf-removal.yml +-------------------------------- + +This playbook ensures the repository servers do not have the ``pip.conf`` in the +root ``pip`` directory locking down the python packages available to install. If +this file exists on the repository servers it will cause build failures. + +.. _repo-server-pip-conf-removal: + +setup-infrastructure.yml +------------------------ + +The ``setup-infrastructure.yml`` playbook is contained in the main +``playbooks`` directory, but is called by ``run-upgrade.sh`` with specific +arguments in order to upgrade infrastructure components such as MariaDB and +RabbitMQ. + +For example, to run an upgrade for both components at once, run the following: + +.. code-block:: console + + # openstack-ansible setup-infrastructure.yml -e 'rabbitmq_upgrade=true' \ + # -e 'galera_upgrade=true' + +The ``rabbitmq_upgrade`` variable tells the ``rabbitmq_server`` role to +upgrade the running major/minor version of RabbitMQ. + +.. note:: + The RabbitMQ server role will install patch releases automatically, + regardless of the value of ``rabbitmq_upgrade``. This variable only + controls upgrading the major or minor version. + + Upgrading RabbitMQ in the Newton release is optional. The + ``run-upgrade.sh`` script will not automatically upgrade it. If a RabbitMQ + upgrade using the script is desired, insert the ``rabbitmq_upgrade: true`` + line into a file such as ``/etc/openstack_deploy/user_variables.yml``. + +The ``galera_upgrade`` variable tells the ``galera_server`` role to remove the +current version of MariaDB/Galera and upgrade to the 10.x series. + +.. _setup-infra-playbook: + +-------------- + +.. include:: navigation.txt diff --git a/scripts/run-upgrade.sh b/scripts/run-upgrade.sh index 14af0652f4..f705d5b0fa 100755 --- a/scripts/run-upgrade.sh +++ b/scripts/run-upgrade.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash + # Copyright 2015, Rackspace US, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -20,23 +21,72 @@ ## Shell Opts ---------------------------------------------------------------- set -e -u -v +export SCRIPTS_PATH="$(dirname $(readlink -f $0))" +export MAIN_PATH="$(dirname ${SCRIPTS_PATH})" +export UPGRADE_PLAYBOOKS="${SCRIPTS_PATH}/upgrade-utilities/playbooks" + ## Functions ----------------------------------------------------------------- -function check_for_juno { - if [ -d "/etc/rpc_deploy" ];then - echo "--------------ERROR--------------" - echo "/etc/rpc_deploy directory found, which looks like you're trying to upgrade from Juno." - echo "Please upgrade your environment to Kilo before proceeding." - exit 1 +function run_lock { + set +e + run_item="${RUN_TASKS[$1]}" + file_part="${run_item}" + + # note(sigmavirus24): this handles tasks like: + # "-e 'rabbitmq_upgrade=true' setup-infrastructure.yml" + # "/tmp/fix_container_interfaces.yml || true" + # so we can get the appropriate basename for the upgrade_marker + for part in $run_item; do + if [[ "$part" == *.yml ]];then + file_part="$part" + break fi + done + + if [ ! -d "/etc/openstack_deploy/upgrade-newton" ]; then + mkdir -p "/etc/openstack_deploy/upgrade-newton" + fi + + upgrade_marker_file=$(basename ${file_part} .yml) + upgrade_marker="/etc/openstack_deploy/upgrade-newton/$upgrade_marker_file.complete" + + if [ ! -f "$upgrade_marker" ];then + # note(sigmavirus24): use eval so that we properly turn strings like + # "/tmp/fix_container_interfaces.yml || true" + # into a command, otherwise we'll get an error that there's no playbook + # named || + eval "openstack-ansible $2 -e 'pip_install_options=--force-reinstall'" + playbook_status="$?" + echo "ran $run_item" + + if [ "$playbook_status" == "0" ];then + RUN_TASKS=("${RUN_TASKS[@]/$run_item}") + touch "$upgrade_marker" + echo "$run_item has been marked as success" + else + echo "******************** failure ********************" + echo "The upgrade script has encountered a failure." + echo "Failed on task $run_item" + echo "Re-run the run-upgrade.sh script, or" + echo "execute the remaining tasks manually:" + # run the tasks in order + for item in ${!RUN_TASKS[@]}; do + echo "openstack-ansible ${RUN_TASKS[$item]} -e 'pip_install_options=--force-reinstall'" + done + echo "******************** failure ********************" + exit 99 + fi + else + RUN_TASKS=("${RUN_TASKS[@]/$run_item.*}") + fi + set -e } - -function check_for_kilo { +function check_for_current { if [[ ! -d "/etc/openstack_deploy" ]]; then echo "--------------ERROR--------------" echo "/etc/openstack_deploy directory not found." - echo "It appears you do not have a Kilo environment installed." + echo "It appears you do not have a current environment installed." exit 2 fi } @@ -70,23 +120,41 @@ function pre_flight { fi } -function disable_security_hardening { - echo 'apply_security_hardening: False' > /etc/openstack_deploy/user_zzz_disable_security_hardening.yml -} - - -function reset_security_hardening { - rm /etc/openstack_deploy/user_zzz_disable_security_hardening.yml +function exit_early { + echo -e " + The upgrade script is still under active development and should not be + run at this time. For test environments the early exit of the script can + skipped by executing ``export I_REALLY_KNOW_WHAT_I_AM_DOING=true`` before + ``run-upgrade.sh``. + " + exit 99 } ## Main ---------------------------------------------------------------------- function main { + [[ "${I_REALLY_KNOW_WHAT_I_AM_DOING}" = true ]] || exit_early + exit_early pre_flight - disable_security_hardening - check_for_juno - check_for_kilo - reset_security_hardening + check_for_current + + "${SCRIPTS_PATH}/bootstrap-ansible.sh" + + pushd ${MAIN_PATH}/playbooks + RUN_TASKS+=("${UPGRADE_PLAYBOOKS}/ansible_fact_cleanup.yml") + RUN_TASKS+=("${UPGRADE_PLAYBOOKS}/deploy-config-changes.yml") + RUN_TASKS+=("${UPGRADE_PLAYBOOKS}/user-secrets-adjustment.yml") + RUN_TASKS+=("setup-hosts.yml --limit '!galera_all[0]'") + RUN_TASKS+=("lxc-containers-create.yml --limit galera_all[0]") + RUN_TASKS+=("${UPGRADE_PLAYBOOKS}/repo-server-pip-conf-removal.yml") + RUN_TASKS+=("setup-infrastructure.yml -e 'galera_upgrade=true' -e 'rabbitmq_upgrade=true'") + RUN_TASKS+=("${UPGRADE_PLAYBOOKS}/memcached-flush.yml") + RUN_TASKS+=("setup-openstack.yml") + # Run the tasks in order + for item in ${!RUN_TASKS[@]}; do + run_lock $item "${RUN_TASKS[$item]}" + done + popd } main diff --git a/scripts/upgrade-utilities/playbooks/ansible_fact_cleanup.yml b/scripts/upgrade-utilities/playbooks/ansible_fact_cleanup.yml new file mode 100644 index 0000000000..89437a3974 --- /dev/null +++ b/scripts/upgrade-utilities/playbooks/ansible_fact_cleanup.yml @@ -0,0 +1,25 @@ +--- +# Copyright 2015, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Ansible fact cleanup + hosts: localhost + connection: local + gather_facts: false + user: root + tasks: + - name: Remove any of the stored facts ansible may already have + command: "{{ upgrade_scripts }}/ansible_fact_cleanup.sh" + vars: + upgrade_scripts: "{{ playbook_dir }}/../scripts" diff --git a/scripts/upgrade-utilities/playbooks/deploy-config-changes.yml b/scripts/upgrade-utilities/playbooks/deploy-config-changes.yml new file mode 100644 index 0000000000..cb0ff39079 --- /dev/null +++ b/scripts/upgrade-utilities/playbooks/deploy-config-changes.yml @@ -0,0 +1,44 @@ +--- +# Copyright 2016, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Upgrade environment/inventory configuration + hosts: localhost + connection: local + gather_facts: false + user: root + tasks: + - name: Create an old copy of openstack_deploy + copy: + src: "/etc/openstack_deploy/" + dest: "/etc/openstack_deploy.NEWTON/" + force: no + + - name: Copy new env.d files into place + copy: + src: "{{ repo_root_dir }}/etc/openstack_deploy/env.d/{{ item }}.yml" + dest: "/etc/openstack_deploy/env.d/{{ item }}.yml" + force: no + with_items: + - ironic + + - name: Update OpenStack variable names + command: "{{ upgrade_scripts }}/migrate_openstack_vars.py {{ item }} {{ (item | basename)[:-4] }}" + args: + creates: "/etc/openstack_deploy.NEWTON/VARS_MIGRATED_{{ (item | basename)[:-4] }}" + with_fileglob: + - "/etc/openstack_deploy/user_*.yml" + vars: + upgrade_scripts: "{{ playbook_dir }}/../scripts" + repo_root_dir: "{{ playbook_dir }}/../../../" diff --git a/scripts/upgrade-utilities/playbooks/memcached-flush.yml b/scripts/upgrade-utilities/playbooks/memcached-flush.yml new file mode 100644 index 0000000000..81e3243221 --- /dev/null +++ b/scripts/upgrade-utilities/playbooks/memcached-flush.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2016, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Memcached cache flush + hosts: memcached_all + gather_facts: false + user: root + tasks: + - name: Flush all of the cache in memcached + shell: | + echo 'flush_all' | nc $(awk '/\-l/ {print $2}' /etc/memcached.conf) $(awk '/\-p/ {print $2}' /etc/memcached.conf) diff --git a/scripts/upgrade-utilities/playbooks/repo-server-pip-conf-removal.yml b/scripts/upgrade-utilities/playbooks/repo-server-pip-conf-removal.yml new file mode 100644 index 0000000000..4919249cee --- /dev/null +++ b/scripts/upgrade-utilities/playbooks/repo-server-pip-conf-removal.yml @@ -0,0 +1,24 @@ +--- +# Copyright 2015, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Remove pip.conf if found + hosts: repo_all + gather_facts: true + user: root + tasks: + - name: Remove pip.conf + file: + path: "{{ ansible_env.HOME }}/.pip/pip.conf" + state: "absent" diff --git a/scripts/upgrade-utilities/playbooks/user-secrets-adjustment.yml b/scripts/upgrade-utilities/playbooks/user-secrets-adjustment.yml new file mode 100644 index 0000000000..77cc58b013 --- /dev/null +++ b/scripts/upgrade-utilities/playbooks/user-secrets-adjustment.yml @@ -0,0 +1,33 @@ +--- +# Copyright 2015, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: User secrets adjustments + hosts: localhost + connection: local + gather_facts: false + user: root + tasks: + - name: Read example user secrets file + shell: "grep '^[a-zA-Z]' {{ playbook_dir }}/../../../etc/openstack_deploy/user_secrets.yml" + register: secrets + - name: Add missing secret + shell: | + if ! grep '^{{ item }}' /etc/openstack_deploy/user_secrets.yml; then + echo {{ item }} | tee -a /etc/openstack_deploy/user_secrets.yml + fi + with_items: secrets.stdout_lines + - name: Generate new secrets + shell: "{{ playbook_dir }}/../../../scripts/pw-token-gen.py --file /etc/openstack_deploy/user_secrets.yml" + diff --git a/scripts/upgrade-utilities/scripts/ansible_fact_cleanup.sh b/scripts/upgrade-utilities/scripts/ansible_fact_cleanup.sh new file mode 100755 index 0000000000..9684d12256 --- /dev/null +++ b/scripts/upgrade-utilities/scripts/ansible_fact_cleanup.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# Copyright 2016, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# find all of the ansible_facts that may be saved and remove them +# prior to running the upgrade. +find /etc/openstack_deploy/ansible_facts/ -type f -exec rm {} \; diff --git a/scripts/upgrade-utilities/scripts/make_rst_table.py b/scripts/upgrade-utilities/scripts/make_rst_table.py new file mode 100755 index 0000000000..594488dfdd --- /dev/null +++ b/scripts/upgrade-utilities/scripts/make_rst_table.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python +# Copyright 2016, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is used to generate a ReStructured Text table suitable for +# documentating the variable name changes. Its contents are meant to be +# inserted into doc/source/upgrade-guide/scripts.rst. + +# As of right now, running this script and inserting the output into +# the file is manual. + +from migrate_openstack_vars import VAR_MAPPINGS + +# Print old/new values in each row, right aligned. +row_format = "| {:>40} | {:>40} |" + +# For the line separators, move the dividing '+' sign over so it's aligned +# with the '|' in the rows. +divider_format = "+-{:->42}---{:->40}" +header_divide_format = "+={:=>42}==={:=>40}" + + +# Header info +print(divider_format.format('+', '+')) +print(row_format.format('Old Value', 'New Value')) +print(header_divide_format.format('+', '+')) + +# If we just used the items method, we'd get an unsorted output. +keys = VAR_MAPPINGS.keys() +keys.sort() + +for key in keys: + print(row_format.format(key, VAR_MAPPINGS[key])) + print(divider_format.format('+', '+')) diff --git a/scripts/upgrade-utilities/scripts/migrate_openstack_vars.py b/scripts/upgrade-utilities/scripts/migrate_openstack_vars.py new file mode 100755 index 0000000000..739baa86c4 --- /dev/null +++ b/scripts/upgrade-utilities/scripts/migrate_openstack_vars.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python +# Copyright 2016, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This could probably be made more generic, since the biggest change per +# service is the variable mappings +import sys + +VAR_MAPPINGS = { + # Add mapped items here + 'test-old': 'test-new' +} + + +def update_variables(old_contents): + """Replace all references to old variables. + + This includes comments and references within values for other variables. + """ + new_contents = [] + + for line in old_contents: + words = line.split() + + for word in words: + # Using the whitespace split above, the keys in the yaml file will + # have a : at the end, so we need to strip that off before + # replacing + if word.endswith(':'): + word = word[:-1] + + if word in VAR_MAPPINGS.keys(): + line = line.replace(word, VAR_MAPPINGS[word]) + + new_contents.append(line) + + return new_contents + + +def main(filename): + with open(filename, 'r') as f: + contents = f.readlines() + + new_contents = update_variables(contents) + + with open(filename, 'w') as f: + f.write(''.join(new_contents)) + +if __name__ == '__main__': + if len(sys.argv) < 3: + sys.exit("Filename and flag file reference required.") + + filename = sys.argv[1] + flag_ref = sys.argv[2] + main(filename) + + flag_file = '/etc/openstack_deploy.NEWTON/VARS_MIGRATED_%s' % flag_ref + with open(flag_file, 'w') as f: + f.write('OpenStack Newton variables migrated.') diff --git a/scripts/upgrade-utilities/scripts/test_migrate_openstack_vars.py b/scripts/upgrade-utilities/scripts/test_migrate_openstack_vars.py new file mode 100755 index 0000000000..63edd134d5 --- /dev/null +++ b/scripts/upgrade-utilities/scripts/test_migrate_openstack_vars.py @@ -0,0 +1,86 @@ +# Copyright 2016, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from migrate_openstack_vars import main +from migrate_openstack_vars import VAR_MAPPINGS + +import os +import sys + + +FILE_NAME = 'test_user_variables.yml' + + +def set_up(): + # Create an example file with key/value pairs, as well as a comment + # The old to new value mappings are written to a file, then later the + # file is inspected to ensure no old values remain. + var_lines = ["{}: {}".format(key, val) for + key, val in VAR_MAPPINGS.items()] + var_lines.append('# A test comment') + sample = VAR_MAPPINGS.items()[0] + var_lines.append('# {} / {}'.format(*sample)) + with open(FILE_NAME, 'w') as f: + f.write('\n'.join(var_lines)) + + +def teardown(): + # Remove files so they don't pollute the directories. + os.remove(FILE_NAME) + +def test(): + main(FILE_NAME) + + with open(FILE_NAME, 'r') as f: + contents = f.readlines() + + for line in contents: + # only split lines that look like a key/value pair. + if ':' in line: + var, value = line.split(':', 1) + value = value.strip() + elif '/' in line: + # For the comment containing a variable, clean up the list + # contents before assigning the parts we want to test. + parts = line.split() + parts.remove('#') + parts.remove('/') + var, value = parts + else: + var = value = line + + + # Once run through the 'main' function, the keys and values should + # match + if not value == var: + import pdb; pdb.set_trace() # NOQA + print("Var and value don't match.") + print("Var: {}, Value: {}".format(var, value)) + sys.exit() + + invalid_variable = var not in VAR_MAPPINGS.values() + # Comments aren't in our test mapping, so make sure we ignore them + is_comment = line.startswith('#') + + if invalid_variable and not is_comment: + err = "Variable {} doesn't appear to be a valid new name." + sys.exit(err.format(var)) + + + print("Tests passed") + + +if __name__ == '__main__': + set_up() + test() + teardown()