Test upgrades in CI

This patch adds two new jobs:

* kolla-ansible-centos-source-upgrade
* kolla-ansible-ubuntu-source-upgrade

These jobs first deploy a control plane using the previous release of
Kolla Ansible, then upgrade to the current release.

Because we can't change the branch of the git repository on the Zuul
executor, we change the branch of the kolla-ansible repository on the
primary node to the branch of the previous release, in this case
stable/rocky. A new remote-template role has been added that supports
generating templates using a remote template source, to generate config
files using the previous kolla-ansible branch.

If the change being tested depends on a kolla change for the current
branch, then we build images. Rather than using the current
kolla-ansible version to tag the images, we now tag them with
change_<gerrit change ID>. This is because the version of kolla-ansible
will change from the previous release to the current one as we upgrade
the system.

Finally, it should be noted that the 'previous_release' variable in the
Zuul config needs to be updated with each release, since this sets the
release of kolla-ansible that is installed initially.

Depends-On: https://review.openstack.org/645089/
Depends-On: https://review.openstack.org/644250/
Depends-On: https://review.openstack.org/645816/
Depends-On: https://review.openstack.org/645840/
Change-Id: If301e0affcd55360fefe3b105f023ae5c47b0853

tests/roles/remote-template/defaults/main.yml

@@ -0,0 +1,12 @@
+---
+# Path to template file.
+remote_template_src:
+
+# Path to destination.
+remote_template_dest:
+
+# Whether to use 'become'.
+remote_template_become: false
+
+# Path on localhost to store a copy of the template.
+remote_template_temp_path: "{{ zuul.executor.work_root }}/{{ remote_template_src | basename }}"

tests/roles/remote-template/tasks/main.yml

@@ -0,0 +1,22 @@
+---
+# Ensure the parent directory exists.
+- name: "ensure {{ remote_template_dest | dirname }} exists"
+  file:
+    path: "{{ remote_template_dest | dirname }}"
+    state: "directory"
+    mode: 0777
+  become: "{{ remote_template_become }}"
+
+# Template sources must be on localhost, so first fetch the remote template
+# file.
+- name: "fetch remote template {{ remote_template_src | basename }}"
+  fetch:
+    src: "{{ remote_template_src }}"
+    dest: "{{ remote_template_temp_path }}"
+    flat: true
+
+- name: "template {{ remote_template_src | basename }}"
+  template:
+    src: "{{ remote_template_temp_path }}"
+    dest: "{{ remote_template_dest }}"
+  become: "{{ remote_template_become }}"

tests/run.yml

@@ -1,11 +1,10 @@
 ---
 - hosts: all
   vars:
-    kolla_ansible_src_dir: "src/{{ zuul.project.canonical_hostname }}/openstack/kolla-ansible"
+    kolla_ansible_local_src_dir: "{{ zuul.executor.work_root }}/src/{{ zuul.project.canonical_hostname }}/openstack/kolla-ansible"
-    kolla_ansible_full_src_dir: "{{ zuul.executor.work_root }}/{{ kolla_ansible_src_dir }}"
   tasks:
     - name: Prepare disks for Ceph or LVM
-      script: "{{ kolla_ansible_full_src_dir }}/tests/setup_disks.sh {{ disk_type }}"
+      script: "setup_disks.sh {{ disk_type }}"
       when: scenario in ["ceph", "cinder-lvm"]
       become: true
       vars:
@@ -16,23 +15,10 @@
   vars:
     kolla_inventory_path: "/etc/kolla/inventory"
     logs_dir: "/tmp/logs"
-    kolla_ansible_src_dir: "src/{{ zuul.project.canonical_hostname }}/openstack/kolla-ansible"
+    kolla_ansible_src_dir: "{{ ansible_env.PWD }}/src/{{ zuul.project.canonical_hostname }}/openstack/kolla-ansible"
-    kolla_ansible_full_src_dir: "{{ zuul.executor.work_root }}/{{ kolla_ansible_src_dir }}"
+    kolla_ansible_local_src_dir: "{{ zuul.executor.work_root }}/src/{{ zuul.project.canonical_hostname }}/openstack/kolla-ansible"
     need_build_image: false
   tasks:
-    - name: ensure /etc/kolla exists
-      file:
-        path: "/etc/kolla"
-        state: "directory"
-        mode: 0777
-      become: true
-
-    - name: copy default ansible kolla-ansible inventory
-      template:
-        src: "{{ kolla_ansible_full_src_dir }}/tests/templates/inventory.j2"
-        dest: "{{ kolla_inventory_path }}"
-      delegate_to: "primary"
-
     # FIXME: in multi node env, api_interface may be different on each node.
     - name: detect api_interface_name variable
       vars:
@@ -45,77 +31,78 @@
         - hostvars[inventory_hostname][ansible_interface_name]['ipv4'] is defined
         - hostvars[inventory_hostname][ansible_interface_name]['ipv4']['address'] == api_interface_address
       with_items: "{{ ansible_interfaces }}"
-      delegate_to: "primary"
 
     - name: detect whether need build images
       set_fact:
         need_build_image: true
       when:
         - item.project.short_name == "kolla"
+        - item.branch == zuul.branch
       with_items: "{{ zuul['items'] }}"
 
-    - name: generate global.yml file
+    # NOTE(mgoddard): This only affects the remote copy of the repo, not the
-      template:
+    # one on the executor.
-        src: "{{ kolla_ansible_full_src_dir }}/tests/templates/globals-default.j2"
+    - name: checkout the previous kolla-ansible branch
-        dest: /etc/kolla/globals.yml
+      command:
-      delegate_to: "primary"
+        cmd: "git checkout stable/{{ previous_release | lower }}"
+        chdir: "{{ kolla_ansible_src_dir }}"
+      when: scenario == "upgrade"
 
-    - name: ensure nova conf overrides dir exists
+    - name: ensure /etc/kolla exists
       file:
-        path: "/etc/kolla/config/nova"
+        path: "/etc/kolla"
-        state: "directory"
-        mode: 0777
-      when: scenario != "bifrost"
-      become: true
-      delegate_to: "primary"
-
-    - name: generate nova config overrides
-      template:
-        src: "{{ kolla_ansible_full_src_dir }}/tests/templates/nova-compute-overrides.j2"
-        dest: /etc/kolla/config/nova/nova-compute.conf
-      when: scenario != "bifrost"
-      delegate_to: "primary"
-
-    - name: ensure bifrost conf overrides dir exists
-      file:
-        path: "/etc/kolla/config/bifrost"
-        state: "directory"
-        mode: 0777
-      when: scenario == "bifrost"
-      become: true
-      delegate_to: "primary"
-
-    - name: generate bifrost DIB config overrides
-      template:
-        src: "{{ kolla_ansible_full_src_dir }}/tests/templates/bifrost-dib-overrides.j2"
-        dest: /etc/kolla/config/bifrost/dib.yml
-      when: scenario == "bifrost"
-      delegate_to: "primary"
-
-    - name: ensure /etc/docker exists
-      file:
-        path: "/etc/docker"
         state: "directory"
         mode: 0777
       become: true
 
-    - name: create deamon.json for nodepool cache
+    # Use the initial repo to generate config files. For upgrade jobs, this
+    # repo is only available on the remote node, so use the remote-template
+    # role.
+    - name: generate configuration files
+      include_role:
+        role: remote-template
       vars:
+        is_previous_release: "{{ scenario == 'upgrade' }}"
         infra_dockerhub_mirror: "http://{{ zuul_site_mirror_fqdn }}:8082/"
-      template:
+        # Role variables.
-        src: "{{ kolla_ansible_full_src_dir }}/tests/templates/docker_daemon.json.j2"
+        remote_template_src: "{{ kolla_ansible_src_dir }}/{{ item.src }}"
+        remote_template_dest: "{{ item.dest }}"
+        remote_template_become: "{{ item.become | default(false) }}"
+      with_items:
+        # Docker daemon.json
+        - src: "tests/templates/docker_daemon.json.j2"
           dest: "/etc/docker/daemon.json"
           become: true
+        # Ansible inventory
+        - src: "tests/templates/inventory.j2"
+          dest: "{{ kolla_inventory_path }}"
+        # globals.yml
+        - src: "tests/templates/globals-default.j2"
+          dest: /etc/kolla/globals.yml
+        # nova-compute.conf
+        - src: "tests/templates/nova-compute-overrides.j2"
+          dest: /etc/kolla/config/nova/nova-compute.conf
+          when: scenario != "bifrost"
+        # ceph.conf
+        - src: "tests/templates/ceph-overrides.j2"
+          dest: /etc/kolla/config/ceph.conf
+          when: scenario == "ceph"
+        # bifrost/dib.yml
+        - src: "tests/templates/bifrost-dib-overrides.j2"
+          dest: /etc/kolla/config/bifrost/dib.yml
+          when: scenario == "bifrost"
+      when: item.when | default(true)
 
     - name: install kolla-ansible requirements
       pip:
-        requirements: "{{ ansible_env.HOME }}/{{ kolla_ansible_src_dir }}/requirements.txt"
+        requirements: "{{ kolla_ansible_src_dir }}/requirements.txt"
       become: true
 
     - name: copy passwords.yml file
       copy:
-        src: "{{ kolla_ansible_full_src_dir }}/etc/kolla/passwords.yml"
+        src: "{{ kolla_ansible_src_dir }}/etc/kolla/passwords.yml"
         dest: /etc/kolla/passwords.yml
+        remote_src: true
 
     - name: generate passwords
       shell: "{{ kolla_ansible_src_dir }}/tools/generate_passwords.py"
@@ -141,30 +128,133 @@
         - all
       delegate_to: "{{ item }}"
 
-    - name: generate ceph config overrides
+    # NOTE(mgoddard): We are using the script module here and later to ensure
-      template:
+    # we use the local copy of these scripts, rather than the one on the remote
-        src: "{{ kolla_ansible_full_src_dir }}/tests/templates/ceph-overrides.j2"
+    # host, which could be checked out to a previous release (in an upgrade
-        dest: /etc/kolla/config/ceph.conf
+    # job).
-      when: scenario == "ceph"
-      delegate_to: "primary"
-
     - name: Run setup_gate.sh script
-      shell:
+      script:
-        cmd: tools/setup_gate.sh
+        cmd: ../tools/setup_gate.sh
         executable: /bin/bash
         chdir: "{{ kolla_ansible_src_dir }}"
       environment:
         BASE_DISTRO: "{{ base_distro }}"
         INSTALL_TYPE: "{{ install_type }}"
-        NODEPOOL_TARBALLS_MIRROR: "http://{{ zuul_site_mirror_fqdn }}:8080/tarballs"
         BUILD_IMAGE: "{{ need_build_image }}"
+        TAG: "change_{{ zuul.change }}"
         KOLLA_SRC_DIR: "{{ ansible_env.HOME }}/src/git.openstack.org/openstack/kolla"
         ACTION: "{{ scenario }}"
 
+    # At this point we have generated all necessary configuration, and are
+    # ready to deploy the control plane services. Control flow now depends on
+    # the scenario being exercised.
+
+    # Deploy control plane. For upgrade jobs this is the previous release.
     - block:
         - name: Run deploy.sh script
+          script:
+            cmd: deploy.sh
+            executable: /bin/bash
+            chdir: "{{ kolla_ansible_src_dir }}"
+          environment:
+            ACTION: "{{ scenario }}"
+
+        - name: Run test-openstack.sh script
+          script:
+            cmd: test-openstack.sh
+            executable: /bin/bash
+            chdir: "{{ kolla_ansible_src_dir }}"
+          environment:
+            ACTION: "{{ scenario }}"
+          when: scenario not in ['scenario_nfv']
+
+        - name: Run test-scenario-nfv.sh script
+          script:
+            cmd: test-scenario-nfv.sh
+            executable: /bin/bash
+            chdir: "{{ kolla_ansible_src_dir }}"
+          when: scenario == "scenario_nfv"
+
+        - name: Run reconfigure.sh script
+          script:
+            cmd: reconfigure.sh
+            executable: /bin/bash
+            chdir: "{{ kolla_ansible_src_dir }}"
+          environment:
+            ACTION: "{{ scenario }}"
+          when: scenario != "upgrade"
+      when: scenario != "bifrost"
+
+    # Upgrade: update config.
+    - block:
+        - name: Run check-failure.sh script
+          script:
+            cmd: check-failure.sh
+            executable: /bin/bash
+            chdir: "{{ kolla_ansible_src_dir }}"
+
+        # NOTE(mgoddard): This only affects the remote copy of the repo, not the
+        # one on the executor.
+        - name: checkout the current kolla-ansible branch
+          command:
+            cmd: "git checkout {{ zuul.branch }}"
+            chdir: "{{ kolla_ansible_src_dir }}"
+
+        # Use the new kolla-ansible repo to generate config files.
+        # This is the branch checked out on the executor, so we can use
+        # template directly rather than the remote-template role.
+        - name: Generate configuration files
+          template:
+            src: "{{ kolla_ansible_local_src_dir }}/{{ item.src }}"
+            dest: "{{ item.dest }}"
+          vars:
+            is_previous_release: false
+          with_items:
+            # Ansible inventory
+            - src: "tests/templates/inventory.j2"
+              dest: "{{ kolla_inventory_path }}"
+            # globals.yml
+            - src: "tests/templates/globals-default.j2"
+              dest: /etc/kolla/globals.yml
+            # nova-compute.conf
+            - src: "tests/templates/nova-compute-overrides.j2"
+              dest: /etc/kolla/config/nova/nova-compute.conf
+            # ceph.conf
+            - src: "tests/templates/ceph-overrides.j2"
+              dest: /etc/kolla/config/ceph.conf
+              when: "'ceph' in scenario"
+          when: item.when | default(true)
+
+        - name: upgrade kolla-ansible requirements
+          pip:
+            requirements: "{{ kolla_ansible_src_dir }}/requirements.txt"
+          become: true
+
+        # Update passwords.yml to include any new passwords added in this
+        # release.
+        - name: move passwords.yml to passwords.yml.old
+          command: mv /etc/kolla/passwords.yml /etc/kolla/passwords.yml.old
+
+        - name: copy passwords.yml file
+          copy:
+            src: "{{ kolla_ansible_src_dir }}/etc/kolla/passwords.yml"
+            dest: /etc/kolla/passwords.yml
+            remote_src: true
+
+        - name: generate new passwords
+          shell: "{{ kolla_ansible_src_dir }}/tools/generate_passwords.py"
+
+        - name: merge old and new passwords
+          shell: >-
+            {{ kolla_ansible_src_dir }}/tools/merge_passwords.py
+            --old /etc/kolla/passwords.yml.old
+            --new /etc/kolla/passwords.yml
+            --final /etc/kolla/passwords.yml
+
+        # Perform an upgrade to the in-development code.
+        - name: Run upgrade.sh script
           shell:
-            cmd: tests/deploy.sh
+            cmd: tests/upgrade.sh
             executable: /bin/bash
             chdir: "{{ kolla_ansible_src_dir }}"
           environment:
@@ -177,32 +267,9 @@
             chdir: "{{ kolla_ansible_src_dir }}"
           environment:
             ACTION: "{{ scenario }}"
-          when: scenario not in ['scenario_nfv']
+      when: scenario == "upgrade"
-
-        - name: Run test-scenario-nfv.sh script
-          shell:
-            cmd: tests/test-scenario-nfv.sh
-            executable: /bin/bash
-            chdir: "{{ kolla_ansible_src_dir }}"
-          when: scenario == "scenario_nfv"
-
-        - name: Run reconfigure.sh script
-          shell:
-            cmd: tests/reconfigure.sh
-            executable: /bin/bash
-            chdir: "{{ kolla_ansible_src_dir }}"
-          environment:
-            ACTION: "{{ scenario }}"
-
-        - name: Run upgrade.sh script
-          shell:
-            cmd: tests/upgrade.sh
-            executable: /bin/bash
-            chdir: "{{ kolla_ansible_src_dir }}"
-          environment:
-            ACTION: "{{ scenario }}"
-      when: scenario != "bifrost"
 
+    # Bifrost testing.
     - block:
         - name: Run deploy-bifrost.sh script
           shell:

tests/templates/globals-default.j2

@@ -11,19 +11,32 @@ keepalived_virtual_router_id: "{{ 250 | random(1) }}"
 {% if enable_core_openstack | bool %}
 kolla_internal_vip_address: "{{ api_interface_address if hostvars | length > 2 else '169.254.169.10' }}"
 enable_haproxy: "{{ 'no' if hostvars | length > 2 else 'yes' }}"
+# TODO(mgoddard): Remove this in the Train cycle when heat is enabled in the
+# initial deployment.
+enable_heat: "{{ scenario != 'upgrade' or previous_release != 'rocky' }}"
 neutron_external_interface: "fake_interface"
 openstack_logging_debug: "True"
 openstack_service_workers: "1"
 {% endif %}
 
-{% if need_build_image %}
+{% if need_build_image and not is_previous_release %}
 # NOTE(Jeffrey4l): use different a docker namespace name in case it pull image from hub.docker.io when deplying
 docker_namespace: "lokolla"
 docker_registry: "{{ api_interface_address }}:4000"
+openstack_release: "change_{{ zuul.change }}"
 {% else %}
 # use docker hub images
 docker_namespace: "kolla"
+{% if need_build_image and is_previous_release %}
+# NOTE(mgoddard): Ensure that the insecure local registry is trusted, since it
+# will be the source of images during the upgrade.
+docker_custom_option: "--insecure-registry {{ api_interface_address }}:4000"
+{% endif %}
+{% if not is_previous_release %}
 openstack_release: "{{ zuul.branch | basename }}"
+{% else %}
+openstack_release: "{{ previous_release }}"
+{% endif %}
 {% endif %}
 
 {% if scenario == "ceph" %}

tools/setup_gate.sh

@@ -8,19 +8,6 @@ export PYTHONUNBUFFERED=1
 
 GIT_PROJECT_DIR=$(mktemp -d)
 
-function clone_repos {
-    cat > /tmp/clonemap <<EOF
-clonemap:
- - name: openstack/kolla
-   dest: ${GIT_PROJECT_DIR}/kolla
- - name: openstack/requirements
-   dest: ${GIT_PROJECT_DIR}/requirements
-EOF
-    /usr/zuul-env/bin/zuul-cloner -m /tmp/clonemap --workspace "$(pwd)" \
-        --cache-dir /opt/git git://git.openstack.org \
-        openstack/kolla openstack/requirements
-}
-
 function setup_config {
     # Use Infra provided pypi.
     # Wheel package mirror may be not compatible. So do not enable it.
@@ -56,9 +43,6 @@ EOF
         GATE_IMAGES+=",tacker,mistral,redis,barbican"
     fi
 
-    # Use the kolla-ansible tag rather than the kolla tag, since this is what
-    # kolla-ansible will use by default.
-    TAG=$(python -c "import pbr.version; print(pbr.version.VersionInfo('kolla-ansible'))")
     cat <<EOF | sudo tee /etc/kolla/kolla-build.conf
 [DEFAULT]
 include_header = /etc/kolla/header
@@ -138,7 +122,6 @@ function prepare_images {
 }
 
 
-clone_repos
 setup_ansible
 setup_config
 setup_node

zuul.d/base.yaml

@@ -22,6 +22,13 @@
     roles:
       - zuul: openstack-infra/zuul-jobs
 
+- job:
+    name: kolla-ansible-upgrade-base
+    parent: kolla-ansible-base
+    vars:
+      previous_release: rocky
+      scenario: upgrade
+
 - job:
     name: kolla-ansible-bifrost-base
     parent: kolla-ansible-base

zuul.d/jobs.yaml

@@ -122,6 +122,22 @@
       install_type: source
       scenario: cinder-lvm
 
+- job:
+    name: kolla-ansible-centos-source-upgrade
+    parent: kolla-ansible-upgrade-base
+    nodeset: kolla-ansible-centos
+    vars:
+      base_distro: centos
+      install_type: source
+
+- job:
+    name: kolla-ansible-ubuntu-source-upgrade
+    parent: kolla-ansible-upgrade-base
+    nodeset: kolla-ansible-bionic
+    vars:
+      base_distro: ubuntu
+      install_type: source
+
 - job:
     name: kolla-ansible-bifrost-centos-source
     parent: kolla-ansible-bifrost-base

zuul.d/project.yaml

@@ -28,10 +28,14 @@
             files: ^ansible\/roles\/(zun|kuryr)\/.*
         - kolla-ansible-centos-source-scenario-nfv:
             files: ^ansible\/roles\/(barbican|heat|mistral|redis|tacker)\/.*
+        - kolla-ansible-centos-source-upgrade
+        - kolla-ansible-ubuntu-source-upgrade
     gate:
       jobs:
         - kolla-ansible-centos-source
         - kolla-ansible-ubuntu-source
+        - kolla-ansible-centos-source-upgrade
+        - kolla-ansible-ubuntu-source-upgrade
     periodic:
       jobs:
         - kolla-ansible-bifrost-centos-source
@@ -39,3 +43,5 @@
         - kolla-ansible-centos-source-scenario-nfv
         - kolla-ansible-ubuntu-source-cinder-lvm
         - kolla-ansible-centos-source-cinder-lvm
+        - kolla-ansible-centos-source-upgrade
+        - kolla-ansible-ubuntu-source-upgrade