93a8baca93
Overcloud upgrade jobs often fail when running on nodes with a 38G root disk. The overcloud upgrade itself generally works, but post-upgrade testing fails, for example when RabbitMQ gets in alarm state due to lack of free disk space, which prevents Nova from functioning correctly and makes it return "Unknown Error (HTTP 504)". Change-Id: I536a526340f6d7dea7a69372904b777292774eba
105 lines
3.9 KiB
YAML
105 lines
3.9 KiB
YAML
---
|
|
- hosts: primary
|
|
tasks:
|
|
|
|
# Install the previous release of Kayobe, and use it to deploy a control
|
|
# plane.
|
|
|
|
- block:
|
|
- name: Ensure kayobe is installed
|
|
shell:
|
|
cmd: dev/install.sh &> {{ logs_dir }}/ansible/install-pre-upgrade
|
|
chdir: "{{ previous_kayobe_src_dir }}"
|
|
|
|
- name: Configure the firewall
|
|
# We run kayobe commands to determine variable values. So this must
|
|
# be run after install.
|
|
shell:
|
|
cmd: dev/configure-firewall.sh
|
|
chdir: "{{ previous_kayobe_src_dir }}"
|
|
|
|
- name: Ensure overcloud is deployed
|
|
shell:
|
|
cmd: dev/overcloud-deploy.sh &> {{ logs_dir }}/ansible/overcloud-deploy-pre-upgrade
|
|
chdir: "{{ previous_kayobe_src_dir }}"
|
|
|
|
environment:
|
|
KAYOBE_CONFIG_SOURCE_PATH: "{{ previous_kayobe_config_src_dir }}"
|
|
|
|
# Update the Kayobe configuration to the current release.
|
|
|
|
- name: Ensure kolla config directory exists
|
|
file:
|
|
path: "{{ kayobe_config_src_dir }}/etc/kolla"
|
|
state: directory
|
|
|
|
- name: Copy across relevant kayobe-config files
|
|
copy:
|
|
src: "{{ previous_kayobe_config_src_dir }}/{{ item }}"
|
|
dest: "{{ kayobe_config_src_dir }}/{{ item }}"
|
|
remote_src: true
|
|
with_items:
|
|
- etc/kayobe/kolla/passwords.yml
|
|
- etc/kolla/admin-openrc.sh
|
|
- etc/kolla/public-openrc.sh
|
|
|
|
# NOTE(mgoddard): Use the name zz-overrides.yml to ensure this takes
|
|
# precedence over the standard config files.
|
|
- name: Ensure kayobe-config override config file exists
|
|
template:
|
|
src: overrides.yml.j2
|
|
dest: "{{ kayobe_config_src_dir }}/etc/kayobe/zz-overrides.yml"
|
|
|
|
- name: Ensure kolla-ansible globals.yml override config file exists
|
|
template:
|
|
src: globals.yml.j2
|
|
dest: "{{ kayobe_config_src_dir }}/etc/kayobe/kolla/globals.yml"
|
|
|
|
# Perform a smoke test against the previous release.
|
|
|
|
- block:
|
|
- name: Ensure test Tenks cluster is deployed
|
|
shell:
|
|
# Pass absolute source directory, since otherwise the `chdir` will
|
|
# cause this to fail.
|
|
cmd: dev/tenks-deploy-compute.sh '{{ tenks_src_dir }}' &> {{ logs_dir }}/ansible/tenks-deploy
|
|
chdir: "{{ kayobe_src_dir }}"
|
|
|
|
- name: Perform testing of the VMs in the overcloud prior to upgrade
|
|
shell:
|
|
cmd: dev/overcloud-test-vm.sh &> {{ logs_dir }}/ansible/overcloud-test-vm-pre-upgrade
|
|
chdir: "{{ previous_kayobe_src_dir }}"
|
|
|
|
- name: Perform testing of the baremetal machines in the overcloud prior to upgrade
|
|
shell:
|
|
cmd: dev/overcloud-test-baremetal.sh &> {{ logs_dir }}/ansible/overcloud-test-bm-pre-upgrade
|
|
chdir: "{{ previous_kayobe_src_dir }}"
|
|
|
|
# Upgrade Kayobe, and use it to perform an upgrade of the control plane.
|
|
|
|
- name: Ensure overcloud is upgraded
|
|
shell:
|
|
cmd: "{{ kayobe_src_dir }}/dev/overcloud-upgrade.sh &> {{ logs_dir }}/ansible/overcloud-upgrade"
|
|
|
|
# Remove unused Docker images to avoid reaching full disk
|
|
- name: Remove unused Docker images
|
|
shell:
|
|
cmd: "docker image prune --all --force"
|
|
become: true
|
|
|
|
# Perform a smoke test against the upgraded current release.
|
|
|
|
- name: Perform testing of VMs in the upgraded overcloud
|
|
shell:
|
|
cmd: dev/overcloud-test-vm.sh &> {{ logs_dir }}/ansible/overcloud-test-vm-post-upgrade
|
|
chdir: "{{ kayobe_src_dir }}"
|
|
|
|
- name: Perform testing of the baremetal machines in the upgraded overcloud
|
|
shell:
|
|
cmd: dev/overcloud-test-baremetal.sh &> {{ logs_dir }}/ansible/overcloud-test-bm-post-upgrade
|
|
chdir: "{{ kayobe_src_dir }}"
|
|
|
|
environment:
|
|
KAYOBE_CONFIG_SOURCE_PATH: "{{ kayobe_config_src_dir }}"
|
|
KAYOBE_SOURCE_PATH: "{{ kayobe_src_dir }}"
|