
There were two problems with our gerrit upgrade config diff checking. The first is that we were comparing using command exit codes after pipeing diff to tee without setting pipefail. This meant that even if the diff failed we got an exit code of 0 from tee and everything passed happily. Second we were not checking our pre gerrit state diff state. If the old gerrit version also updated the config on disk we wouldn't get a diff after upgrading to the new version. I think that is ultimately what broke for us here because the 3.6 and 3.7 config diffs are empty, but they differ from what we initially write to disk. As for explaining why this might happen I can only assume some update to 3.6 made the changes we saw after we had deployed 3.6. As a result of checking things more thoroughly we need to update our config to remove any delta. This removes some extraneous quoting around gitweb config entries to do that. Change-Id: I9c7e73d5f64546fb57a21a249b29e2aca9229ac7
144 lines
4.8 KiB
YAML
144 lines
4.8 KiB
YAML
# Note this playbook is in the zuul/ dir because it is very test specific
|
|
# currently. We could potentially rewrite things so that this can be used
|
|
# in production but it isn't currently ready for that.
|
|
#
|
|
# In particular it bootstraps users and test changes assuming a test env.
|
|
|
|
- name: Ensure initial gerrit state without starting Gerrit
|
|
import_playbook: ../service-review.yaml
|
|
vars:
|
|
gerrit_container_image: docker.io/opendevorg/gerrit:3.6
|
|
gerrit_run_init: false
|
|
gerrit_run_reindex: false
|
|
gerrit_run_compose_up: false
|
|
|
|
- hosts: "review:!disabled"
|
|
name: "Record pre Gerrit deployment configuration state"
|
|
tasks:
|
|
# This allows us to check that our config file isn't modified by newer
|
|
# Gerrit versions.
|
|
- name: Backup config files pre deploy
|
|
block:
|
|
- name: Find .config files
|
|
find:
|
|
paths: /home/gerrit2/review_site/etc
|
|
patterns: '*.config'
|
|
register: _config_files_pre_deploy
|
|
|
|
- name: 'Backup config file'
|
|
copy:
|
|
src: '{{ item }}'
|
|
dest: '{{ item }}.pre-deploy'
|
|
remote_src: true
|
|
loop: "{{ _config_files_pre_deploy.files | map(attribute='path') | list }}"
|
|
|
|
- name: Start Gerrit on the old version of Gerrit
|
|
import_playbook: ../service-review.yaml
|
|
vars:
|
|
gerrit_container_image: docker.io/opendevorg/gerrit:3.6
|
|
|
|
- hosts: "review:!disabled"
|
|
name: "Wait for gerrit to be up and running"
|
|
tasks:
|
|
- name: Pause for a few seconds to give gerrit time to start
|
|
wait_for:
|
|
timeout: 30
|
|
|
|
- name: Bootstrap gerrit to be semi useable
|
|
# This is necessary to perform actions on the old side pre upgrade
|
|
import_playbook: ./bootstrap-test-review.yaml
|
|
|
|
- hosts: "review:!disabled"
|
|
name: "Prepare Gerrit for Upgrade"
|
|
tasks:
|
|
- name: Stop gerrit before we upgrade
|
|
shell:
|
|
cmd: docker-compose down
|
|
chdir: /etc/gerrit-compose/
|
|
|
|
# This allows us to check that our config file isn't modified by newer
|
|
# Gerrit versions.
|
|
- name: Backup config files pre upgrade
|
|
block:
|
|
- name: Find .config files
|
|
find:
|
|
paths: /home/gerrit2/review_site/etc
|
|
patterns: '*.config'
|
|
register: _config_files_pre_upgrade
|
|
|
|
- name: 'Backup config file'
|
|
copy:
|
|
src: '{{ item }}'
|
|
dest: '{{ item }}.pre-upgrade'
|
|
remote_src: true
|
|
loop: "{{ _config_files_pre_upgrade.files | map(attribute='path') | list }}"
|
|
|
|
# Record h2 cache files. We will use this to highlight any new caches
|
|
# under the new Gerrit version.
|
|
- name: Record Gerrit old cache files
|
|
find:
|
|
paths: /home/gerrit2/review_site/cache
|
|
patterns: '*.h2.db'
|
|
register: _old_cache_files
|
|
|
|
- name: Perform gerrit upgrade
|
|
import_playbook: ../service-review.yaml
|
|
vars:
|
|
gerrit_container_image: docker.io/opendevorg/gerrit:3.7
|
|
gerrit_run_init: true
|
|
# Gerrit 3.6 -> 3.7 upgrade requires an offline reindex
|
|
gerrit_run_reindex: true
|
|
|
|
- hosts: "review:!disabled"
|
|
name: "Post upgrade config check"
|
|
tasks:
|
|
- name: Diff config files pre deploy to pre upgrade
|
|
shell: |
|
|
set -o pipefail
|
|
diff -u {{ item }}.pre-deploy {{ item }}.pre-upgrade | tee {{ item }}.deploy.diff
|
|
args:
|
|
executable: /bin/bash
|
|
loop: "{{ _config_files_pre_upgrade.files | map(attribute='path') | list }}"
|
|
register: _diff_output_deploy
|
|
|
|
- name: Diff config files pre upgrade to post upgrade
|
|
shell: |
|
|
set -o pipefail
|
|
diff -u {{ item }}.pre-upgrade {{ item }} | tee {{ item }}.upgrade.diff
|
|
args:
|
|
executable: /bin/bash
|
|
loop: "{{ _config_files_pre_upgrade.files | map(attribute='path') | list }}"
|
|
register: _diff_output_upgrade
|
|
|
|
- name: Check the config diffs after deployment
|
|
fail:
|
|
msg: 'Difference detected in file {{ item.item }} '
|
|
when: item.rc != 0
|
|
loop: '{{ _diff_output_deploy.results }}'
|
|
|
|
- name: Check the config diffs after upgrade
|
|
fail:
|
|
msg: 'Difference detected in file {{ item.item }} '
|
|
when: item.rc != 0
|
|
loop: '{{ _diff_output_upgrade.results }}'
|
|
|
|
- name: Record Gerrit new cache files
|
|
find:
|
|
paths: /home/gerrit2/review_site/cache
|
|
patterns: '*.h2.db'
|
|
register: _new_cache_files
|
|
|
|
- name: Manipulate find data for caches
|
|
set_fact:
|
|
_old_cache_paths: "{{ _old_cache_files.files | map(attribute='path') | list }}"
|
|
_new_cache_paths: "{{ _new_cache_files.files | map(attribute='path') | list }}"
|
|
|
|
- name: Find delta between cache listings
|
|
set_fact:
|
|
_gerrit_cache_difference: "{{ _old_cache_paths | symmetric_difference(_new_cache_paths) }}"
|
|
|
|
- name: Check for new cache files
|
|
debug:
|
|
msg: "The new Gerrit version produces new on disk caches: {{ _gerrit_cache_difference }}"
|
|
when: _gerrit_cache_difference | length > 0
|