c8af267282
We should run manage-projects when the manage-projects code on system-config changes. To do that, we need to run the system-config playbook so that the system-config content will be updated. In order to that properly, we need to run base, which means we need to run bridge. So we really want to do all three so that we're doing the correct dependent sequence. Subsequent changes can then just pick off single service playbooks and make them jobs that depend on base. Change-Id: I3560feff4309f6be21b72b30a7a6d61a60829e52
152 lines
4.3 KiB
Bash
Executable File
152 lines
4.3 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
# Copyright 2014 Hewlett-Packard Development Company, L.P.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
# If updating the puppet system-config repo or installing puppet modules
|
|
# fails then abort the puppet run as we will not get the results we
|
|
# expect.
|
|
set -e
|
|
|
|
SYSTEM_CONFIG=/opt/system-config
|
|
ANSIBLE_PLAYBOOKS=$SYSTEM_CONFIG/playbooks
|
|
|
|
# We only send stats if running under cron
|
|
UNDER_CRON=0
|
|
|
|
while getopts ":c" arg; do
|
|
case $arg in
|
|
c)
|
|
UNDER_CRON=1
|
|
;;
|
|
esac
|
|
done
|
|
|
|
GLOBAL_START_TIME=$(date '+%s')
|
|
|
|
# Send a timer stat to statsd
|
|
# send_timer metric [start_time]
|
|
# * uses timer metric bridge.ansible.run_all.<$1>
|
|
# * time will be taken from last call of start_timer, or $2 if set
|
|
function send_timer {
|
|
# Only send stats under cron conditions
|
|
if [[ ${UNDER_CRON} != 1 ]]; then
|
|
return
|
|
fi
|
|
|
|
local current
|
|
current=$(date '+%s')
|
|
local name
|
|
name=$1
|
|
local start
|
|
start=${2-$_START_TIME}
|
|
local elapsed_ms
|
|
elapsed_ms=$(( (current - start) * 1000 ))
|
|
|
|
echo "bridge.ansible.run_all.${name}:${elapsed_ms}|ms" | nc -w 1 -u graphite.opendev.org 8125
|
|
echo "End $name"
|
|
}
|
|
# See send_timer
|
|
function start_timer {
|
|
_START_TIME=$(date '+%s')
|
|
}
|
|
|
|
echo "--- begin run @ $(date -Is) ---"
|
|
|
|
# It's possible for connectivity to a server or manifest application to break
|
|
# for indeterminate periods of time, so the playbooks should be run without
|
|
# errexit
|
|
set +e
|
|
|
|
# Run all the ansible playbooks under timeout to prevent them from getting
|
|
# stuck if they are oomkilled
|
|
|
|
# Service playbooks
|
|
start_timer
|
|
timeout -k 2m 30m ansible-playbook -f 50 ${ANSIBLE_PLAYBOOKS}/service-bridge.yaml
|
|
send_timer service-bridge
|
|
|
|
start_timer
|
|
timeout -k 2m 30m ansible-playbook -f 50 ${ANSIBLE_PLAYBOOKS}/service-gitea-lb.yaml
|
|
send_timer gitea-lb
|
|
|
|
start_timer
|
|
timeout -k 2m 30m ansible-playbook -f 50 ${ANSIBLE_PLAYBOOKS}/service-letsencrypt.yaml
|
|
send_timer letsencrypt
|
|
|
|
start_timer
|
|
timeout -k 2m 30m ansible-playbook -f 50 ${ANSIBLE_PLAYBOOKS}/service-nameserver.yaml
|
|
send_timer nameserver
|
|
|
|
start_timer
|
|
timeout -k 2m 30m ansible-playbook -f 50 ${ANSIBLE_PLAYBOOKS}/service-nodepool.yaml
|
|
send_timer nodepool
|
|
|
|
start_timer
|
|
timeout -k 2m 30m ansible-playbook -f 50 ${ANSIBLE_PLAYBOOKS}/service-meetpad.yaml
|
|
send_timer meetpad
|
|
|
|
start_timer
|
|
timeout -k 2m 30m ansible-playbook -f 50 ${ANSIBLE_PLAYBOOKS}/service-mirror-update.yaml
|
|
send_timer mirror-update
|
|
|
|
start_timer
|
|
timeout -k 2m 30m ansible-playbook -f 50 ${ANSIBLE_PLAYBOOKS}/service-mirror.yaml
|
|
send_timer mirror
|
|
|
|
start_timer
|
|
timeout -k 2m 30m ansible-playbook -f 50 ${ANSIBLE_PLAYBOOKS}/service-static.yaml
|
|
send_timer static
|
|
|
|
start_timer
|
|
timeout -k 2m 30m ansible-playbook -f 50 ${ANSIBLE_PLAYBOOKS}/service-backup.yaml
|
|
send_timer backup
|
|
|
|
start_timer
|
|
timeout -k 2m 30m ansible-playbook -f 50 ${ANSIBLE_PLAYBOOKS}/service-registry.yaml
|
|
send_timer registry
|
|
|
|
start_timer
|
|
timeout -k 2m 30m ansible-playbook -f 50 ${ANSIBLE_PLAYBOOKS}/service-zuul.yaml
|
|
send_timer zuul
|
|
|
|
start_timer
|
|
timeout -k 2m 30m ansible-playbook -f 50 ${ANSIBLE_PLAYBOOKS}/service-review-dev.yaml
|
|
send_timer nodepool
|
|
|
|
start_timer
|
|
timeout -k 2m 30m ansible-playbook -f 50 ${ANSIBLE_PLAYBOOKS}/service-review.yaml
|
|
send_timer nodepool
|
|
|
|
start_timer
|
|
timeout -k 2m 30m ansible-playbook -f 50 ${ANSIBLE_PLAYBOOKS}/service-gitea.yaml
|
|
send_timer nodepool
|
|
|
|
# Run AFS changes separately so we can make sure to only do one at a time
|
|
# (turns out quorum is nice to have)
|
|
start_timer
|
|
timeout -k 2m 30m ansible-playbook -f 1 ${ANSIBLE_PLAYBOOKS}/remote_puppet_afs.yaml
|
|
send_timer afs
|
|
|
|
# Run everything else. We do not care if the other things worked
|
|
start_timer
|
|
timeout -k 2m 30m ansible-playbook -f 50 ${ANSIBLE_PLAYBOOKS}/remote_puppet_else.yaml
|
|
send_timer else
|
|
|
|
# Send the combined time for everything
|
|
send_timer total $GLOBAL_START_TIME
|
|
|
|
echo "--- end run @ $(date -Is) ---"
|
|
echo
|