Merge "Revert "Revert Patch of puppet-manifest-apply.sh""
This commit is contained in:
Zuul
Gerrit Code Review
commit
94f80ab772
@ -10,7 +10,7 @@ override_dh_usrlocal:
|
||||
echo "Do Nothing"
|
||||
|
||||
override_dh_install:
|
||||
$(MAKE) install hiera_v5=true \
|
||||
$(MAKE) install hiera_v5=true ignore_puppet_warnings=true \
|
||||
BINDIR=$(CURDIR)/debian/tmp/usr/local/bin \
|
||||
CONFIGDIR=$(CURDIR)/debian/tmp/etc/puppet \
|
||||
MODULEDIR=$(CURDIR)/debian/tmp/usr/share/puppet/modules
|
||||
|
||||
@ -9,7 +9,11 @@ CONFIGDIR ?= /etc/puppet
|
||||
MODULEDIR ?= /usr/share/puppet/modules
|
||||
|
||||
install:
|
||||
ifdef ignore_puppet_warnings
|
||||
install -m 755 -D bin/puppet-manifest-apply-ignore-warnings.sh $(BINDIR)/puppet-manifest-apply.sh
|
||||
else
|
||||
install -m 755 -D bin/puppet-manifest-apply.sh $(BINDIR)/puppet-manifest-apply.sh
|
||||
endif
|
||||
install -m 755 -D bin/apply_network_config.sh $(BINDIR)/apply_network_config.sh
|
||||
install -m 755 -D bin/k8s_wait_for_endpoints_health.py $(BINDIR)/k8s_wait_for_endpoints_health.py
|
||||
install -m 755 -D bin/kube-wait-control-plane-terminated.sh $(BINDIR)/kube-wait-control-plane-terminated.sh
|
||||
|
||||
200
puppet-manifests/src/bin/puppet-manifest-apply-ignore-warnings.sh
Executable file
200
puppet-manifests/src/bin/puppet-manifest-apply-ignore-warnings.sh
Executable file
@ -0,0 +1,200 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Grab a lock before doing anything else
|
||||
LOCKFILE=/var/lock/.puppet.applyscript.lock
|
||||
LOCK_FD=200
|
||||
LOCK_TIMEOUT=60
|
||||
|
||||
eval "exec ${LOCK_FD}>$LOCKFILE"
|
||||
|
||||
while :; do
|
||||
flock -w $LOCK_TIMEOUT $LOCK_FD && break
|
||||
logger -t $0 "Failed to get lock for puppet applyscript after $LOCK_TIMEOUT seconds. Trying again"
|
||||
sleep 1
|
||||
done
|
||||
|
||||
HIERADATA=$1
|
||||
HOST=$2
|
||||
# subfunctions is a list of subfunctions, separated by comma
|
||||
SUBFUNCTIONS=$3
|
||||
IFS=, read PERSONALITY SUBFUNCTION LL <<< $SUBFUNCTIONS
|
||||
if [ "${SUBFUNCTION}" = "worker" ]; then
|
||||
MANIFEST="aio"
|
||||
else
|
||||
PERSONALITY=${SUBFUNCTIONS}
|
||||
MANIFEST=${PERSONALITY}
|
||||
fi
|
||||
MANIFEST=${4:-$MANIFEST}
|
||||
RUNTIMEDATA=$5
|
||||
|
||||
|
||||
logger -t $0 "puppet-manifest-apply ${HIERADATA} ${HOST} ${SUBFUNCTIONS} ${MANIFEST} ${RUNTIMEDATA}"
|
||||
|
||||
|
||||
PUPPET_MODULES_PATH=/usr/share/puppet/modules:/usr/share/openstack-puppet/modules
|
||||
PUPPET_MANIFEST=/etc/puppet/manifests/${MANIFEST}.pp
|
||||
PUPPET_TMP=/tmp/puppet
|
||||
FILEBUCKET_PATH=/var/lib/puppet/clientbucket
|
||||
|
||||
# Setup log directory and file
|
||||
DATETIME=$(date -u +"%Y-%m-%d-%H-%M-%S")
|
||||
LOGDIR="/var/log/puppet/${DATETIME}_${MANIFEST}"
|
||||
LOGFILE=${LOGDIR}/puppet.log
|
||||
|
||||
mkdir -p ${LOGDIR}
|
||||
chmod 700 ${LOGDIR}
|
||||
rm -f /var/log/puppet/latest
|
||||
ln -s ${LOGDIR} /var/log/puppet/latest
|
||||
|
||||
touch ${LOGFILE}
|
||||
chmod 600 ${LOGFILE}
|
||||
|
||||
|
||||
# Remove old log directories
|
||||
declare -i NUM_DIRS=`ls -d1 /var/log/puppet/[0-9]* 2>/dev/null | wc -l`
|
||||
declare -i MAX_DIRS=50
|
||||
if [ ${NUM_DIRS} -gt ${MAX_DIRS} ]; then
|
||||
let -i RMDIRS=${NUM_DIRS}-${MAX_DIRS}
|
||||
ls -d1 /var/log/puppet/[0-9]* | head -${RMDIRS} | xargs --no-run-if-empty rm -rf
|
||||
fi
|
||||
|
||||
|
||||
# Setup staging area and hiera data configuration
|
||||
# (must match hierarchy defined in hiera.yaml)
|
||||
rm -rf ${PUPPET_TMP}
|
||||
mkdir -p ${PUPPET_TMP}/hieradata
|
||||
cp /etc/puppet/hieradata/global.yaml ${PUPPET_TMP}/hieradata/global.yaml
|
||||
|
||||
if [ "${MANIFEST}" = 'aio' ]; then
|
||||
cat /etc/puppet/hieradata/controller.yaml /etc/puppet/hieradata/worker.yaml > ${PUPPET_TMP}/hieradata/personality.yaml
|
||||
else
|
||||
cp /etc/puppet/hieradata/${PERSONALITY}.yaml ${PUPPET_TMP}/hieradata/personality.yaml
|
||||
fi
|
||||
|
||||
# When the worker node is first booted and goes online, sysinv-agent reports
|
||||
# host CPU inventory which triggers the first runtime manifest apply that updates
|
||||
# the grub. At this time, copying the host file failed due to a timing issue that
|
||||
# has not yet been fully understood. Subsequent retries worked.
|
||||
#
|
||||
# When back to back runtime manifests (e.g. as on https modify certificate
|
||||
# install) are issued, copying of the hieradata file may fail. Suspect this is due
|
||||
# to potential update of hieradata on the controller while the file is being
|
||||
# copied. Check rsync status and retry if needed.
|
||||
|
||||
declare -i MAX_RETRIES=3
|
||||
|
||||
HIERA_HOST=()
|
||||
if [ "${MANIFEST}" == 'ansible_bootstrap' ]; then
|
||||
HIERA_SYS=("${HIERADATA}/secure_static.yaml" "${HIERADATA}/static.yaml")
|
||||
elif [ "${MANIFEST}" == 'restore' ]; then
|
||||
HIERA_SYS=("${HIERADATA}/secure_static.yaml" "${HIERADATA}/static.yaml" "${HIERADATA}/system.yaml" "${HIERADATA}/secure_system.yaml")
|
||||
elif [ "${MANIFEST}" == 'upgrade' ]; then
|
||||
HIERA_SYS=("${HIERADATA}/secure_static.yaml" "${HIERADATA}/static.yaml" "${HIERADATA}/system.yaml")
|
||||
else
|
||||
HIERA_SYS=("${HIERADATA}/secure_static.yaml" "${HIERADATA}/static.yaml" "${HIERADATA}/system.yaml" "${HIERADATA}/secure_system.yaml")
|
||||
HIERA_HOST=("${HIERADATA}/${HOST}.yaml")
|
||||
fi
|
||||
|
||||
if [ -n "${RUNTIMEDATA}" ]; then
|
||||
HIERA_RUNTIME=("${RUNTIMEDATA}")
|
||||
else
|
||||
HIERA_RUNTIME=()
|
||||
fi
|
||||
|
||||
DELAY_SECS=15
|
||||
for (( iter=1; iter<=$MAX_RETRIES; iter++ )); do
|
||||
if [ ${#HIERA_HOST[@]} -ne 0 ]; then
|
||||
rsync -c "${HIERA_HOST[@]}" ${PUPPET_TMP}/hieradata/host.yaml
|
||||
if [ $? -eq 0 ]; then
|
||||
HIERA_HOST=()
|
||||
fi
|
||||
fi
|
||||
|
||||
rsync -c "${HIERA_SYS[@]}" ${PUPPET_TMP}/hieradata
|
||||
if [ $? -eq 0 ]; then
|
||||
HIERA_SYS=()
|
||||
fi
|
||||
|
||||
if [ ${#HIERA_RUNTIME[@]} -ne 0 ]; then
|
||||
rsync -c "${HIERA_RUNTIME[@]}" ${PUPPET_TMP}/hieradata/runtime.yaml
|
||||
if [ $? -eq 0 ]; then
|
||||
HIERA_RUNTIME=()
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ ${#HIERA_HOST[@]} -eq 0 ] && [ ${#HIERA_SYS[@]} -eq 0 ] && [ ${#HIERA_SYS[@]} -eq 0 ]; then
|
||||
break
|
||||
fi
|
||||
|
||||
logger -t $0 "Failed to copy ${HIERA_HOST[*]}:${HIERA_SYS[*]}:${HIERA_FILES_RUNTIME[*]} iteration: ${iter}."
|
||||
if [ ${iter} -eq ${MAX_RETRIES} ]; then
|
||||
echo "[FAILED]"
|
||||
echo "Exiting, failed to rsync hieradata"
|
||||
logger -t $0 "Exiting, failed to rsync hieradata"
|
||||
exit 1
|
||||
else
|
||||
logger -t $0 "Failed to rsync hieradata iteration: ${iter}. Retry in ${DELAY_SECS} seconds"
|
||||
sleep ${DELAY_SECS}
|
||||
fi
|
||||
done
|
||||
|
||||
|
||||
# Exit function to save logs from initial apply
|
||||
function finish {
|
||||
local SAVEDLOGS=/var/log/puppet/first_apply.tgz
|
||||
if [ ! -f ${SAVEDLOGS} ]; then
|
||||
# Save the logs
|
||||
tar czf ${SAVEDLOGS} ${LOGDIR} 2>/dev/null
|
||||
chmod 600 ${SAVEDLOGS}
|
||||
fi
|
||||
|
||||
# To avoid the ever growing contents of filebucket which may trigger inode
|
||||
# issues, clean up its contents after every apply.
|
||||
if [ -d ${FILEBUCKET_PATH} ]; then
|
||||
rm -fr ${FILEBUCKET_PATH}/*
|
||||
fi
|
||||
}
|
||||
trap finish EXIT
|
||||
|
||||
|
||||
# Set Keystone endpoint type to internal to prevent SSL cert failures during config
|
||||
export OS_ENDPOINT_TYPE=internalURL
|
||||
export CINDER_ENDPOINT_TYPE=internalURL
|
||||
# Suppress stdlib deprecation warnings until all puppet modules can be updated
|
||||
export STDLIB_LOG_DEPRECATIONS=false
|
||||
|
||||
mask_passwd() {
|
||||
sed -i -r 's/(bootstrap-password) (\"[^\"]*\"|'\''[^'"'"']*'"'"'|[^ ]*)/\1 xxxxxx/g;
|
||||
s/(set_keystone_user_option\.sh admin) (\"[^\"]*\"|'\''[^'"'"']*'"'"'|[^ ]*)/\1 xxxxxx/g' \
|
||||
${LOGFILE}
|
||||
}
|
||||
|
||||
echo "Applying puppet ${MANIFEST} manifest..."
|
||||
|
||||
# puppet wants to write to current directory. Need to move current directory to a writable place.
|
||||
# it is not possible to fail cd command, but tox doesn't like it without an exit.
|
||||
cd $PUPPET_TMP || exit
|
||||
flock /var/run/puppet.lock \
|
||||
puppet apply --trace --modulepath ${PUPPET_MODULES_PATH} ${PUPPET_MANIFEST} \
|
||||
< /dev/null 2>&1 | awk ' { system("date -u +%FT%T.%3N | tr \"\n\" \" \""); print $0; fflush(); } ' > ${LOGFILE}
|
||||
|
||||
rc=$?
|
||||
mask_passwd
|
||||
|
||||
if [ ${rc} -ne 0 ]; then
|
||||
echo "[FAILED]"
|
||||
echo "See ${LOGFILE} for details"
|
||||
exit 1
|
||||
else
|
||||
#Directly patched for: sed -i 's@Warning|@MMAAAAAAAAAASKED|@g' /usr/local/bin/puppet-manifest-apply.sh
|
||||
#TODO: Revert patch when all puppet warnings are resolved on Debian
|
||||
grep -qE '^(.......)?MMAAAAAAAAAASKED|^....-..-..T..:..:..([.]...)?(.......)?.MMAAAAAAAAAASKED|^(.......)?Error|^....-..-..T..:..:..([.]...)?(.......)?.Error' ${LOGFILE}
|
||||
if [ $? -eq 0 ]; then
|
||||
echo "[WARNING]"
|
||||
echo "Warnings found. See ${LOGFILE} for details"
|
||||
exit 1
|
||||
fi
|
||||
echo "[DONE]"
|
||||
fi
|
||||
|
||||
exit 0
|
||||
@ -169,49 +169,6 @@ mask_passwd() {
|
||||
${LOGFILE}
|
||||
}
|
||||
|
||||
virtual_env_whitelist() {
|
||||
# For virtual environments it's possible to ignore Warnings in the manifest execution using a whitelist.
|
||||
# To do so, add the text description to the whitelist following the example:
|
||||
# warnings_whitelist=("Text of warning number 1" "Text of warning number 2" "Text of warning number 3")
|
||||
|
||||
warnings_whitelist=("Could not retrieve fact ipaddress")
|
||||
|
||||
# Check for errors before continuing with warnings whitelist check.
|
||||
if grep -qE '^(.......)?Error|^....-..-..T..:..:..([.]...)?(.......)?.Error' "${LOGFILE}"; then
|
||||
echo "[WARNING]"
|
||||
echo "Errors found, not proceeding whit warnings whitelist check. See ${LOGFILE} for details"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Extract Warnings from the manifest execution log:
|
||||
WARNINGS_LOG_FILE="/tmp/${DATETIME}_${MANIFEST}_puppet_warnings.log"
|
||||
grep -E '^(.......)?Warning|^....-..-..T..:..:..([.]...)?(.......)?.Warning' "${LOGFILE}" > "${WARNINGS_LOG_FILE}"
|
||||
|
||||
# Count of Warnings present in the manifest execution log:
|
||||
warnings_in_log_count=$(wc -l <"${WARNINGS_LOG_FILE}")
|
||||
|
||||
# Count of Warnings present in the manifest execution log that matches with the whitelist:
|
||||
warnings_matches_count=0
|
||||
|
||||
for warning in "${warnings_whitelist[@]}"; do
|
||||
if grep -q "${warning}" "${WARNINGS_LOG_FILE}"; then
|
||||
warnings_matches_count=$((warnings_matches_count+=1))
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ ${warnings_matches_count} -ne 0 ]] && [[ ${warnings_matches_count} -eq ${warnings_in_log_count} ]]; then
|
||||
# All warnings in the logs are in the whitelist, ignore warnings
|
||||
echo "The warnings that appear in the manifest execution are the same of the whitelist;"\
|
||||
"Ignoring warnings..."
|
||||
else
|
||||
# Warnings that appear in the log file are different from warnings on whitelist
|
||||
echo "[WARNING] The warnings that appear in the manifest execution are different of the whitelist..."
|
||||
echo "Warnings found. See ${LOGFILE} or ${WARNINGS_LOG_FILE} for details"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
}
|
||||
|
||||
echo "Applying puppet ${MANIFEST} manifest..."
|
||||
|
||||
# puppet wants to write to current directory. Need to move current directory to a writable place.
|
||||
@ -231,15 +188,9 @@ if [ ${rc} -ne 0 ]; then
|
||||
else
|
||||
grep -qE '^(.......)?Warning|^....-..-..T..:..:..([.]...)?(.......)?.Warning|^(.......)?Error|^....-..-..T..:..:..([.]...)?(.......)?.Error' ${LOGFILE}
|
||||
if [ $? -eq 0 ]; then
|
||||
# If in a virtual environment, check if the Warnings are present in the whitelist:
|
||||
is_virtual=$(/usr/bin/facter is_virtual)
|
||||
if ${is_virtual} ; then
|
||||
virtual_env_whitelist
|
||||
else
|
||||
echo "[WARNING]"
|
||||
echo "Warnings found. See ${LOGFILE} for details"
|
||||
exit 1
|
||||
fi
|
||||
echo "[WARNING]"
|
||||
echo "Warnings found. See ${LOGFILE} for details"
|
||||
exit 1
|
||||
fi
|
||||
echo "[DONE]"
|
||||
fi
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user