Revert "Revert Patch of puppet-manifest-apply.sh"
This reverts commit a1784deca9d30848f05d2ca53e66cf832d54b0da. Reason for revert: The white list created to ignore only for VM, however, the ignored warning was also seen in real server. This needs to be more extensively tested in different types of server. Story: 2010757 Task: 48644 Change-Id: I979b4269d0e8f68b5ea0c8471b14e666a437730d Signed-off-by: Lucas Borges <lucas.borges@windriver.com>
This commit is contained in:
Lucas Borges
parent
a1784deca9
commit
e44ff4ecfe
@ -10,7 +10,7 @@ override_dh_usrlocal:
|
||||
echo "Do Nothing"
|
||||
|
||||
override_dh_install:
|
||||
$(MAKE) install hiera_v5=true \
|
||||
$(MAKE) install hiera_v5=true ignore_puppet_warnings=true \
|
||||
BINDIR=$(CURDIR)/debian/tmp/usr/local/bin \
|
||||
CONFIGDIR=$(CURDIR)/debian/tmp/etc/puppet \
|
||||
MODULEDIR=$(CURDIR)/debian/tmp/usr/share/puppet/modules
|
||||
|
||||
@ -9,7 +9,11 @@ CONFIGDIR ?= /etc/puppet
|
||||
MODULEDIR ?= /usr/share/puppet/modules
|
||||
|
||||
install:
|
||||
ifdef ignore_puppet_warnings
|
||||
install -m 755 -D bin/puppet-manifest-apply-ignore-warnings.sh $(BINDIR)/puppet-manifest-apply.sh
|
||||
else
|
||||
install -m 755 -D bin/puppet-manifest-apply.sh $(BINDIR)/puppet-manifest-apply.sh
|
||||
endif
|
||||
install -m 755 -D bin/apply_network_config.sh $(BINDIR)/apply_network_config.sh
|
||||
install -m 755 -D bin/k8s_wait_for_endpoints_health.py $(BINDIR)/k8s_wait_for_endpoints_health.py
|
||||
install -m 755 -D bin/kube-wait-control-plane-terminated.sh $(BINDIR)/kube-wait-control-plane-terminated.sh
|
||||
|
||||
200
puppet-manifests/src/bin/puppet-manifest-apply-ignore-warnings.sh
Executable file
200
puppet-manifests/src/bin/puppet-manifest-apply-ignore-warnings.sh
Executable file
@ -0,0 +1,200 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Grab a lock before doing anything else
|
||||
LOCKFILE=/var/lock/.puppet.applyscript.lock
|
||||
LOCK_FD=200
|
||||
LOCK_TIMEOUT=60
|
||||
|
||||
eval "exec ${LOCK_FD}>$LOCKFILE"
|
||||
|
||||
while :; do
|
||||
flock -w $LOCK_TIMEOUT $LOCK_FD && break
|
||||
logger -t $0 "Failed to get lock for puppet applyscript after $LOCK_TIMEOUT seconds. Trying again"
|
||||
sleep 1
|
||||
done
|
||||
|
||||
HIERADATA=$1
|
||||
HOST=$2
|
||||
# subfunctions is a list of subfunctions, separated by comma
|
||||
SUBFUNCTIONS=$3
|
||||
IFS=, read PERSONALITY SUBFUNCTION LL <<< $SUBFUNCTIONS
|
||||
if [ "${SUBFUNCTION}" = "worker" ]; then
|
||||
MANIFEST="aio"
|
||||
else
|
||||
PERSONALITY=${SUBFUNCTIONS}
|
||||
MANIFEST=${PERSONALITY}
|
||||
fi
|
||||
MANIFEST=${4:-$MANIFEST}
|
||||
RUNTIMEDATA=$5
|
||||
|
||||
|
||||
logger -t $0 "puppet-manifest-apply ${HIERADATA} ${HOST} ${SUBFUNCTIONS} ${MANIFEST} ${RUNTIMEDATA}"
|
||||
|
||||
|
||||
PUPPET_MODULES_PATH=/usr/share/puppet/modules:/usr/share/openstack-puppet/modules
|
||||
PUPPET_MANIFEST=/etc/puppet/manifests/${MANIFEST}.pp
|
||||
PUPPET_TMP=/tmp/puppet
|
||||
FILEBUCKET_PATH=/var/lib/puppet/clientbucket
|
||||
|
||||
# Setup log directory and file
|
||||
DATETIME=$(date -u +"%Y-%m-%d-%H-%M-%S")
|
||||
LOGDIR="/var/log/puppet/${DATETIME}_${MANIFEST}"
|
||||
LOGFILE=${LOGDIR}/puppet.log
|
||||
|
||||
mkdir -p ${LOGDIR}
|
||||
chmod 700 ${LOGDIR}
|
||||
rm -f /var/log/puppet/latest
|
||||
ln -s ${LOGDIR} /var/log/puppet/latest
|
||||
|
||||
touch ${LOGFILE}
|
||||
chmod 600 ${LOGFILE}
|
||||
|
||||
|
||||
# Remove old log directories
|
||||
declare -i NUM_DIRS=`ls -d1 /var/log/puppet/[0-9]* 2>/dev/null | wc -l`
|
||||
declare -i MAX_DIRS=50
|
||||
if [ ${NUM_DIRS} -gt ${MAX_DIRS} ]; then
|
||||
let -i RMDIRS=${NUM_DIRS}-${MAX_DIRS}
|
||||
ls -d1 /var/log/puppet/[0-9]* | head -${RMDIRS} | xargs --no-run-if-empty rm -rf
|
||||
fi
|
||||
|
||||
|
||||
# Setup staging area and hiera data configuration
|
||||
# (must match hierarchy defined in hiera.yaml)
|
||||
rm -rf ${PUPPET_TMP}
|
||||
mkdir -p ${PUPPET_TMP}/hieradata
|
||||
cp /etc/puppet/hieradata/global.yaml ${PUPPET_TMP}/hieradata/global.yaml
|
||||
|
||||
if [ "${MANIFEST}" = 'aio' ]; then
|
||||
cat /etc/puppet/hieradata/controller.yaml /etc/puppet/hieradata/worker.yaml > ${PUPPET_TMP}/hieradata/personality.yaml
|
||||
else
|
||||
cp /etc/puppet/hieradata/${PERSONALITY}.yaml ${PUPPET_TMP}/hieradata/personality.yaml
|
||||
fi
|
||||
|
||||
# When the worker node is first booted and goes online, sysinv-agent reports
|
||||
# host CPU inventory which triggers the first runtime manifest apply that updates
|
||||
# the grub. At this time, copying the host file failed due to a timing issue that
|
||||
# has not yet been fully understood. Subsequent retries worked.
|
||||
#
|
||||
# When back to back runtime manifests (e.g. as on https modify certificate
|
||||
# install) are issued, copying of the hieradata file may fail. Suspect this is due
|
||||
# to potential update of hieradata on the controller while the file is being
|
||||
# copied. Check rsync status and retry if needed.
|
||||
|
||||
declare -i MAX_RETRIES=3
|
||||
|
||||
HIERA_HOST=()
|
||||
if [ "${MANIFEST}" == 'ansible_bootstrap' ]; then
|
||||
HIERA_SYS=("${HIERADATA}/secure_static.yaml" "${HIERADATA}/static.yaml")
|
||||
elif [ "${MANIFEST}" == 'restore' ]; then
|
||||
HIERA_SYS=("${HIERADATA}/secure_static.yaml" "${HIERADATA}/static.yaml" "${HIERADATA}/system.yaml" "${HIERADATA}/secure_system.yaml")
|
||||
elif [ "${MANIFEST}" == 'upgrade' ]; then
|
||||
HIERA_SYS=("${HIERADATA}/secure_static.yaml" "${HIERADATA}/static.yaml" "${HIERADATA}/system.yaml")
|
||||
else
|
||||
HIERA_SYS=("${HIERADATA}/secure_static.yaml" "${HIERADATA}/static.yaml" "${HIERADATA}/system.yaml" "${HIERADATA}/secure_system.yaml")
|
||||
HIERA_HOST=("${HIERADATA}/${HOST}.yaml")
|
||||
fi
|
||||
|
||||
if [ -n "${RUNTIMEDATA}" ]; then
|
||||
HIERA_RUNTIME=("${RUNTIMEDATA}")
|
||||
else
|
||||
HIERA_RUNTIME=()
|
||||
fi
|
||||
|
||||
DELAY_SECS=15
|
||||
for (( iter=1; iter<=$MAX_RETRIES; iter++ )); do
|
||||
if [ ${#HIERA_HOST[@]} -ne 0 ]; then
|
||||
rsync -c "${HIERA_HOST[@]}" ${PUPPET_TMP}/hieradata/host.yaml
|
||||
if [ $? -eq 0 ]; then
|
||||
HIERA_HOST=()
|
||||
fi
|
||||
fi
|
||||
|
||||
rsync -c "${HIERA_SYS[@]}" ${PUPPET_TMP}/hieradata
|
||||
if [ $? -eq 0 ]; then
|
||||
HIERA_SYS=()
|
||||
fi
|
||||
|
||||
if [ ${#HIERA_RUNTIME[@]} -ne 0 ]; then
|
||||
rsync -c "${HIERA_RUNTIME[@]}" ${PUPPET_TMP}/hieradata/runtime.yaml
|
||||
if [ $? -eq 0 ]; then
|
||||
HIERA_RUNTIME=()
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ ${#HIERA_HOST[@]} -eq 0 ] && [ ${#HIERA_SYS[@]} -eq 0 ] && [ ${#HIERA_SYS[@]} -eq 0 ]; then
|
||||
break
|
||||
fi
|
||||
|
||||
logger -t $0 "Failed to copy ${HIERA_HOST[*]}:${HIERA_SYS[*]}:${HIERA_FILES_RUNTIME[*]} iteration: ${iter}."
|
||||
if [ ${iter} -eq ${MAX_RETRIES} ]; then
|
||||
echo "[FAILED]"
|
||||
echo "Exiting, failed to rsync hieradata"
|
||||
logger -t $0 "Exiting, failed to rsync hieradata"
|
||||
exit 1
|
||||
else
|
||||
logger -t $0 "Failed to rsync hieradata iteration: ${iter}. Retry in ${DELAY_SECS} seconds"
|
||||
sleep ${DELAY_SECS}
|
||||
fi
|
||||
done
|
||||
|
||||
|
||||
# Exit function to save logs from initial apply
|
||||
function finish {
|
||||
local SAVEDLOGS=/var/log/puppet/first_apply.tgz
|
||||
if [ ! -f ${SAVEDLOGS} ]; then
|
||||
# Save the logs
|
||||
tar czf ${SAVEDLOGS} ${LOGDIR} 2>/dev/null
|
||||
chmod 600 ${SAVEDLOGS}
|
||||
fi
|
||||
|
||||
# To avoid the ever growing contents of filebucket which may trigger inode
|
||||
# issues, clean up its contents after every apply.
|
||||
if [ -d ${FILEBUCKET_PATH} ]; then
|
||||
rm -fr ${FILEBUCKET_PATH}/*
|
||||
fi
|
||||
}
|
||||
trap finish EXIT
|
||||
|
||||
|
||||
# Set Keystone endpoint type to internal to prevent SSL cert failures during config
|
||||
export OS_ENDPOINT_TYPE=internalURL
|
||||
export CINDER_ENDPOINT_TYPE=internalURL
|
||||
# Suppress stdlib deprecation warnings until all puppet modules can be updated
|
||||
export STDLIB_LOG_DEPRECATIONS=false
|
||||
|
||||
mask_passwd() {
|
||||
sed -i -r 's/(bootstrap-password) (\"[^\"]*\"|'\''[^'"'"']*'"'"'|[^ ]*)/\1 xxxxxx/g;
|
||||
s/(set_keystone_user_option\.sh admin) (\"[^\"]*\"|'\''[^'"'"']*'"'"'|[^ ]*)/\1 xxxxxx/g' \
|
||||
${LOGFILE}
|
||||
}
|
||||
|
||||
echo "Applying puppet ${MANIFEST} manifest..."
|
||||
|
||||
# puppet wants to write to current directory. Need to move current directory to a writable place.
|
||||
# it is not possible to fail cd command, but tox doesn't like it without an exit.
|
||||
cd $PUPPET_TMP || exit
|
||||
flock /var/run/puppet.lock \
|
||||
puppet apply --trace --modulepath ${PUPPET_MODULES_PATH} ${PUPPET_MANIFEST} \
|
||||
< /dev/null 2>&1 | awk ' { system("date -u +%FT%T.%3N | tr \"\n\" \" \""); print $0; fflush(); } ' > ${LOGFILE}
|
||||
|
||||
rc=$?
|
||||
mask_passwd
|
||||
|
||||
if [ ${rc} -ne 0 ]; then
|
||||
echo "[FAILED]"
|
||||
echo "See ${LOGFILE} for details"
|
||||
exit 1
|
||||
else
|
||||
#Directly patched for: sed -i 's@Warning|@MMAAAAAAAAAASKED|@g' /usr/local/bin/puppet-manifest-apply.sh
|
||||
#TODO: Revert patch when all puppet warnings are resolved on Debian
|
||||
grep -qE '^(.......)?MMAAAAAAAAAASKED|^....-..-..T..:..:..([.]...)?(.......)?.MMAAAAAAAAAASKED|^(.......)?Error|^....-..-..T..:..:..([.]...)?(.......)?.Error' ${LOGFILE}
|
||||
if [ $? -eq 0 ]; then
|
||||
echo "[WARNING]"
|
||||
echo "Warnings found. See ${LOGFILE} for details"
|
||||
exit 1
|
||||
fi
|
||||
echo "[DONE]"
|
||||
fi
|
||||
|
||||
exit 0
|
||||
@ -169,49 +169,6 @@ mask_passwd() {
|
||||
${LOGFILE}
|
||||
}
|
||||
|
||||
virtual_env_whitelist() {
|
||||
# For virtual environments it's possible to ignore Warnings in the manifest execution using a whitelist.
|
||||
# To do so, add the text description to the whitelist following the example:
|
||||
# warnings_whitelist=("Text of warning number 1" "Text of warning number 2" "Text of warning number 3")
|
||||
|
||||
warnings_whitelist=("Could not retrieve fact ipaddress")
|
||||
|
||||
# Check for errors before continuing with warnings whitelist check.
|
||||
if grep -qE '^(.......)?Error|^....-..-..T..:..:..([.]...)?(.......)?.Error' "${LOGFILE}"; then
|
||||
echo "[WARNING]"
|
||||
echo "Errors found, not proceeding whit warnings whitelist check. See ${LOGFILE} for details"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Extract Warnings from the manifest execution log:
|
||||
WARNINGS_LOG_FILE="/tmp/${DATETIME}_${MANIFEST}_puppet_warnings.log"
|
||||
grep -E '^(.......)?Warning|^....-..-..T..:..:..([.]...)?(.......)?.Warning' "${LOGFILE}" > "${WARNINGS_LOG_FILE}"
|
||||
|
||||
# Count of Warnings present in the manifest execution log:
|
||||
warnings_in_log_count=$(wc -l <"${WARNINGS_LOG_FILE}")
|
||||
|
||||
# Count of Warnings present in the manifest execution log that matches with the whitelist:
|
||||
warnings_matches_count=0
|
||||
|
||||
for warning in "${warnings_whitelist[@]}"; do
|
||||
if grep -q "${warning}" "${WARNINGS_LOG_FILE}"; then
|
||||
warnings_matches_count=$((warnings_matches_count+=1))
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ ${warnings_matches_count} -ne 0 ]] && [[ ${warnings_matches_count} -eq ${warnings_in_log_count} ]]; then
|
||||
# All warnings in the logs are in the whitelist, ignore warnings
|
||||
echo "The warnings that appear in the manifest execution are the same of the whitelist;"\
|
||||
"Ignoring warnings..."
|
||||
else
|
||||
# Warnings that appear in the log file are different from warnings on whitelist
|
||||
echo "[WARNING] The warnings that appear in the manifest execution are different of the whitelist..."
|
||||
echo "Warnings found. See ${LOGFILE} or ${WARNINGS_LOG_FILE} for details"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
}
|
||||
|
||||
echo "Applying puppet ${MANIFEST} manifest..."
|
||||
|
||||
# puppet wants to write to current directory. Need to move current directory to a writable place.
|
||||
@ -231,15 +188,9 @@ if [ ${rc} -ne 0 ]; then
|
||||
else
|
||||
grep -qE '^(.......)?Warning|^....-..-..T..:..:..([.]...)?(.......)?.Warning|^(.......)?Error|^....-..-..T..:..:..([.]...)?(.......)?.Error' ${LOGFILE}
|
||||
if [ $? -eq 0 ]; then
|
||||
# If in a virtual environment, check if the Warnings are present in the whitelist:
|
||||
is_virtual=$(/usr/bin/facter is_virtual)
|
||||
if ${is_virtual} ; then
|
||||
virtual_env_whitelist
|
||||
else
|
||||
echo "[WARNING]"
|
||||
echo "Warnings found. See ${LOGFILE} for details"
|
||||
exit 1
|
||||
fi
|
||||
echo "[WARNING]"
|
||||
echo "Warnings found. See ${LOGFILE} for details"
|
||||
exit 1
|
||||
fi
|
||||
echo "[DONE]"
|
||||
fi
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user