From e44ff4ecfe42724191aa0873ca2d5aeba5b507b9 Mon Sep 17 00:00:00 2001 From: Lucas Borges Date: Mon, 11 Sep 2023 15:11:16 +0000 Subject: [PATCH] Revert "Revert Patch of puppet-manifest-apply.sh" This reverts commit a1784deca9d30848f05d2ca53e66cf832d54b0da. Reason for revert: The white list created to ignore only for VM, however, the ignored warning was also seen in real server. This needs to be more extensively tested in different types of server. Story: 2010757 Task: 48644 Change-Id: I979b4269d0e8f68b5ea0c8471b14e666a437730d Signed-off-by: Lucas Borges --- puppet-manifests/debian/deb_folder/rules | 2 +- puppet-manifests/src/Makefile | 4 + .../puppet-manifest-apply-ignore-warnings.sh | 200 ++++++++++++++++++ .../src/bin/puppet-manifest-apply.sh | 55 +---- 4 files changed, 208 insertions(+), 53 deletions(-) create mode 100755 puppet-manifests/src/bin/puppet-manifest-apply-ignore-warnings.sh diff --git a/puppet-manifests/debian/deb_folder/rules b/puppet-manifests/debian/deb_folder/rules index 663428b2d..ec2bca6ca 100755 --- a/puppet-manifests/debian/deb_folder/rules +++ b/puppet-manifests/debian/deb_folder/rules @@ -10,7 +10,7 @@ override_dh_usrlocal: echo "Do Nothing" override_dh_install: - $(MAKE) install hiera_v5=true \ + $(MAKE) install hiera_v5=true ignore_puppet_warnings=true \ BINDIR=$(CURDIR)/debian/tmp/usr/local/bin \ CONFIGDIR=$(CURDIR)/debian/tmp/etc/puppet \ MODULEDIR=$(CURDIR)/debian/tmp/usr/share/puppet/modules diff --git a/puppet-manifests/src/Makefile b/puppet-manifests/src/Makefile index 5eb469124..37d01ea31 100644 --- a/puppet-manifests/src/Makefile +++ b/puppet-manifests/src/Makefile @@ -9,7 +9,11 @@ CONFIGDIR ?= /etc/puppet MODULEDIR ?= /usr/share/puppet/modules install: +ifdef ignore_puppet_warnings + install -m 755 -D bin/puppet-manifest-apply-ignore-warnings.sh $(BINDIR)/puppet-manifest-apply.sh +else install -m 755 -D bin/puppet-manifest-apply.sh $(BINDIR)/puppet-manifest-apply.sh +endif install -m 755 -D bin/apply_network_config.sh $(BINDIR)/apply_network_config.sh install -m 755 -D bin/k8s_wait_for_endpoints_health.py $(BINDIR)/k8s_wait_for_endpoints_health.py install -m 755 -D bin/kube-wait-control-plane-terminated.sh $(BINDIR)/kube-wait-control-plane-terminated.sh diff --git a/puppet-manifests/src/bin/puppet-manifest-apply-ignore-warnings.sh b/puppet-manifests/src/bin/puppet-manifest-apply-ignore-warnings.sh new file mode 100755 index 000000000..d00693117 --- /dev/null +++ b/puppet-manifests/src/bin/puppet-manifest-apply-ignore-warnings.sh @@ -0,0 +1,200 @@ +#!/bin/bash + +# Grab a lock before doing anything else +LOCKFILE=/var/lock/.puppet.applyscript.lock +LOCK_FD=200 +LOCK_TIMEOUT=60 + +eval "exec ${LOCK_FD}>$LOCKFILE" + +while :; do + flock -w $LOCK_TIMEOUT $LOCK_FD && break + logger -t $0 "Failed to get lock for puppet applyscript after $LOCK_TIMEOUT seconds. Trying again" + sleep 1 +done + +HIERADATA=$1 +HOST=$2 +# subfunctions is a list of subfunctions, separated by comma +SUBFUNCTIONS=$3 +IFS=, read PERSONALITY SUBFUNCTION LL <<< $SUBFUNCTIONS +if [ "${SUBFUNCTION}" = "worker" ]; then + MANIFEST="aio" +else + PERSONALITY=${SUBFUNCTIONS} + MANIFEST=${PERSONALITY} +fi +MANIFEST=${4:-$MANIFEST} +RUNTIMEDATA=$5 + + +logger -t $0 "puppet-manifest-apply ${HIERADATA} ${HOST} ${SUBFUNCTIONS} ${MANIFEST} ${RUNTIMEDATA}" + + +PUPPET_MODULES_PATH=/usr/share/puppet/modules:/usr/share/openstack-puppet/modules +PUPPET_MANIFEST=/etc/puppet/manifests/${MANIFEST}.pp +PUPPET_TMP=/tmp/puppet +FILEBUCKET_PATH=/var/lib/puppet/clientbucket + +# Setup log directory and file +DATETIME=$(date -u +"%Y-%m-%d-%H-%M-%S") +LOGDIR="/var/log/puppet/${DATETIME}_${MANIFEST}" +LOGFILE=${LOGDIR}/puppet.log + +mkdir -p ${LOGDIR} +chmod 700 ${LOGDIR} +rm -f /var/log/puppet/latest +ln -s ${LOGDIR} /var/log/puppet/latest + +touch ${LOGFILE} +chmod 600 ${LOGFILE} + + +# Remove old log directories +declare -i NUM_DIRS=`ls -d1 /var/log/puppet/[0-9]* 2>/dev/null | wc -l` +declare -i MAX_DIRS=50 +if [ ${NUM_DIRS} -gt ${MAX_DIRS} ]; then + let -i RMDIRS=${NUM_DIRS}-${MAX_DIRS} + ls -d1 /var/log/puppet/[0-9]* | head -${RMDIRS} | xargs --no-run-if-empty rm -rf +fi + + +# Setup staging area and hiera data configuration +# (must match hierarchy defined in hiera.yaml) +rm -rf ${PUPPET_TMP} +mkdir -p ${PUPPET_TMP}/hieradata +cp /etc/puppet/hieradata/global.yaml ${PUPPET_TMP}/hieradata/global.yaml + +if [ "${MANIFEST}" = 'aio' ]; then + cat /etc/puppet/hieradata/controller.yaml /etc/puppet/hieradata/worker.yaml > ${PUPPET_TMP}/hieradata/personality.yaml +else + cp /etc/puppet/hieradata/${PERSONALITY}.yaml ${PUPPET_TMP}/hieradata/personality.yaml +fi + +# When the worker node is first booted and goes online, sysinv-agent reports +# host CPU inventory which triggers the first runtime manifest apply that updates +# the grub. At this time, copying the host file failed due to a timing issue that +# has not yet been fully understood. Subsequent retries worked. +# +# When back to back runtime manifests (e.g. as on https modify certificate +# install) are issued, copying of the hieradata file may fail. Suspect this is due +# to potential update of hieradata on the controller while the file is being +# copied. Check rsync status and retry if needed. + +declare -i MAX_RETRIES=3 + +HIERA_HOST=() +if [ "${MANIFEST}" == 'ansible_bootstrap' ]; then + HIERA_SYS=("${HIERADATA}/secure_static.yaml" "${HIERADATA}/static.yaml") +elif [ "${MANIFEST}" == 'restore' ]; then + HIERA_SYS=("${HIERADATA}/secure_static.yaml" "${HIERADATA}/static.yaml" "${HIERADATA}/system.yaml" "${HIERADATA}/secure_system.yaml") +elif [ "${MANIFEST}" == 'upgrade' ]; then + HIERA_SYS=("${HIERADATA}/secure_static.yaml" "${HIERADATA}/static.yaml" "${HIERADATA}/system.yaml") +else + HIERA_SYS=("${HIERADATA}/secure_static.yaml" "${HIERADATA}/static.yaml" "${HIERADATA}/system.yaml" "${HIERADATA}/secure_system.yaml") + HIERA_HOST=("${HIERADATA}/${HOST}.yaml") +fi + +if [ -n "${RUNTIMEDATA}" ]; then + HIERA_RUNTIME=("${RUNTIMEDATA}") +else + HIERA_RUNTIME=() +fi + +DELAY_SECS=15 +for (( iter=1; iter<=$MAX_RETRIES; iter++ )); do + if [ ${#HIERA_HOST[@]} -ne 0 ]; then + rsync -c "${HIERA_HOST[@]}" ${PUPPET_TMP}/hieradata/host.yaml + if [ $? -eq 0 ]; then + HIERA_HOST=() + fi + fi + + rsync -c "${HIERA_SYS[@]}" ${PUPPET_TMP}/hieradata + if [ $? -eq 0 ]; then + HIERA_SYS=() + fi + + if [ ${#HIERA_RUNTIME[@]} -ne 0 ]; then + rsync -c "${HIERA_RUNTIME[@]}" ${PUPPET_TMP}/hieradata/runtime.yaml + if [ $? -eq 0 ]; then + HIERA_RUNTIME=() + fi + fi + + if [ ${#HIERA_HOST[@]} -eq 0 ] && [ ${#HIERA_SYS[@]} -eq 0 ] && [ ${#HIERA_SYS[@]} -eq 0 ]; then + break + fi + + logger -t $0 "Failed to copy ${HIERA_HOST[*]}:${HIERA_SYS[*]}:${HIERA_FILES_RUNTIME[*]} iteration: ${iter}." + if [ ${iter} -eq ${MAX_RETRIES} ]; then + echo "[FAILED]" + echo "Exiting, failed to rsync hieradata" + logger -t $0 "Exiting, failed to rsync hieradata" + exit 1 + else + logger -t $0 "Failed to rsync hieradata iteration: ${iter}. Retry in ${DELAY_SECS} seconds" + sleep ${DELAY_SECS} + fi +done + + +# Exit function to save logs from initial apply +function finish { + local SAVEDLOGS=/var/log/puppet/first_apply.tgz + if [ ! -f ${SAVEDLOGS} ]; then + # Save the logs + tar czf ${SAVEDLOGS} ${LOGDIR} 2>/dev/null + chmod 600 ${SAVEDLOGS} + fi + + # To avoid the ever growing contents of filebucket which may trigger inode + # issues, clean up its contents after every apply. + if [ -d ${FILEBUCKET_PATH} ]; then + rm -fr ${FILEBUCKET_PATH}/* + fi +} +trap finish EXIT + + +# Set Keystone endpoint type to internal to prevent SSL cert failures during config +export OS_ENDPOINT_TYPE=internalURL +export CINDER_ENDPOINT_TYPE=internalURL +# Suppress stdlib deprecation warnings until all puppet modules can be updated +export STDLIB_LOG_DEPRECATIONS=false + +mask_passwd() { + sed -i -r 's/(bootstrap-password) (\"[^\"]*\"|'\''[^'"'"']*'"'"'|[^ ]*)/\1 xxxxxx/g; + s/(set_keystone_user_option\.sh admin) (\"[^\"]*\"|'\''[^'"'"']*'"'"'|[^ ]*)/\1 xxxxxx/g' \ + ${LOGFILE} +} + +echo "Applying puppet ${MANIFEST} manifest..." + +# puppet wants to write to current directory. Need to move current directory to a writable place. +# it is not possible to fail cd command, but tox doesn't like it without an exit. +cd $PUPPET_TMP || exit +flock /var/run/puppet.lock \ + puppet apply --trace --modulepath ${PUPPET_MODULES_PATH} ${PUPPET_MANIFEST} \ + < /dev/null 2>&1 | awk ' { system("date -u +%FT%T.%3N | tr \"\n\" \" \""); print $0; fflush(); } ' > ${LOGFILE} + +rc=$? +mask_passwd + +if [ ${rc} -ne 0 ]; then + echo "[FAILED]" + echo "See ${LOGFILE} for details" + exit 1 +else + #Directly patched for: sed -i 's@Warning|@MMAAAAAAAAAASKED|@g' /usr/local/bin/puppet-manifest-apply.sh + #TODO: Revert patch when all puppet warnings are resolved on Debian + grep -qE '^(.......)?MMAAAAAAAAAASKED|^....-..-..T..:..:..([.]...)?(.......)?.MMAAAAAAAAAASKED|^(.......)?Error|^....-..-..T..:..:..([.]...)?(.......)?.Error' ${LOGFILE} + if [ $? -eq 0 ]; then + echo "[WARNING]" + echo "Warnings found. See ${LOGFILE} for details" + exit 1 + fi + echo "[DONE]" +fi + +exit 0 diff --git a/puppet-manifests/src/bin/puppet-manifest-apply.sh b/puppet-manifests/src/bin/puppet-manifest-apply.sh index 3f9c5b945..7a4d9bd64 100755 --- a/puppet-manifests/src/bin/puppet-manifest-apply.sh +++ b/puppet-manifests/src/bin/puppet-manifest-apply.sh @@ -169,49 +169,6 @@ mask_passwd() { ${LOGFILE} } -virtual_env_whitelist() { - # For virtual environments it's possible to ignore Warnings in the manifest execution using a whitelist. - # To do so, add the text description to the whitelist following the example: - # warnings_whitelist=("Text of warning number 1" "Text of warning number 2" "Text of warning number 3") - - warnings_whitelist=("Could not retrieve fact ipaddress") - - # Check for errors before continuing with warnings whitelist check. - if grep -qE '^(.......)?Error|^....-..-..T..:..:..([.]...)?(.......)?.Error' "${LOGFILE}"; then - echo "[WARNING]" - echo "Errors found, not proceeding whit warnings whitelist check. See ${LOGFILE} for details" - exit 1 - fi - - # Extract Warnings from the manifest execution log: - WARNINGS_LOG_FILE="/tmp/${DATETIME}_${MANIFEST}_puppet_warnings.log" - grep -E '^(.......)?Warning|^....-..-..T..:..:..([.]...)?(.......)?.Warning' "${LOGFILE}" > "${WARNINGS_LOG_FILE}" - - # Count of Warnings present in the manifest execution log: - warnings_in_log_count=$(wc -l <"${WARNINGS_LOG_FILE}") - - # Count of Warnings present in the manifest execution log that matches with the whitelist: - warnings_matches_count=0 - - for warning in "${warnings_whitelist[@]}"; do - if grep -q "${warning}" "${WARNINGS_LOG_FILE}"; then - warnings_matches_count=$((warnings_matches_count+=1)) - fi - done - - if [[ ${warnings_matches_count} -ne 0 ]] && [[ ${warnings_matches_count} -eq ${warnings_in_log_count} ]]; then - # All warnings in the logs are in the whitelist, ignore warnings - echo "The warnings that appear in the manifest execution are the same of the whitelist;"\ - "Ignoring warnings..." - else - # Warnings that appear in the log file are different from warnings on whitelist - echo "[WARNING] The warnings that appear in the manifest execution are different of the whitelist..." - echo "Warnings found. See ${LOGFILE} or ${WARNINGS_LOG_FILE} for details" - exit 1 - fi - -} - echo "Applying puppet ${MANIFEST} manifest..." # puppet wants to write to current directory. Need to move current directory to a writable place. @@ -231,15 +188,9 @@ if [ ${rc} -ne 0 ]; then else grep -qE '^(.......)?Warning|^....-..-..T..:..:..([.]...)?(.......)?.Warning|^(.......)?Error|^....-..-..T..:..:..([.]...)?(.......)?.Error' ${LOGFILE} if [ $? -eq 0 ]; then - # If in a virtual environment, check if the Warnings are present in the whitelist: - is_virtual=$(/usr/bin/facter is_virtual) - if ${is_virtual} ; then - virtual_env_whitelist - else - echo "[WARNING]" - echo "Warnings found. See ${LOGFILE} for details" - exit 1 - fi + echo "[WARNING]" + echo "Warnings found. See ${LOGFILE} for details" + exit 1 fi echo "[DONE]" fi