starlingx/metal
Code Issues Proposed changes

Add a ostree pull retry if the first pull attempt fails

Browse Source 
This update adds 2 retries to the ostree pull operation for
system node or remote subcloud installs in attempt to handle
transient http server connection failure during the pull
operation.

Test Plan:

PASS: Verify successful system node install with retry code present.
PASS: Verify successful handling of a single transient connection loss
PASS: Verify handling of a persistent connection loss ; all tries fail
PASS: Verify successful subcloud install with retry code present.
PASS: Verify subcloud ostree pull retry handling ; 1,2 and max failures

Closes-Bug: 2068651
Change-Id: I538d5a179188966882c494731111d36b89c03415
Signed-off-by: Eric MacDonald <eric.macdonald@windriver.com>
This commit is contained in:
Eric MacDonald 2024-06-05 18:32:53 +00:00
parent 42253f7b25
commit 55aa34dde7
2 changed files with 49 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
kickstart/files/kickstart.cfg
View File

@ -2042,11 +2042,23 @@ else
fi
ilog "Executing: ostree --repo=${repo} pull --depth=-1 --mirror ${instbr}:${instbr}"
ostree --repo=${repo} pull --depth=-1 --mirror ${instbr}:${instbr}
rc=$?
if [ $rc -ne 0 ]; then
report_failure_with_msg "ostree pull failed, rc=$rc"
fi
MAX_TRIES=3
RETRY_WAIT=10
for try in 1 2 3
do
ostree --repo=${repo} pull --depth=-1 --mirror ${instbr}:${instbr}
rc=$?
if [ ${rc} -ne 0 ]; then
if [ ${try} -lt ${MAX_TRIES} ] ; then
wlog "ostree pull failed on try ${try} of ${MAX_TRIES}, rc=${rc} ; retry in ${RETRY_WAIT} seconds ..."
sleep ${RETRY_WAIT}
else
report_failure_with_msg "ostree pull failed, rc=${rc} ; max tries ${try} of ${MAX_TRIES}"
fi
else
break
fi
done
umount ${PHYS_SYSROOT}

43
kickstart/files/miniboot.cfg
View File

@ -2178,11 +2178,23 @@ else
fi
ilog "Executing: ostree --repo=${repo} pull --depth=-1 --mirror ${instbr}:${instbr}"
ostree --repo=${repo} pull --depth=-1 --mirror ${instbr}:${instbr}
rc=$?
if [ $rc -ne 0 ]; then
report_failure_with_msg "ostree pull failed, rc=$rc"
fi
MAX_TRIES=3
RETRY_WAIT=10
for try in 1 2 3
do
ostree --repo=${repo} pull --depth=-1 --mirror ${instbr}:${instbr}
rc=$?
if [ ${rc} -ne 0 ]; then
if [ ${try} -lt ${MAX_TRIES} ] ; then
wlog "ostree pull failed on try ${try} of ${MAX_TRIES}, rc=${rc} ; retry in ${RETRY_WAIT} seconds ..."
sleep ${RETRY_WAIT}
else
report_failure_with_msg "ostree pull failed, rc=${rc} ; max tries ${try} of ${MAX_TRIES}"
fi
else
break
fi
done
if [ -n "${remote_insturl}" ]; then
# In this case, we've initialized our ostree repo from local disk
@ -2204,12 +2216,21 @@ else
ostree config --repo=${repo} set "remote \"${instbr}\"".gpg-verify false
fi
ilog "Executing ostree pull from ${remote_insturl}:"
ilog "ostree --repo=${repo} pull --depth=-1 --mirror ${instbr}:${instbr}"
ostree --repo=${repo} pull --depth=-1 --mirror ${instbr}:${instbr}
rc=$?
if [ $rc -ne 0 ]; then
report_failure_with_msg "ostree pull failed, rc=$rc"
fi
for try in 1 2 3
do
ostree --repo=${repo} pull --depth=-1 --mirror ${instbr}:${instbr}
rc=$?
if [ ${rc} -ne 0 ]; then
if [ ${try} -lt ${MAX_TRIES} ] ; then
wlog "ostree pull failed on try ${try} of ${MAX_TRIES}, rc=${rc} ; retry in ${RETRY_WAIT} seconds ..."
sleep ${RETRY_WAIT}
else
report_failure_with_msg "ostree pull failed, rc=${rc} ; max tries ${try} of ${MAX_TRIES}"
fi
else
break
fi
done
fi
ilog "ostree log for ${repo}, branch=${instbr}:"
ostree --repo=${repo} log ${instbr}