From 7715e8bef2f8af1a48a7740055bacf0abae3df51 Mon Sep 17 00:00:00 2001 From: Scott Little Date: Tue, 21 Jan 2025 22:06:09 -0500 Subject: [PATCH] Make builder nice to it's host StarlingX build tools consume excessive cpu and io which can starve non-build services, including kubernetes and calico. This results in timeouts, restarts, and general instability of calico and kubernetes which in turn can the build to fail. This affect is observed on the main StarlingX build server. The major build steps should employ 'nice' and 'ionice' to ensure that critical services on the build host are not starved for cpu or io. Depends-On: https://review.opendev.org/c/starlingx/tools/+/939799 Closes-bug: 2095512 Change-Id: Ib904fb28d403c958c10311d5940cc96c752728b4 Signed-off-by: Scott Little --- build-tools/build-docker-images/build-stx-base.sh | 4 ++++ build-tools/build-docker-images/build-stx-images.sh | 4 ++++ build-tools/build-wheels/build-wheel-tarball.sh | 5 +++++ build-tools/stx/build-image | 5 +++++ build-tools/stx/build-pkgs | 4 ++++ build-tools/stx/downloader | 6 ++++++ 6 files changed, 28 insertions(+) diff --git a/build-tools/build-docker-images/build-stx-base.sh b/build-tools/build-docker-images/build-stx-base.sh index 7acd62f5..55c5c102 100755 --- a/build-tools/build-docker-images/build-stx-base.sh +++ b/build-tools/build-docker-images/build-stx-base.sh @@ -18,6 +18,10 @@ if [ -z "${MY_WORKSPACE}" -o -z "${MY_REPO}" ]; then exit 1 fi +# make this process nice +renice -n 10 -p $$ +ionice -c 3 -p $$ + SUPPORTED_OS_ARGS=( 'debian' ) OS= # default: autodetect OS_VERSION= # default: lookup "ARG RELEASE" in Dockerfile diff --git a/build-tools/build-docker-images/build-stx-images.sh b/build-tools/build-docker-images/build-stx-images.sh index 8ea449cf..a24071a2 100755 --- a/build-tools/build-docker-images/build-stx-images.sh +++ b/build-tools/build-docker-images/build-stx-images.sh @@ -17,6 +17,10 @@ fi source ${MY_REPO}/build-tools/git-utils.sh +# make this process nice +renice -n 10 -p $$ +ionice -c 3 -p $$ + SUPPORTED_OS_ARGS=('centos' 'debian' 'distroless') OS= OS_LABEL= diff --git a/build-tools/build-wheels/build-wheel-tarball.sh b/build-tools/build-wheels/build-wheel-tarball.sh index 3dae69bc..86805505 100755 --- a/build-tools/build-wheels/build-wheel-tarball.sh +++ b/build-tools/build-wheels/build-wheel-tarball.sh @@ -17,6 +17,11 @@ if [ -z "${MY_WORKSPACE}" -o -z "${MY_REPO}" ]; then exit 1 fi + +# make this process nice +renice -n 10 -p $$ +ionice -c 3 -p $$ + SUPPORTED_OS_ARGS=( 'debian' ) OS= OS_VERSION= diff --git a/build-tools/stx/build-image b/build-tools/stx/build-image index e08fb8f2..7b60f3f4 100755 --- a/build-tools/stx/build-image +++ b/build-tools/stx/build-image @@ -30,6 +30,11 @@ import time import utils import yaml +# make ourself nice +pid = os.getpid() +os.setpriority(os.PRIO_PROCESS, 0, 15) +subprocess.run(['ionice', '-c', '3', '-p', str(pid)]) + STX_DEFAULT_DISTRO = discovery.STX_DEFAULT_DISTRO ALL_LAYERS = discovery.get_all_layers(distro=STX_DEFAULT_DISTRO) ALL_BUILD_TYPES = discovery.get_all_build_types(distro=STX_DEFAULT_DISTRO) diff --git a/build-tools/stx/build-pkgs b/build-tools/stx/build-pkgs index 8d644746..ec812a4f 100755 --- a/build-tools/stx/build-pkgs +++ b/build-tools/stx/build-pkgs @@ -38,6 +38,10 @@ import time import utils import yaml +# make ourself nice +pid = os.getpid() +os.setpriority(os.PRIO_PROCESS, 0, 15) +subprocess.run(['ionice', '-c', '3', '-p', str(pid)]) BUILDER_URL = os.environ.get('BUILDER_URL') REPOMGR_URL = os.environ.get('REPOMGR_URL') diff --git a/build-tools/stx/downloader b/build-tools/stx/downloader index 9940d833..fc2f4ed6 100755 --- a/build-tools/stx/downloader +++ b/build-tools/stx/downloader @@ -26,9 +26,15 @@ import pathlib import repo_manage import shutil import signal +import subprocess import sys import utils +# make ourself nice +pid = os.getpid() +os.setpriority(os.PRIO_PROCESS, 0, 15) +subprocess.run(['ionice', '-c', '3', '-p', str(pid)]) + DEFAULT_ARCH = 'amd64' REPO_BIN = 'deb-local-binary' mirror_root = os.environ.get('STX_MIRROR')