diff --git a/stx/dockerfiles/stx-pkgbuilder.Dockerfile b/stx/dockerfiles/stx-pkgbuilder.Dockerfile index 0c11c110..116a9e37 100644 --- a/stx/dockerfiles/stx-pkgbuilder.Dockerfile +++ b/stx/dockerfiles/stx-pkgbuilder.Dockerfile @@ -33,7 +33,9 @@ RUN apt-get update && apt-get install --no-install-recommends -y \ apt-utils \ sbuild \ osc \ + python3-fs \ python3-pip \ + python3-psutil \ git \ wget \ curl \ @@ -58,6 +60,7 @@ RUN groupadd crontab COPY stx/toCOPY/pkgbuilder/app.py /opt/ COPY stx/toCOPY/pkgbuilder/debbuilder.py /opt/ COPY stx/toCOPY/pkgbuilder/schrootspool.py /opt/ +COPY stx/toCOPY/pkgbuilder/utils.py /opt/ COPY stx/toCOPY/pkgbuilder/setup.sh /opt/ COPY stx/toCOPY/pkgbuilder/debbuilder.conf /etc/sbuild/sbuild.conf diff --git a/stx/toCOPY/pkgbuilder/app.py b/stx/toCOPY/pkgbuilder/app.py index da24eb73..0e6be8cb 100644 --- a/stx/toCOPY/pkgbuilder/app.py +++ b/stx/toCOPY/pkgbuilder/app.py @@ -18,6 +18,7 @@ from flask import Flask from flask import jsonify from flask import request import logging +import utils STX_DISTRO = 'bullseye' STX_ARCH = 'amd64' @@ -33,6 +34,7 @@ log_format = logging.Formatter("%(asctime)s - %(levelname)s: %(message)s") handler.setFormatter(log_format) log.addHandler(handler) +utils.set_logger(log) dbuilder = Debbuilder('private', STX_DISTRO, STX_ARCH, log) response = {} diff --git a/stx/toCOPY/pkgbuilder/debbuilder.py b/stx/toCOPY/pkgbuilder/debbuilder.py index ea2bc1c3..c7946e62 100644 --- a/stx/toCOPY/pkgbuilder/debbuilder.py +++ b/stx/toCOPY/pkgbuilder/debbuilder.py @@ -12,11 +12,14 @@ # # Copyright (C) 2021-2022 Wind River Systems,Inc # +import fs import os +import psutil import schrootspool import shutil import signal import subprocess +import utils BUILD_ROOT = '/localdisk/loadbuild/' STORE_ROOT = '/localdisk/pkgbuilder' @@ -149,7 +152,7 @@ class Debbuilder(object): self.logger.error(str(e)) def has_chroot(self, chroot): - chroots = os.popen('schroot -l') + chroots = os.popen('schroot --list') target_line = "chroot:" + chroot for line in chroots: if line.strip() == target_line: @@ -199,7 +202,7 @@ class Debbuilder(object): self.attrs['dist'], user_chroot]) if 'mirror' in request_form: chroot_cmd = ' '.join([chroot_cmd, request_form['mirror']]) - self.logger.debug("Command to creat chroot:%s" % chroot_cmd) + self.logger.debug("Command to create chroot:%s" % chroot_cmd) p = subprocess.Popen(chroot_cmd, shell=True, stdout=self.ctlog, stderr=self.ctlog) @@ -249,6 +252,8 @@ class Debbuilder(object): user = request_form['user'] project = request_form['project'] required_instances = int(request_form['instances']) + tmpfs_instances = 0 + tmpfs_percentage = int(request_form['tmpfs_percentage']) chroot_sequence = 1 # Try to find the parent chroot @@ -263,12 +268,52 @@ class Debbuilder(object): response['msg'] = 'The parent chroot %s does not exist' % parent_chroot_path return response + # tmpfs calculations + if required_instances > 1: + GB = 1024 * 1024 * 1024 + min_tmpfs_size_gb = 10 + mem = psutil.virtual_memory() + avail_mem_gb = int(mem.available * tmpfs_percentage / (100 * GB)) + for tmpfs_instances in range(required_instances - 1, -1, -1): + if tmpfs_instances <= 0: + mem_per_instance_gb = 0 + break + mem_per_instance_gb = int(avail_mem_gb / tmpfs_instances) + if mem_per_instance_gb >= min_tmpfs_size_gb: + break + self.logger.debug("The parent chroot %s exists, start to clone chroot with it", parent_chroot_path) + self.logger.debug("creating %s instances, including %s instances using %s gb of tmpfs", required_instances, tmpfs_instances, mem_per_instance_gb) for instance in range(required_instances): cloned_chroot_name = parent_chroot_name + '-' + str(chroot_sequence) cloned_chroot_path = parent_chroot_path + '-' + str(chroot_sequence) - if not os.path.exists(cloned_chroot_path): + use_tmpfs = (instance >= (required_instances - tmpfs_instances)) + + # Delete old chroot + if os.path.exists(cloned_chroot_path): try: + if utils.is_tmpfs(cloned_chroot_path): + utils.unmount_tmpfs(cloned_chroot_path) + shell_cmd = 'rm -rf --one-file-system %s' % cloned_chroot_path + subprocess.check_call(shell_cmd, shell=True) + except Exception as e: + self.logger.error(str(e)) + response['status'] = 'fail' + if not response['msg']: + response['msg'] = 'Failed to delete old chroot instances:' + response['msg'].append(str(instance) + ' ') + continue + + # Create new chroot + self.logger.info("Cloning chroot %s from the parent %s", cloned_chroot_path, parent_chroot_path) + try: + if use_tmpfs: + os.makedirs(cloned_chroot_path) + shell_cmd = 'mount -t tmpfs -o size=%sG tmpfs %s' % (mem_per_instance_gb, cloned_chroot_path) + subprocess.check_call(shell_cmd, shell=True) + shell_cmd = 'cp -ar %s/. %s/' % (parent_chroot_path, cloned_chroot_path) + subprocess.check_call(shell_cmd, shell=True) + else: self.logger.info("Cloning chroot %s from the parent %s", cloned_chroot_path, parent_chroot_path) shell_cmd = 'rm -rf %s.tmp' % cloned_chroot_path subprocess.check_call(shell_cmd, shell=True) @@ -276,15 +321,15 @@ class Debbuilder(object): subprocess.check_call(shell_cmd, shell=True) shell_cmd = 'mv %s.tmp %s' % (cloned_chroot_path, cloned_chroot_path) subprocess.check_call(shell_cmd, shell=True) - except Exception as e: - self.logger.error(str(e)) - response['status'] = 'fail' - if not response['msg']: - response['msg'] = 'The failed chroot instances:' - response['msg'].append(str(instance) + ' ') - continue - else: - self.logger.info("Successfully cloned chroot %s", cloned_chroot_path) + except Exception as e: + self.logger.error(str(e)) + response['status'] = 'fail' + if not response['msg']: + response['msg'] = 'Failed to create chroot instances:' + response['msg'].append(str(instance) + ' ') + continue + else: + self.logger.info("Successfully cloned chroot %s", cloned_chroot_path) self.logger.info("Target cloned chroot %s is ready, updated config", cloned_chroot_path) # For the cloned chroot, the schroot config file also need to be created @@ -403,23 +448,25 @@ class Debbuilder(object): user = request_form['user'] project = request_form['project'] - dst_chroots = self.chroots_pool.get_idle() - if not dst_chroots: + dst_chroots = self.chroots_pool.get_busy() + if dst_chroots: self.logger.warning('Some chroots are busy') - self.logger.warning('Force to refresh chroots') + + self.logger.warning('Force the termination of busy chroots prior to refresh') self.stop_task(request_form) self.chroots_pool.release_all() # Stop all schroot sessions - subprocess.call('schroot -a -e', shell=True) + subprocess.call('schroot --all --end-session', shell=True) + dst_chroots = self.chroots_pool.get_idle() backup_chroot = None user_dir = os.path.join(STORE_ROOT, user, project) user_chroots_dir = os.path.join(user_dir, 'chroots') for chroot in dst_chroots: # e.g. the chroot name is 'chroot:bullseye-amd64--1' self.logger.debug('The current chroot is %s', chroot) - chroot = chroot.split(':')[1] + # chroot = chroot.split(':')[1] self.logger.debug('The name of chroot: %s', chroot) if not backup_chroot: backup_chroot = chroot[0:chroot.rindex('-')] @@ -433,14 +480,26 @@ class Debbuilder(object): continue backup_chroot_path = os.path.join(user_chroots_dir, backup_chroot) + self.logger.debug('The backup chroot path: %s', backup_chroot_path) chroot_path = os.path.join(user_chroots_dir, chroot) + self.logger.debug('The chroot path: %s', chroot_path) + is_tmpfs = self.chroots_pool.is_tmpfs(chroot) + self.logger.debug('is_tmpfs: %s', is_tmpfs) try: - cp_cmd = 'cp -ra %s %s' % (backup_chroot_path, chroot_path + '.tmp') - subprocess.check_call(cp_cmd, shell=True) - rm_cmd = 'rm -rf ' + chroot_path - subprocess.check_call(rm_cmd, shell=True) - mv_cmd = 'mv -f %s %s' % (chroot_path + '.tmp', chroot_path) - subprocess.check_call(mv_cmd, shell=True) + if is_tmpfs: + self.logger.debug('clean directory: %s', chroot_path) + utils.clear_directory(chroot_path) + shell_cmd = 'cp -ar %s/. %s/' % (backup_chroot_path, chroot_path) + self.logger.debug('shell_cmd: %s', shell_cmd) + subprocess.check_call(shell_cmd, shell=True) + self.logger.debug('cmd exits: %s', shell_cmd) + else: + cp_cmd = 'cp -ra %s %s' % (backup_chroot_path, chroot_path + '.tmp') + subprocess.check_call(cp_cmd, shell=True) + rm_cmd = 'rm -rf --one-file-system ' + chroot_path + subprocess.check_call(rm_cmd, shell=True) + mv_cmd = 'mv -f %s %s' % (chroot_path + '.tmp', chroot_path) + subprocess.check_call(mv_cmd, shell=True) except subprocess.CalledProcessError as e: self.logger.error(str(e)) self.logger.error('Failed to refresh the chroot %s', chroot) @@ -481,12 +540,14 @@ class Debbuilder(object): def add_task(self, request_form): response = check_request(request_form, - ['user', 'project', 'type', 'dsc', 'snapshot_idx', 'layer']) + ['user', 'project', 'type', 'dsc', 'snapshot_idx', 'layer', 'size', 'allow_tmpfs']) if response: return response user = request_form['user'] snapshot_index = request_form['snapshot_idx'] layer = request_form['layer'] + size = request_form['size'] + allow_tmpfs = request_form['allow_tmpfs'] chroot = '-'.join([self.attrs['dist'], self.attrs['arch'], user]) if not self.has_chroot(chroot): @@ -505,7 +566,7 @@ class Debbuilder(object): bcommand = ' '.join([BUILD_ENGINE, '-d', self.attrs['dist']]) dsc_build_dir = os.path.dirname(dsc) - chroot = self.chroots_pool.apply() + chroot = self.chroots_pool.acquire(needed_size=size, allow_tmpfs=allow_tmpfs) self.chroots_pool.show() if not chroot: self.logger.error("There is not idle chroot for %s", dsc) diff --git a/stx/toCOPY/pkgbuilder/schrootspool.py b/stx/toCOPY/pkgbuilder/schrootspool.py index 619094e1..b1498e20 100644 --- a/stx/toCOPY/pkgbuilder/schrootspool.py +++ b/stx/toCOPY/pkgbuilder/schrootspool.py @@ -13,15 +13,88 @@ # Copyright (C) 2022 Wind River Systems,Inc # import logging +import os +import re import subprocess +import utils SCHROOTS_CONFIG = '/etc/schroot/chroot.d/' +def bytes_to_human_readable(size): + if size < 1024: + return f'{size}B' + if size < 1024 * 1024: + x = int(size / 102.4) / 10 + return f'{x}KB' + if size < 1024 * 1024 * 1024: + x = int(size / (1024 * 102.4)) / 10 + return f'{x}MB' + if size < 1024 * 1024 * 1024 * 1024: + x = int(size / (1024 * 1024 * 102.4)) / 10 + return f'{x}GB' + x = int(size / (1024 * 1024 * 1024 * 102.4)) / 10 + return f'{x}TB' + + +# Define unit multipliers +unit_multipliers = { + '': 1, # No unit + 'B': 1, # No unit + 'K': 1024, # Kilobytes + 'KB': 1024, # Kilobytes + 'M': 1024**2, # Megabytes + 'MB': 1024**2, # Megabytes + 'G': 1024**3, # Gigabytes + 'GB': 1024**3, # Gigabytes + 'T': 1024**4, # Terabytes + 'TB': 1024**4, # Terabytes +} + + +def human_readable_to_bytes(human_size): + # Define a regular expression pattern to match size strings + pattern = re.compile(r'(?P[.\d]+)(?P[KMGTB]+?)', re.IGNORECASE) + # Match the input string + match = pattern.match(str(human_size).strip()) + if match: + # Extract the value and unit + value = match.group('value') + unit = match.group('unit').upper() + else: + pattern = re.compile(r'(?P[.\d]+)') + match = pattern.match(str(human_size).strip()) + if not match: + raise ValueError(f"Invalid size string: '{human_size}'") + value = match.group('value') + unit = "B" + + if unit not in unit_multipliers: + raise ValueError(f"Unknown unit: '{unit}'") + multiplier = int(unit_multipliers[unit]) + value = int(float(value) * multiplier) + return value + + class Schroot(object): def __init__(self, name, state='idle'): self.name = name self.state = state + self.path = "" + self.size = 0 + self.tmpfs = False + + # Get path to schroot + schroot_config_lines = subprocess.run(['schroot', '--config', '--chroot', name], + stdout=subprocess.PIPE, + universal_newlines=True).stdout.splitlines() + for line in schroot_config_lines: + if line.startswith('directory='): + self.path = line.split('=')[1].strip() + statvfs = os.statvfs(self.path) + self.size = statvfs.f_frsize * statvfs.f_bavail + self.tmpfs = utils.is_tmpfs(self.path) + break def is_idle(self): if self.state == 'idle': @@ -34,11 +107,23 @@ class Schroot(object): def get_name(self): return self.name + def get_size(self): + return self.size + + def get_path(self): + return self.path + + def get_state(self): + return self.state + + def is_tmpfs(self): + return self.tmpfs + class SchrootsPool(object): """ schrootsPool manages all the schroots in current container - The schroots listed by schroot -l will be registered + The schroots listed by schroot --list will be registered and assigned the build task """ def __init__(self, logger): @@ -52,25 +137,38 @@ class SchrootsPool(object): return False def load(self): - schroots = subprocess.run(['schroot', '-l'], stdout=subprocess.PIPE, + self.schroots = [] + schroots = subprocess.run(['schroot', '--list'], stdout=subprocess.PIPE, universal_newlines=True).stdout.splitlines() if len(schroots) < 1: self.logger.error('There are no schroots found, exit') return False for sname in schroots: # Filter 'chroot:bullseye-amd64-' as the backup chroot - if len(sname.split('-')) >= 4 and not self.exists(sname): - self.schroots.append(Schroot(sname.strip(), 'idle')) + name = sname.split(':')[1] + if len(name.split('-')) >= 4 and not self.exists(sname): + self.schroots.append(Schroot(name.strip(), 'idle')) return True - def apply(self): + def acquire(self, needed_size=1, allow_tmpfs=True): self.logger.debug("schroot pool status:") self.show() + needed_size_bytes = human_readable_to_bytes(needed_size) + if allow_tmpfs: + # tmpfs is allowed. Try to find an idle tmpfs build environment. + for schroot in self.schroots: + if schroot.is_idle() and schroot.is_tmpfs() and (needed_size_bytes <= schroot.get_size()): + schroot.set_busy() + self.logger.debug('%s has been assigned', schroot.name) + return schroot.name + + # Find any suitable build environment that is idle for schroot in self.schroots: - if schroot.is_idle(): - schroot.set_busy() - self.logger.debug('%s has been assigned', schroot.name) - return schroot.name + if schroot.is_idle() and (needed_size_bytes <= schroot.get_size()): + if allow_tmpfs or not schroot.is_tmpfs(): + schroot.set_busy() + self.logger.debug('%s has been assigned', schroot.name) + return schroot.name self.logger.debug("No idle schroot can be used") return None @@ -81,12 +179,28 @@ class SchrootsPool(object): schroot.state = 'idle' self.logger.debug('%s has been released', name) + def is_tmpfs(self, name): + for schroot in self.schroots: + if schroot.name == name.strip(): + # Fixme, whether need to end session here + return schroot.is_tmpfs() + return False + + def get_busy(self): + busy_schroots = [] + for schroot in self.schroots: + schroot_name = schroot.get_name() + if schroot.is_idle(): + continue + busy_schroots.append(schroot_name) + self.logger.warning('schroot %s is busy and can not be refreshed', schroot_name) + return busy_schroots + def get_idle(self): idle_schroots = [] for schroot in self.schroots: schroot_name = schroot.get_name() if not schroot.is_idle(): - self.logger.error('schroot %s is busy and can not be refreshed', schroot_name) continue idle_schroots.append(schroot_name) self.logger.debug('schroot %s is idle and can be refreshed', schroot_name) @@ -96,11 +210,14 @@ class SchrootsPool(object): for schroot in self.schroots: # Fixme, whether need to end session here schroot.state = 'idle' - self.logger.debug('All chroots has been released') + self.logger.debug('All chroots have been released') def show(self): for schroot in self.schroots: - self.logger.info("schroot name:%s state:%s", schroot.name, schroot.state) + self.logger.info("schroot name:%s state:%s tmpfs:%s size:%s path=%s", + schroot.get_name(), schroot.get_state(), + schroot.is_tmpfs(), bytes_to_human_readable(schroot.get_size()), + schroot.get_path()) if __name__ == "__main__": @@ -112,9 +229,9 @@ if __name__ == "__main__": schroots_pool = SchrootsPool(logger) schroots_pool.load() - s0 = schroots_pool.apply() - s1 = schroots_pool.apply() - s2 = schroots_pool.apply() + s0 = schroots_pool.acquire() + s1 = schroots_pool.acquire() + s2 = schroots_pool.acquire() schroots_pool.show() schroots_pool.release(s0) schroots_pool.release(s1) diff --git a/stx/toCOPY/pkgbuilder/utils.py b/stx/toCOPY/pkgbuilder/utils.py new file mode 100644 index 00000000..53af0ae2 --- /dev/null +++ b/stx/toCOPY/pkgbuilder/utils.py @@ -0,0 +1,55 @@ +import os +import shutil +import subprocess + +logger = None + + +def set_logger(log): + global logger + logger = log + + +def is_tmpfs(mount_point): + # Check if the given mount point is a tmpfs filesystem. + try: + # Run the 'mount' command to get filesystem details + result = subprocess.run(['mount', '-v'], capture_output=True, text=True, check=True) + # Check if the mount point is listed as tmpfs + for line in result.stdout.splitlines(): + if mount_point in line and 'tmpfs' in line: + return True + return False + except subprocess.CalledProcessError as e: + logger.error(f"Failed to check filesystem type: {e}") + return False + + +def is_directory_empty(directory): + if not os.path.isdir(directory): + raise NotADirectoryError(f"The path {directory} is not a directory or does not exist.") + return not any(os.scandir(directory)) + + +def unmount_tmpfs(mount_point): + # Unmount the tmpfs + if not is_tmpfs(mount_point): + return False + try: + subprocess.run(['umount', mount_point], check=True) + except subprocess.CalledProcessError as e: + logger.error(f"Failed to unmount {mount_point}: {e}") + return False + logger.debug(f"Unmounted {mount_point}") + + +def clear_directory(directory): + # Empty the directory without actually deleting it. + # The intended use is for mount points. + if not is_directory_empty(directory): + for item in os.listdir(directory): + item_path = os.path.join(directory, item) + if os.path.isfile(item_path) or os.path.islink(item_path): + os.remove(item_path) + elif os.path.isdir(item_path): + shutil.rmtree(item_path)