Add option to use tmpfs during build-pkgs

Building in a tmpfs speeds the build process and greatly reduces
the disk io required during a build, leaving the host far more
responsive to other tasks.

Added option --tmpfs_percentage=N to the build-pkgs command
to control how much memory can be used for tmpfs build environments.
The valid range is 0-50%. The default is 0% resulting in a traditional
disk only build.

The available memory on the host is multiplied by the percentage
to determine how much is availabe for tmpfs in total.  This total
is then assigned to up to N-1 of the N parallel build environments,
with the minimum tmpfs environment size being 10GB.
One environment is reserved for disk based builds.  First time
package builds are assigned to tmpfs based build environments in
preference over the disk base environments. However, if a package
has failed a prior build attempt, subsequents attemps will only
occure on a disk based build environmnet.  This may help if the
build failed due to a too small tmpfs build environment.  It also
may leave the environment intact for mor indepth debugging.

Testing has revealed that 20gb is required to build the largest
packages (linux, ceph, kubernetes).  In order to avoid costly
rebuilds of these large packages, the choice of tmpfs percentage
and number of parallel builds is important.

e.g. on a single user host with 128GB of memory, an appropriate
choice might be...

    --parallel=4 --tmpfs_percentage=50

...yielding 3 tmpfs build environments of aprox 20GB each.
A higher parallelism, or a lower tempfs percentage will result
in build environments that drop below 20GB, and you might start
seeing rebuilds of large packages.

Further development is suggested.  If we can add an advisory to the
dsc metadata suggesting a minimum space requireemnt for the build of
a package, we can proactively assign large package to a build environment
that is large enough to support it, avoiding rebuilds.

Testing:
   - build-pkgs without --tmpfs_percentage
     Only disk base build envoronments are used.
   - build-pkgs --parallel=4 --tmpfs_percentage=20
     On 128GB machine, Only 2 10-12 GB tmpfs are used,
     the other two remanin disk based.
     Large packages fail in 10GB tmpfs, and a pass when retried on disk.
   - build-pkgs --parallel=4 --tmpfs_percentage=50
     On 128GB machine, Creates 3 18-20 GB tmpfs.
     Large packages build in tmpfs without need for rebuild.

partial-bug: 2081843
Change-Id: I09dd2f60afc3e866ec8f86b6898d41f19a419d87
Signed-off-by: Scott Little <scott.little@windriver.com>

stx/dockerfiles/stx-pkgbuilder.Dockerfile

@@ -33,7 +33,9 @@ RUN     apt-get update && apt-get install --no-install-recommends -y \
         apt-utils \
         sbuild \
         osc \
+        python3-fs \
         python3-pip \
+        python3-psutil \
         git \
         wget \
         curl \
@@ -58,6 +60,7 @@ RUN groupadd crontab
 COPY stx/toCOPY/pkgbuilder/app.py /opt/
 COPY stx/toCOPY/pkgbuilder/debbuilder.py /opt/
 COPY stx/toCOPY/pkgbuilder/schrootspool.py /opt/
+COPY stx/toCOPY/pkgbuilder/utils.py /opt/
 COPY stx/toCOPY/pkgbuilder/setup.sh /opt/
 COPY stx/toCOPY/pkgbuilder/debbuilder.conf /etc/sbuild/sbuild.conf
 

stx/toCOPY/pkgbuilder/app.py

@@ -18,6 +18,7 @@ from flask import Flask
 from flask import jsonify
 from flask import request
 import logging
+import utils
 
 STX_DISTRO = 'bullseye'
 STX_ARCH = 'amd64'
@@ -33,6 +34,7 @@ log_format = logging.Formatter("%(asctime)s - %(levelname)s: %(message)s")
 handler.setFormatter(log_format)
 log.addHandler(handler)
 
+utils.set_logger(log)
 dbuilder = Debbuilder('private', STX_DISTRO, STX_ARCH, log)
 response = {}
 

stx/toCOPY/pkgbuilder/debbuilder.py

@@ -12,11 +12,14 @@
 #
 # Copyright (C) 2021-2022 Wind River Systems,Inc
 #
+import fs
 import os
+import psutil
 import schrootspool
 import shutil
 import signal
 import subprocess
+import utils
 
 BUILD_ROOT = '/localdisk/loadbuild/'
 STORE_ROOT = '/localdisk/pkgbuilder'
@@ -149,7 +152,7 @@ class Debbuilder(object):
                 self.logger.error(str(e))
 
     def has_chroot(self, chroot):
-        chroots = os.popen('schroot -l')
+        chroots = os.popen('schroot --list')
         target_line = "chroot:" + chroot
         for line in chroots:
             if line.strip() == target_line:
@@ -199,7 +202,7 @@ class Debbuilder(object):
                                    self.attrs['dist'], user_chroot])
             if 'mirror' in request_form:
                 chroot_cmd = ' '.join([chroot_cmd, request_form['mirror']])
-            self.logger.debug("Command to creat chroot:%s" % chroot_cmd)
+            self.logger.debug("Command to create chroot:%s" % chroot_cmd)
 
             p = subprocess.Popen(chroot_cmd, shell=True, stdout=self.ctlog,
                                  stderr=self.ctlog)
@@ -249,6 +252,8 @@ class Debbuilder(object):
         user = request_form['user']
         project = request_form['project']
         required_instances = int(request_form['instances'])
+        tmpfs_instances = 0
+        tmpfs_percentage = int(request_form['tmpfs_percentage'])
         chroot_sequence = 1
 
         # Try to find the parent chroot
@@ -263,12 +268,52 @@ class Debbuilder(object):
             response['msg'] = 'The parent chroot %s does not exist' % parent_chroot_path
             return response
 
+        # tmpfs calculations
+        if required_instances > 1:
+            GB = 1024 * 1024 * 1024
+            min_tmpfs_size_gb = 10
+            mem = psutil.virtual_memory()
+            avail_mem_gb = int(mem.available * tmpfs_percentage / (100 * GB))
+            for tmpfs_instances in range(required_instances - 1, -1, -1):
+                if tmpfs_instances <= 0:
+                    mem_per_instance_gb = 0
+                    break
+                mem_per_instance_gb = int(avail_mem_gb / tmpfs_instances)
+                if mem_per_instance_gb >= min_tmpfs_size_gb:
+                    break
+
         self.logger.debug("The parent chroot %s exists, start to clone chroot with it", parent_chroot_path)
+        self.logger.debug("creating %s instances, including %s instances using %s gb of tmpfs", required_instances, tmpfs_instances, mem_per_instance_gb)
         for instance in range(required_instances):
             cloned_chroot_name = parent_chroot_name + '-' + str(chroot_sequence)
             cloned_chroot_path = parent_chroot_path + '-' + str(chroot_sequence)
-            if not os.path.exists(cloned_chroot_path):
+            use_tmpfs = (instance >= (required_instances - tmpfs_instances))
+
+            # Delete old chroot
+            if os.path.exists(cloned_chroot_path):
                 try:
+                    if utils.is_tmpfs(cloned_chroot_path):
+                        utils.unmount_tmpfs(cloned_chroot_path)
+                    shell_cmd = 'rm -rf --one-file-system %s' % cloned_chroot_path
+                    subprocess.check_call(shell_cmd, shell=True)
+                except Exception as e:
+                    self.logger.error(str(e))
+                    response['status'] = 'fail'
+                    if not response['msg']:
+                        response['msg'] = 'Failed to delete old chroot instances:'
+                    response['msg'].append(str(instance) + ' ')
+                    continue
+
+            # Create new chroot
+            self.logger.info("Cloning chroot %s from the parent %s", cloned_chroot_path, parent_chroot_path)
+            try:
+                if use_tmpfs:
+                    os.makedirs(cloned_chroot_path)
+                    shell_cmd = 'mount -t tmpfs -o size=%sG tmpfs %s' % (mem_per_instance_gb, cloned_chroot_path)
+                    subprocess.check_call(shell_cmd, shell=True)
+                    shell_cmd = 'cp -ar %s/. %s/' % (parent_chroot_path, cloned_chroot_path)
+                    subprocess.check_call(shell_cmd, shell=True)
+                else:
                     self.logger.info("Cloning chroot %s from the parent %s", cloned_chroot_path, parent_chroot_path)
                     shell_cmd = 'rm -rf %s.tmp' % cloned_chroot_path
                     subprocess.check_call(shell_cmd, shell=True)
@@ -276,15 +321,15 @@ class Debbuilder(object):
                     subprocess.check_call(shell_cmd, shell=True)
                     shell_cmd = 'mv %s.tmp %s' % (cloned_chroot_path, cloned_chroot_path)
                     subprocess.check_call(shell_cmd, shell=True)
-                except Exception as e:
-                    self.logger.error(str(e))
-                    response['status'] = 'fail'
-                    if not response['msg']:
-                        response['msg'] = 'The failed chroot instances:'
-                    response['msg'].append(str(instance) + ' ')
-                    continue
-                else:
-                    self.logger.info("Successfully cloned chroot %s", cloned_chroot_path)
+            except Exception as e:
+                self.logger.error(str(e))
+                response['status'] = 'fail'
+                if not response['msg']:
+                    response['msg'] = 'Failed to create chroot instances:'
+                response['msg'].append(str(instance) + ' ')
+                continue
+            else:
+                self.logger.info("Successfully cloned chroot %s", cloned_chroot_path)
 
             self.logger.info("Target cloned chroot %s is ready, updated config", cloned_chroot_path)
             # For the cloned chroot, the schroot config file also need to be created
@@ -403,23 +448,25 @@ class Debbuilder(object):
         user = request_form['user']
         project = request_form['project']
 
-        dst_chroots = self.chroots_pool.get_idle()
-        if not dst_chroots:
+        dst_chroots = self.chroots_pool.get_busy()
+        if dst_chroots:
             self.logger.warning('Some chroots are busy')
-        self.logger.warning('Force to refresh chroots')
+
+        self.logger.warning('Force the termination of busy chroots prior to refresh')
         self.stop_task(request_form)
         self.chroots_pool.release_all()
 
         # Stop all schroot sessions
-        subprocess.call('schroot -a -e', shell=True)
+        subprocess.call('schroot --all --end-session', shell=True)
 
+        dst_chroots = self.chroots_pool.get_idle()
         backup_chroot = None
         user_dir = os.path.join(STORE_ROOT, user, project)
         user_chroots_dir = os.path.join(user_dir, 'chroots')
         for chroot in dst_chroots:
             # e.g. the chroot name is 'chroot:bullseye-amd64-<user>-1'
             self.logger.debug('The current chroot is %s', chroot)
-            chroot = chroot.split(':')[1]
+            # chroot = chroot.split(':')[1]
             self.logger.debug('The name of chroot: %s', chroot)
             if not backup_chroot:
                 backup_chroot = chroot[0:chroot.rindex('-')]
@@ -433,14 +480,26 @@ class Debbuilder(object):
                 continue
 
             backup_chroot_path = os.path.join(user_chroots_dir, backup_chroot)
+            self.logger.debug('The backup chroot path: %s', backup_chroot_path)
             chroot_path = os.path.join(user_chroots_dir, chroot)
+            self.logger.debug('The chroot path: %s', chroot_path)
+            is_tmpfs = self.chroots_pool.is_tmpfs(chroot)
+            self.logger.debug('is_tmpfs: %s', is_tmpfs)
             try:
-                cp_cmd = 'cp -ra %s %s' % (backup_chroot_path, chroot_path + '.tmp')
-                subprocess.check_call(cp_cmd, shell=True)
-                rm_cmd = 'rm -rf ' + chroot_path
-                subprocess.check_call(rm_cmd, shell=True)
-                mv_cmd = 'mv -f %s %s' % (chroot_path + '.tmp', chroot_path)
-                subprocess.check_call(mv_cmd, shell=True)
+                if is_tmpfs:
+                    self.logger.debug('clean directory: %s', chroot_path)
+                    utils.clear_directory(chroot_path)
+                    shell_cmd = 'cp -ar %s/. %s/' % (backup_chroot_path, chroot_path)
+                    self.logger.debug('shell_cmd: %s', shell_cmd)
+                    subprocess.check_call(shell_cmd, shell=True)
+                    self.logger.debug('cmd exits: %s', shell_cmd)
+                else:
+                    cp_cmd = 'cp -ra %s %s' % (backup_chroot_path, chroot_path + '.tmp')
+                    subprocess.check_call(cp_cmd, shell=True)
+                    rm_cmd = 'rm -rf --one-file-system ' + chroot_path
+                    subprocess.check_call(rm_cmd, shell=True)
+                    mv_cmd = 'mv -f %s %s' % (chroot_path + '.tmp', chroot_path)
+                    subprocess.check_call(mv_cmd, shell=True)
             except subprocess.CalledProcessError as e:
                 self.logger.error(str(e))
                 self.logger.error('Failed to refresh the chroot %s', chroot)
@@ -481,12 +540,14 @@ class Debbuilder(object):
 
     def add_task(self, request_form):
         response = check_request(request_form,
-                                 ['user', 'project', 'type', 'dsc', 'snapshot_idx', 'layer'])
+                                 ['user', 'project', 'type', 'dsc', 'snapshot_idx', 'layer', 'size', 'allow_tmpfs'])
         if response:
             return response
         user = request_form['user']
         snapshot_index = request_form['snapshot_idx']
         layer = request_form['layer']
+        size = request_form['size']
+        allow_tmpfs = request_form['allow_tmpfs']
 
         chroot = '-'.join([self.attrs['dist'], self.attrs['arch'], user])
         if not self.has_chroot(chroot):
@@ -505,7 +566,7 @@ class Debbuilder(object):
 
         bcommand = ' '.join([BUILD_ENGINE, '-d', self.attrs['dist']])
         dsc_build_dir = os.path.dirname(dsc)
-        chroot = self.chroots_pool.apply()
+        chroot = self.chroots_pool.acquire(needed_size=size, allow_tmpfs=allow_tmpfs)
         self.chroots_pool.show()
         if not chroot:
             self.logger.error("There is not idle chroot for %s", dsc)

stx/toCOPY/pkgbuilder/schrootspool.py

@@ -13,15 +13,88 @@
 # Copyright (C) 2022 Wind River Systems,Inc
 #
 import logging
+import os
+import re
 import subprocess
+import utils
 
 SCHROOTS_CONFIG = '/etc/schroot/chroot.d/'
 
 
+def bytes_to_human_readable(size):
+    if size < 1024:
+        return f'{size}B'
+    if size < 1024 * 1024:
+        x = int(size / 102.4) / 10
+        return f'{x}KB'
+    if size < 1024 * 1024 * 1024:
+        x = int(size / (1024 * 102.4)) / 10
+        return f'{x}MB'
+    if size < 1024 * 1024 * 1024 * 1024:
+        x = int(size / (1024 * 1024 * 102.4)) / 10
+        return f'{x}GB'
+    x = int(size / (1024 * 1024 * 1024 * 102.4)) / 10
+    return f'{x}TB'
+
+
+# Define unit multipliers
+unit_multipliers = {
+    '': 1,          # No unit
+    'B': 1,         # No unit
+    'K': 1024,      # Kilobytes
+    'KB': 1024,     # Kilobytes
+    'M': 1024**2,   # Megabytes
+    'MB': 1024**2,  # Megabytes
+    'G': 1024**3,   # Gigabytes
+    'GB': 1024**3,  # Gigabytes
+    'T': 1024**4,   # Terabytes
+    'TB': 1024**4,  # Terabytes
+}
+
+
+def human_readable_to_bytes(human_size):
+    # Define a regular expression pattern to match size strings
+    pattern = re.compile(r'(?P<value>[.\d]+)(?P<unit>[KMGTB]+?)', re.IGNORECASE)
+    # Match the input string
+    match = pattern.match(str(human_size).strip())
+    if match:
+        # Extract the value and unit
+        value = match.group('value')
+        unit = match.group('unit').upper()
+    else:
+        pattern = re.compile(r'(?P<value>[.\d]+)')
+        match = pattern.match(str(human_size).strip())
+        if not match:
+            raise ValueError(f"Invalid size string: '{human_size}'")
+        value = match.group('value')
+        unit = "B"
+
+    if unit not in unit_multipliers:
+        raise ValueError(f"Unknown unit: '{unit}'")
+    multiplier = int(unit_multipliers[unit])
+    value = int(float(value) * multiplier)
+    return value
+
+
 class Schroot(object):
     def __init__(self, name, state='idle'):
         self.name = name
         self.state = state
+        self.path = ""
+        self.size = 0
+        self.tmpfs = False
+
+        # Get path to schroot
+        schroot_config_lines = subprocess.run(['schroot', '--config', '--chroot', name],
+                                              stdout=subprocess.PIPE,
+                                              universal_newlines=True).stdout.splitlines()
+        for line in schroot_config_lines:
+            if line.startswith('directory='):
+                self.path = line.split('=')[1].strip()
+                statvfs = os.statvfs(self.path)
+                self.size = statvfs.f_frsize * statvfs.f_bavail
+                self.tmpfs = utils.is_tmpfs(self.path)
+                break
 
     def is_idle(self):
         if self.state == 'idle':
@@ -34,11 +107,23 @@ class Schroot(object):
     def get_name(self):
         return self.name
 
+    def get_size(self):
+        return self.size
+
+    def get_path(self):
+        return self.path
+
+    def get_state(self):
+        return self.state
+
+    def is_tmpfs(self):
+        return self.tmpfs
+
 
 class SchrootsPool(object):
     """
     schrootsPool manages all the schroots in current container
-    The schroots listed by schroot -l will be registered
+    The schroots listed by schroot --list will be registered
     and assigned the build task
     """
     def __init__(self, logger):
@@ -52,25 +137,38 @@ class SchrootsPool(object):
         return False
 
     def load(self):
-        schroots = subprocess.run(['schroot', '-l'], stdout=subprocess.PIPE,
+        self.schroots = []
+        schroots = subprocess.run(['schroot', '--list'], stdout=subprocess.PIPE,
                                   universal_newlines=True).stdout.splitlines()
         if len(schroots) < 1:
             self.logger.error('There are no schroots found, exit')
             return False
         for sname in schroots:
             # Filter 'chroot:bullseye-amd64-<user>' as the backup chroot
-            if len(sname.split('-')) >= 4 and not self.exists(sname):
-                self.schroots.append(Schroot(sname.strip(), 'idle'))
+            name = sname.split(':')[1]
+            if len(name.split('-')) >= 4 and not self.exists(sname):
+                self.schroots.append(Schroot(name.strip(), 'idle'))
         return True
 
-    def apply(self):
+    def acquire(self, needed_size=1, allow_tmpfs=True):
         self.logger.debug("schroot pool status:")
         self.show()
+        needed_size_bytes = human_readable_to_bytes(needed_size)
+        if allow_tmpfs:
+            # tmpfs is allowed. Try to find an idle tmpfs build environment.
+            for schroot in self.schroots:
+                if schroot.is_idle() and schroot.is_tmpfs() and (needed_size_bytes <= schroot.get_size()):
+                    schroot.set_busy()
+                    self.logger.debug('%s has been assigned', schroot.name)
+                    return schroot.name
+
+        # Find any suitable build environment that is idle
         for schroot in self.schroots:
-            if schroot.is_idle():
-                schroot.set_busy()
-                self.logger.debug('%s has been assigned', schroot.name)
-                return schroot.name
+            if schroot.is_idle() and (needed_size_bytes <= schroot.get_size()):
+                if allow_tmpfs or not schroot.is_tmpfs():
+                    schroot.set_busy()
+                    self.logger.debug('%s has been assigned', schroot.name)
+                    return schroot.name
         self.logger.debug("No idle schroot can be used")
         return None
 
@@ -81,12 +179,28 @@ class SchrootsPool(object):
                 schroot.state = 'idle'
                 self.logger.debug('%s has been released', name)
 
+    def is_tmpfs(self, name):
+        for schroot in self.schroots:
+            if schroot.name == name.strip():
+                # Fixme, whether need to end session here
+                return schroot.is_tmpfs()
+        return False
+
+    def get_busy(self):
+        busy_schroots = []
+        for schroot in self.schroots:
+            schroot_name = schroot.get_name()
+            if schroot.is_idle():
+                continue
+            busy_schroots.append(schroot_name)
+            self.logger.warning('schroot %s is busy and can not be refreshed', schroot_name)
+        return busy_schroots
+
     def get_idle(self):
         idle_schroots = []
         for schroot in self.schroots:
             schroot_name = schroot.get_name()
             if not schroot.is_idle():
-                self.logger.error('schroot %s is busy and can not be refreshed', schroot_name)
                 continue
             idle_schroots.append(schroot_name)
             self.logger.debug('schroot %s is idle and can be refreshed', schroot_name)
@@ -96,11 +210,14 @@ class SchrootsPool(object):
         for schroot in self.schroots:
             # Fixme, whether need to end session here
             schroot.state = 'idle'
-        self.logger.debug('All chroots has been released')
+        self.logger.debug('All chroots have been released')
 
     def show(self):
         for schroot in self.schroots:
-            self.logger.info("schroot name:%s state:%s", schroot.name, schroot.state)
+            self.logger.info("schroot name:%s state:%s tmpfs:%s size:%s path=%s",
+                             schroot.get_name(), schroot.get_state(),
+                             schroot.is_tmpfs(), bytes_to_human_readable(schroot.get_size()),
+                             schroot.get_path())
 
 
 if __name__ == "__main__":
@@ -112,9 +229,9 @@ if __name__ == "__main__":
 
     schroots_pool = SchrootsPool(logger)
     schroots_pool.load()
-    s0 = schroots_pool.apply()
-    s1 = schroots_pool.apply()
-    s2 = schroots_pool.apply()
+    s0 = schroots_pool.acquire()
+    s1 = schroots_pool.acquire()
+    s2 = schroots_pool.acquire()
     schroots_pool.show()
     schroots_pool.release(s0)
     schroots_pool.release(s1)

stx/toCOPY/pkgbuilder/utils.py

@@ -0,0 +1,55 @@
+import os
+import shutil
+import subprocess
+
+logger = None
+
+
+def set_logger(log):
+    global logger
+    logger = log
+
+
+def is_tmpfs(mount_point):
+    #  Check if the given mount point is a tmpfs filesystem.
+    try:
+        # Run the 'mount' command to get filesystem details
+        result = subprocess.run(['mount', '-v'], capture_output=True, text=True, check=True)
+        # Check if the mount point is listed as tmpfs
+        for line in result.stdout.splitlines():
+            if mount_point in line and 'tmpfs' in line:
+                return True
+        return False
+    except subprocess.CalledProcessError as e:
+        logger.error(f"Failed to check filesystem type: {e}")
+        return False
+
+
+def is_directory_empty(directory):
+    if not os.path.isdir(directory):
+        raise NotADirectoryError(f"The path {directory} is not a directory or does not exist.")
+    return not any(os.scandir(directory))
+
+
+def unmount_tmpfs(mount_point):
+    # Unmount the tmpfs
+    if not is_tmpfs(mount_point):
+        return False
+    try:
+        subprocess.run(['umount', mount_point], check=True)
+    except subprocess.CalledProcessError as e:
+        logger.error(f"Failed to unmount {mount_point}: {e}")
+        return False
+    logger.debug(f"Unmounted {mount_point}")
+
+
+def clear_directory(directory):
+    # Empty the directory without actually deleting it.
+    # The intended use is for mount points.
+    if not is_directory_empty(directory):
+        for item in os.listdir(directory):
+            item_path = os.path.join(directory, item)
+            if os.path.isfile(item_path) or os.path.islink(item_path):
+                os.remove(item_path)
+            elif os.path.isdir(item_path):
+                shutil.rmtree(item_path)