build-image: Merge the local repositories and feed to debootstrap

The function of aptly snapshot merging is used to merge
the local repositories 'deb-local-binary' and
'deb-local-build' to a published unified snapshot.
This snapshot will be feeded to 'debootstrap' in LAT
and to do the rootfs.

In order to meet this changes, the corresponding changes
are also done in build-pkgs and downloader.

It should be noted that this commit can not fix the
package conflicting issues, the purpose is to replace
the upstream official repo for debootstrap with local
repositories, so the developer can fix the package
conflicit issues through modifying the local repositories.

Test Plan:
Pass: The replacement is not be triggered until
      'deb-merge-all' is set to 'debootstrap-mirror' in
      base-bullseye.yaml or in base-initramfs-bullseye.yaml
Pass: If triggered, an aptly snapshot
      'http://<REPOMGR_DEPLOY_URL>/deb-merge-all' is published
      and consumed by debootstrap in LAT, the debootstrap works as normal

Pass: The messages "WARNING: Drop duplicate package" reports by
      repo_manage is used to sort out the packages list

Depends-On: https://review.opendev.org/c/starlingx/root/+/826685

Story: 2008846
Task: 44359

Signed-off-by: hbai <haiqing.bai@windriver.com>
Change-Id: I4038135e4148277a3065397a571e25901086798a

build-tools/stx/build-image

@@ -25,6 +25,7 @@ import time
 import utils
 import yaml
 
+REPO_ALL = 'deb-merge-all'
 REPO_BINARY = 'deb-local-binary'
 REPO_BUILD = 'deb-local-build'
 DEB_CONFIG_DIR = 'stx-tools/debian-mirror-tools/config/'
@@ -34,6 +35,50 @@ img_pkgs = []
 logger = logging.getLogger('build-image')
 utils.set_logger(logger)
 
+
+def merge_local_repos(repomgr):
+    logger.debug('Calls repo manager to create/udpate the snapshot %s which is merged from local repositories', REPO_ALL)
+    # REPO_BUILD is higher priority than REPO_BINARY for repomgr to select package
+    try:
+        pubname = repomgr.merge(REPO_ALL, ','.join([REPO_BUILD, REPO_BINARY]))
+    except Exception as e:
+        logger.error(str(e))
+        logger.error('Exception when repo_manager creates/updates snapshot %s', REPO_ALL)
+        return False
+    if pubname:
+        logger.debug('repo manager successfully created/updated snapshot %s', REPO_ALL)
+    else:
+        logger.debug('repo manager failed to create/update snapshot %s', REPO_ALL)
+        return False
+    return True
+
+
+def update_debootstrap_mirror(img_yaml):
+    repomgr_url = os.environ.get('REPOMGR_DEPLOY_URL')
+    if not repomgr_url:
+        logger.error('REPOMGR_URL is not in current sys ENV')
+        return False
+
+    repo_all_url = '/'.join([repomgr_url, REPO_ALL])
+    try:
+        with open(img_yaml) as f:
+            yaml_doc = yaml.safe_load(f)
+            if not yaml_doc['debootstrap-mirror']:
+                logger.warning("There is not debootstrap-mirror in %s", img_yaml)
+            else:
+                mirror = yaml_doc['debootstrap-mirror']
+                if mirror == REPO_ALL:
+                    yaml_doc['debootstrap-mirror'] = repo_all_url
+                    with open(img_yaml, 'w') as f:
+                        yaml.safe_dump(yaml_doc, f, default_flow_style=False, sort_keys=False)
+                    logger.debug('Updating %s, setting debootstrap_mirror to %s', img_yaml, repo_all_url)
+                    return True
+    except IOError as e:
+        logger.error(str(e))
+        logger.debug('Failed to update %s, could not set debootstrap_mirror to %s', img_yaml, repo_all_url)
+    return False
+
+
 def update_ostree_osname(img_yaml):
 
     ostree_osname = os.environ.get('OSTREE_OSNAME')
@@ -53,6 +98,7 @@ def update_ostree_osname(img_yaml):
     logger.debug(' '.join(['Update', img_yaml, 'to update the ostree_osname']))
     return True
 
+
 def feed_lat_src_repos(img_yaml, repo_url):
     if not os.path.exists(img_yaml):
         logger.error(' '.join(['LAT yaml file', img_yaml, 'does not exist']))
@@ -97,7 +143,7 @@ def check_base_os_binaries(repomgr):
                                'does not exist']))
         return False
 
-    results = query_repo(repomgr, REPO_BINARY, base_bins_list)
+    results = verify_pkgs_in_repo(repomgr, REPO_BINARY, base_bins_list)
     if results:
         logger.error("====OS binaries checking fail:")
         for deb in results:
@@ -117,7 +163,7 @@ def check_stx_binaries(repomgr, btype='std'):
         # Assume no such list here means ok
         return True
 
-    results = query_repo(repomgr, REPO_BINARY, stx_bins_list)
+    results = verify_pkgs_in_repo(repomgr, REPO_BINARY, stx_bins_list)
     if results:
         logger.error("====STX binaries checking fail:")
         for deb in results:
@@ -136,7 +182,7 @@ def check_stx_patched(repomgr, btype='std'):
                                 'does not exist']))
         return False
 
-    results = query_repo(repomgr, REPO_BUILD, stx_patched_list)
+    results = verify_pkgs_in_repo(repomgr, REPO_BUILD, stx_patched_list)
     if results:
         logger.error("====STX patched packages checking fail:")
         for deb in results:
@@ -147,7 +193,7 @@ def check_stx_patched(repomgr, btype='std'):
     return True
 
 
-def query_repo(repomgr, repo_name, pkg_list_path):
+def verify_pkgs_in_repo(repomgr, repo_name, pkg_list_path):
     failed_pkgs = []
     with open(pkg_list_path, 'r') as flist:
         lines = list(line for line in (lpkg.strip() for lpkg in flist) if line)
@@ -169,7 +215,7 @@ def query_repo(repomgr, repo_name, pkg_list_path):
                                  'is missing in local binary repo']))
                     failed_pkgs.append(pkg_name)
             else:
-                if repomgr.search_pkg(repo_name, name):
+                if repomgr.search_pkg(repo_name, name, None, True):
                     img_pkgs.append(name)
                     logger.debug(''.join(['Found package with name:', name]))
                 else:
@@ -205,8 +251,10 @@ if __name__ == "__main__":
             logger.error(e.msg)
             usage()
 
+    rmg_logger = logging.getLogger('repo_manager')
+    utils.set_logger(rmg_logger)
     repo_manager = repo_manage.RepoMgr('aptly', os.environ.get('REPOMGR_URL'),
-                                       '/tmp/', logger)
+                                       '/tmp/', rmg_logger)
     repo_manager.upload_pkg(REPO_BUILD, None)
     repo_manager.upload_pkg(REPO_BINARY, None)
 
@@ -242,9 +290,16 @@ if __name__ == "__main__":
         shutil.copyfile(base_yaml, lat_yaml)
         shutil.copyfile(base_initramfs_yaml, lat_initramfs_yaml)
     except IOError as e:
+        logger.error(str(e))
         logger.error('Fail to copy image yaml files to /localdisk/deploy')
         sys.exit(1)
 
+    if merge_local_repos(repo_manager):
+        if update_debootstrap_mirror(lat_yaml):
+            logger.debug("Debootstrap switches to mirror %s in %s", REPO_ALL, lat_yaml)
+        if update_debootstrap_mirror(lat_initramfs_yaml):
+            logger.debug("Debootstrap switches to mirror %s in %s", REPO_ALL, lat_initramfs_yaml)
+
     binary_repo_url = ''.join(['deb [trusted=yes] ',
                                os.environ.get('REPOMGR_DEPLOY_URL'),
                                REPO_BINARY, ' bullseye main'])

build-tools/stx/build-pkgs

@@ -450,12 +450,9 @@ class BuildController():
                 msg = ''.join(['package ', pkg_item[0], '(', pkg_item[1], ')'])
 
                 logger.info(' '.join(['Searching for binary', msg, 'in repository', REPO_BUILD]))
-                if self.kits['repo_mgr'].search_pkg(REPO_BUILD, pkg_item[0],
+                if self.kits['repo_mgr'].search_pkg(REPO_BUILD, pkg_item[0], None, True):
-                                                    pkg_item[1]):
                     logger.info('Found binary %s in repository %s', msg, REPO_BUILD)
-                    if self.kits['repo_mgr'].delete_pkg(REPO_BUILD,
+                    if self.kits['repo_mgr'].delete_pkg(REPO_BUILD, pkg_item[0], 'binary', None):
-                                                        pkg_item[0], 'binary',
-                                                        pkg_item[1]):
                         logger.info('Successfully deleted binary %s from repository %s',
                                     msg, REPO_BUILD)
                     else:

build-tools/stx/downloader

@@ -126,8 +126,10 @@ class BaseDownloader():
         self.dl_need = []
         self.dl_success = []
         self.dl_failed = []
+        rlogger = logging.getLogger('repo_manager')
+        utils.set_logger(rlogger)
         self.repomgr = repo_manage.RepoMgr('aptly', os.environ.get('REPOMGR_URL'),
-                                           '/tmp/', logger)
+                                           '/tmp/', rlogger)
         self.repomgr.upload_pkg(REPO_BIN, None)
 
     def clean(self):
@@ -177,21 +179,22 @@ class DebDownloader(BaseDownloader):
             self.repomgr.upload_pkg(REPO_BIN, None)
 
     def download(self, _name, _version):
-        package = self.apt_cache[_name]
-        candidate = package.versions.get(_version)
-        if not candidate:
-            logger.error(' '.join(['Fail to download', _name,
-                         'with wrong version', _version, '?']))
-            logger.error('May need to update the package list file')
-            return None
-
-        package.candidate = candidate
         try:
+            package = self.apt_cache[_name]
+            candidate = package.versions.get(_version)
+            if not candidate:
+                logger.error(' '.join(['Fail to download', _name,
+                             'with wrong version', _version, '?']))
+                logger.error('May need to update the package list file')
+                return None
+
+            package.candidate = candidate
             ret = package.candidate.fetch_binary(self.dl_dir)
             if ret:
                 return ret
-        except apt.package.FetchError:
+        except Exception as e:
             logger.debug("Fail to fetch binray %s_%s", _name, _version)
+            logger.debug(str(e))
         return None
 
     def reports(self):
@@ -246,7 +249,9 @@ class DebDownloader(BaseDownloader):
 
         for deb in self.need_upload:
             name, ver, arch = deb.split('_')
-            if not self.repomgr.search_pkg(REPO_BIN, name, ver):
+            if not self.repomgr.search_pkg(REPO_BIN, name, ver, True):
+                if name and ver:
+                    logger.debug('Package %s-%s not found in %s', name, ver, REPO_BIN)
                 if self.repomgr.upload_pkg(REPO_BIN, os.path.join(stx_bin_mirror, deb)):
                     logger.info(' '.join([os.path.join(stx_bin_mirror, deb),
                                           'is uploaded to', REPO_BIN]))