repo_manage.py: add feature to merge several repositories together

This will not create a real repository but just a publication,
contains all packages of some other repositories. Only used for
debootstrap:

https://wiki.debian.org/Debootstrap
... Debootstrap can only use one repository for its packages...

Story: 2008846
Task: 44318

Test Plan:
Pass: Merge several local repos, merge several remote repos(mirror),
      merge several local and remote repos.

Signed-off-by: ZhangXiao <xiao.zhang@windriver.com>
Change-Id: If271d431f75716417379321228a49f79e3aaf220

build-tools/stx/aptly_deb_usage.py

@@ -27,6 +27,7 @@ from typing import Optional
 
 PREFIX_LOCAL = 'deb-local-'
 PREFIX_REMOTE = 'deb-remote-'
+PREFIX_MERGE = 'deb-merge-'
 
 # Class used to manage aptly data base, it can:
 #     create_remote: Create a repository link to a remote mirror
@@ -121,6 +122,79 @@ class Deb_aptly():
             if task_state == 'SUCCEEDED':
                 return True
 
+    # Create a snapshot based on several others
+    # name : string, the name of new build snapshot
+    # source_snapshots: list, snapshots to be merged
+    # Return False if failed
+    def __merge_snapshot(self, name, source_snapshots):
+        '''Merge several snapshots into one, prepare for later deploy.'''
+        if not name.startswith(PREFIX_MERGE):
+            self.logger.error('%s did not start with %s, Failed.' % (name, PREFIX_MERGE))
+            return False
+        package_refs = []
+        source_snapshots = [x.strip() for x in source_snapshots if x.strip() != '']
+        snap_list = self.aptly.snapshots.list()
+        for snapshot in source_snapshots:
+            snap_exist = False
+            for snap in snap_list:
+                if snap.name == snapshot:
+                    snap_exist = True
+                    package_list = self.aptly.snapshots.list_packages(snap.name, with_deps=False, detailed=False)
+                    for package in package_list:
+                        package_refs.append(package.key)
+                    break
+            if not snap_exist:
+                self.logger.error('snapshot %s does not exist, merge failed.' % snapshot)
+                return False
+
+        # Remove a same name publish if exists
+        # For exist snapshot called NAME, we will:
+        # 1, rename it to backup-NAME
+        # 2, Create a new snapshot: NAME
+        # 3, delete snapshot backup-name
+        backup_name = None
+        publish_list = self.aptly.publish.list()
+        for publish in publish_list:
+            if publish.prefix == name:
+                task = self.aptly.publish.drop(prefix=name, distribution=publish.distribution, force_delete=True)
+                self.aptly.tasks.wait_for_task_by_id(task.id)
+                if self.aptly.tasks.show(task.id).state != 'SUCCEEDED':
+                    self.logger.warning('Drop publication failed %s : %s' % (name, self.aptly.tasks.show(task.id).state))
+                    return False
+        # Remove the backup snapshot if it exists
+        snap_list = self.aptly.snapshots.list()
+        for snap in snap_list:
+            if snap.name == 'backup-' + name:
+                backup_name = 'backup-' + name
+                task = self.aptly.snapshots.delete(snapshotname=backup_name, force=True)
+                if self.aptly.tasks.show(task.id).state != 'SUCCEEDED':
+                    self.logger.warning('Drop snapshot failed %s : %s' % (backup_name, self.aptly.tasks.show(task.id).state))
+                    return False
+        # Rename the snapshot if it exists
+        for snap in snap_list:
+            if snap.name == name:
+                backup_name = 'backup-' + name
+                self.aptly.tasks.wait_for_task_by_id(self.aptly.snapshots.update(name, backup_name).id)
+
+        # crate a snapshot with package_refs. Duplicate package_refs is harmless.
+        # Note: The key is "package_refs" instead of "source_snapshots", for function
+        #       "create_from_packages", paramter "source_snapshots" almost has no means.
+        task = None
+        task = self.aptly.snapshots.create_from_packages(name, source_snapshots=source_snapshots, package_refs=package_refs)
+        self.aptly.tasks.wait_for_task_by_id(task.id)
+        if self.aptly.tasks.show(task.id).state != 'SUCCEEDED':
+            if backup_name:
+                self.aptly.tasks.wait_for_task_by_id(self.aptly.snapshots.update(backup_name, name).id)
+            self.logger.warning('Snapshot for %s creation failed: %s. ' % (name, self.aptly.tasks.show(task.id).state))
+            return False
+        # Remove the backup snapshot if it is created above
+        if backup_name:
+            task = self.aptly.snapshots.delete(snapshotname=backup_name, force=True)
+            self.aptly.tasks.wait_for_task_by_id(task.id)
+            if self.aptly.tasks.show(task.id).state != 'SUCCEEDED':
+                self.logger.warning('Drop snapshot failed %s : %s' % (backup_name, self.aptly.tasks.show(task.id).state))
+        return True
+
     # Create a snapshot based on "name" with same name
     # local: True ==> local_repo False ==> remote_mirror
     # Return False if failed
@@ -418,7 +492,7 @@ class Deb_aptly():
     # pkg_name: package name
     # architecture: Architecture of the package, now, only check 'source' or not
     # pkg_version:  the version of the package, None means version insensitive
-    def pkg_exist(self, repo_list, pkg_name, architecture, pkg_version: Optional[str] = None):
+    def pkg_exist(self, repo_list, pkg_name, architecture, pkg_version=None):
         '''Search a package in a bundle of repositories including local repo and remote one.'''
         for repo_name in repo_list:
             if repo_name.startswith(PREFIX_LOCAL):
@@ -447,6 +521,19 @@ class Deb_aptly():
                             return True
         return False
 
+    # Merge several repositories into a new one(just snapshot and publish)
+    # name: the name of the new build snapshot/publish
+    # source_snapshots: list, snapshots to be merged
+    def merge_repos(self, name, source_snapshots):
+        '''Merge several repositories into a new publish.'''
+        if not name.startswith(PREFIX_MERGE):
+            self.logger.warning('The name should started with %s.', PREFIX_MERGE)
+            return None
+
+        if self.__merge_snapshot(name, source_snapshots):
+            ret = self.__publish_snap(name)
+            return ret
+
     # deploy a loacl repository
     # Input: the name of the local repository
     # Output: None or DebAptDistributionResponse

build-tools/stx/repo_manage.py

@@ -372,6 +372,12 @@ class RepoMgr():
             shutil.rmtree(self.workdir)
         self.logger.info('local repo can be accessed through: %s ' % repo_str)
 
+    # Merge all packages of several repositories into a new publication(aptly)
+    # NOTE: aptly only. Not find similar feature in pulp...
+    def merge(self, name, source_snapshots):
+        '''Merge several repositories into a new aptly publication.'''
+        self.repo.merge_repos(name, source_snapshots.split(','))
+
     # Construct a repository mirror to an upstream Debian repository
     # kwargs:url: URL of the upstream repo (http://deb.debian.org/debian)
     # kwargs:distribution: the distribution of the repo (bullseye)
@@ -627,6 +633,11 @@ def _handleMirror(args):
     repomgr.mirror(args.name, **kwargs)
 
 
+def _handleMerge(args):
+    repomgr = RepoMgr('aptly', REPOMGR_URL, '/tmp', applogger)
+    repomgr.merge(args.name, args.repo_list)
+
+
 def _handleUploadPkg(args):
     repomgr = RepoMgr('aptly', REPOMGR_URL, '/tmp', applogger)
     repomgr.upload_pkg(args.repository, args.package)
@@ -722,6 +733,14 @@ def subcmd_mirror(subparsers):
     mirror_parser.set_defaults(handle=_handleMirror)
 
 
+def subcmd_merge(subparsers):
+    merge_parser = subparsers.add_parser('merge',
+                                         help='Merge several repositories into a new publication.\n\n')
+    merge_parser.add_argument('--name', '-n', help='Name of the new merged publication', required=True)
+    merge_parser.add_argument('--repo_list', '-l', help='a set of repos, seperate by comma', required=True)
+    merge_parser.set_defaults(handle=_handleMerge)
+
+
 def main():
     # command line arguments
     parser = argparse.ArgumentParser(add_help=False,
@@ -735,6 +754,7 @@ def main():
     subcmd_download(subparsers)
     subcmd_sync(subparsers)
     subcmd_mirror(subparsers)
+    subcmd_merge(subparsers)
 
     clean_parser = subparsers.add_parser('clean', help='Clear all aptly repos.\n\n')
     clean_parser.set_defaults(handle=_handleClean)