From 53d56b655fc95feb0b6e8c2485fb7c773bc4ee50 Mon Sep 17 00:00:00 2001 From: ZhangXiao Date: Wed, 19 Jan 2022 16:50:41 +0800 Subject: [PATCH] repo_manage.py: add feature to merge several repositories together This will not create a real repository but just a publication, contains all packages of some other repositories. Only used for debootstrap: https://wiki.debian.org/Debootstrap ... Debootstrap can only use one repository for its packages... Story: 2008846 Task: 44318 Test Plan: Pass: Merge several local repos, merge several remote repos(mirror), merge several local and remote repos. Signed-off-by: ZhangXiao Change-Id: If271d431f75716417379321228a49f79e3aaf220 --- build-tools/stx/aptly_deb_usage.py | 89 +++++++++++++++++++++++++++++- build-tools/stx/repo_manage.py | 20 +++++++ 2 files changed, 108 insertions(+), 1 deletion(-) diff --git a/build-tools/stx/aptly_deb_usage.py b/build-tools/stx/aptly_deb_usage.py index d221a83e..94bbb4c7 100755 --- a/build-tools/stx/aptly_deb_usage.py +++ b/build-tools/stx/aptly_deb_usage.py @@ -27,6 +27,7 @@ from typing import Optional PREFIX_LOCAL = 'deb-local-' PREFIX_REMOTE = 'deb-remote-' +PREFIX_MERGE = 'deb-merge-' # Class used to manage aptly data base, it can: # create_remote: Create a repository link to a remote mirror @@ -121,6 +122,79 @@ class Deb_aptly(): if task_state == 'SUCCEEDED': return True + # Create a snapshot based on several others + # name : string, the name of new build snapshot + # source_snapshots: list, snapshots to be merged + # Return False if failed + def __merge_snapshot(self, name, source_snapshots): + '''Merge several snapshots into one, prepare for later deploy.''' + if not name.startswith(PREFIX_MERGE): + self.logger.error('%s did not start with %s, Failed.' % (name, PREFIX_MERGE)) + return False + package_refs = [] + source_snapshots = [x.strip() for x in source_snapshots if x.strip() != ''] + snap_list = self.aptly.snapshots.list() + for snapshot in source_snapshots: + snap_exist = False + for snap in snap_list: + if snap.name == snapshot: + snap_exist = True + package_list = self.aptly.snapshots.list_packages(snap.name, with_deps=False, detailed=False) + for package in package_list: + package_refs.append(package.key) + break + if not snap_exist: + self.logger.error('snapshot %s does not exist, merge failed.' % snapshot) + return False + + # Remove a same name publish if exists + # For exist snapshot called NAME, we will: + # 1, rename it to backup-NAME + # 2, Create a new snapshot: NAME + # 3, delete snapshot backup-name + backup_name = None + publish_list = self.aptly.publish.list() + for publish in publish_list: + if publish.prefix == name: + task = self.aptly.publish.drop(prefix=name, distribution=publish.distribution, force_delete=True) + self.aptly.tasks.wait_for_task_by_id(task.id) + if self.aptly.tasks.show(task.id).state != 'SUCCEEDED': + self.logger.warning('Drop publication failed %s : %s' % (name, self.aptly.tasks.show(task.id).state)) + return False + # Remove the backup snapshot if it exists + snap_list = self.aptly.snapshots.list() + for snap in snap_list: + if snap.name == 'backup-' + name: + backup_name = 'backup-' + name + task = self.aptly.snapshots.delete(snapshotname=backup_name, force=True) + if self.aptly.tasks.show(task.id).state != 'SUCCEEDED': + self.logger.warning('Drop snapshot failed %s : %s' % (backup_name, self.aptly.tasks.show(task.id).state)) + return False + # Rename the snapshot if it exists + for snap in snap_list: + if snap.name == name: + backup_name = 'backup-' + name + self.aptly.tasks.wait_for_task_by_id(self.aptly.snapshots.update(name, backup_name).id) + + # crate a snapshot with package_refs. Duplicate package_refs is harmless. + # Note: The key is "package_refs" instead of "source_snapshots", for function + # "create_from_packages", paramter "source_snapshots" almost has no means. + task = None + task = self.aptly.snapshots.create_from_packages(name, source_snapshots=source_snapshots, package_refs=package_refs) + self.aptly.tasks.wait_for_task_by_id(task.id) + if self.aptly.tasks.show(task.id).state != 'SUCCEEDED': + if backup_name: + self.aptly.tasks.wait_for_task_by_id(self.aptly.snapshots.update(backup_name, name).id) + self.logger.warning('Snapshot for %s creation failed: %s. ' % (name, self.aptly.tasks.show(task.id).state)) + return False + # Remove the backup snapshot if it is created above + if backup_name: + task = self.aptly.snapshots.delete(snapshotname=backup_name, force=True) + self.aptly.tasks.wait_for_task_by_id(task.id) + if self.aptly.tasks.show(task.id).state != 'SUCCEEDED': + self.logger.warning('Drop snapshot failed %s : %s' % (backup_name, self.aptly.tasks.show(task.id).state)) + return True + # Create a snapshot based on "name" with same name # local: True ==> local_repo False ==> remote_mirror # Return False if failed @@ -418,7 +492,7 @@ class Deb_aptly(): # pkg_name: package name # architecture: Architecture of the package, now, only check 'source' or not # pkg_version: the version of the package, None means version insensitive - def pkg_exist(self, repo_list, pkg_name, architecture, pkg_version: Optional[str] = None): + def pkg_exist(self, repo_list, pkg_name, architecture, pkg_version=None): '''Search a package in a bundle of repositories including local repo and remote one.''' for repo_name in repo_list: if repo_name.startswith(PREFIX_LOCAL): @@ -447,6 +521,19 @@ class Deb_aptly(): return True return False + # Merge several repositories into a new one(just snapshot and publish) + # name: the name of the new build snapshot/publish + # source_snapshots: list, snapshots to be merged + def merge_repos(self, name, source_snapshots): + '''Merge several repositories into a new publish.''' + if not name.startswith(PREFIX_MERGE): + self.logger.warning('The name should started with %s.', PREFIX_MERGE) + return None + + if self.__merge_snapshot(name, source_snapshots): + ret = self.__publish_snap(name) + return ret + # deploy a loacl repository # Input: the name of the local repository # Output: None or DebAptDistributionResponse diff --git a/build-tools/stx/repo_manage.py b/build-tools/stx/repo_manage.py index d57fbafd..9641409b 100755 --- a/build-tools/stx/repo_manage.py +++ b/build-tools/stx/repo_manage.py @@ -372,6 +372,12 @@ class RepoMgr(): shutil.rmtree(self.workdir) self.logger.info('local repo can be accessed through: %s ' % repo_str) + # Merge all packages of several repositories into a new publication(aptly) + # NOTE: aptly only. Not find similar feature in pulp... + def merge(self, name, source_snapshots): + '''Merge several repositories into a new aptly publication.''' + self.repo.merge_repos(name, source_snapshots.split(',')) + # Construct a repository mirror to an upstream Debian repository # kwargs:url: URL of the upstream repo (http://deb.debian.org/debian) # kwargs:distribution: the distribution of the repo (bullseye) @@ -627,6 +633,11 @@ def _handleMirror(args): repomgr.mirror(args.name, **kwargs) +def _handleMerge(args): + repomgr = RepoMgr('aptly', REPOMGR_URL, '/tmp', applogger) + repomgr.merge(args.name, args.repo_list) + + def _handleUploadPkg(args): repomgr = RepoMgr('aptly', REPOMGR_URL, '/tmp', applogger) repomgr.upload_pkg(args.repository, args.package) @@ -722,6 +733,14 @@ def subcmd_mirror(subparsers): mirror_parser.set_defaults(handle=_handleMirror) +def subcmd_merge(subparsers): + merge_parser = subparsers.add_parser('merge', + help='Merge several repositories into a new publication.\n\n') + merge_parser.add_argument('--name', '-n', help='Name of the new merged publication', required=True) + merge_parser.add_argument('--repo_list', '-l', help='a set of repos, seperate by comma', required=True) + merge_parser.set_defaults(handle=_handleMerge) + + def main(): # command line arguments parser = argparse.ArgumentParser(add_help=False, @@ -735,6 +754,7 @@ def main(): subcmd_download(subparsers) subcmd_sync(subparsers) subcmd_mirror(subparsers) + subcmd_merge(subparsers) clean_parser = subparsers.add_parser('clean', help='Clear all aptly repos.\n\n') clean_parser.set_defaults(handle=_handleClean)