starlingx/root
Code Issues Proposed changes

repo_manage.py: add feature to merge several repositories together

Browse Source 
This will not create a real repository but just a publication,
contains all packages of some other repositories. Only used for
debootstrap:

https://wiki.debian.org/Debootstrap
... Debootstrap can only use one repository for its packages...

Story: 2008846
Task: 44318

Test Plan:
Pass: Merge several local repos, merge several remote repos(mirror),
      merge several local and remote repos.

Signed-off-by: ZhangXiao <xiao.zhang@windriver.com>
Change-Id: If271d431f75716417379321228a49f79e3aaf220
This commit is contained in:
ZhangXiao 2022-01-19 16:50:41 +08:00
parent 42624afcfe
commit 53d56b655f
2 changed files with 108 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

89
build-tools/stx/aptly_deb_usage.py
View File

@ -27,6 +27,7 @@ from typing import Optional
PREFIX_LOCAL = 'deb-local-'
PREFIX_REMOTE = 'deb-remote-'
PREFIX_MERGE = 'deb-merge-'
# Class used to manage aptly data base, it can:
# create_remote: Create a repository link to a remote mirror
@ -121,6 +122,79 @@ class Deb_aptly():
if task_state == 'SUCCEEDED':
return True
# Create a snapshot based on several others
# name : string, the name of new build snapshot
# source_snapshots: list, snapshots to be merged
# Return False if failed
def __merge_snapshot(self, name, source_snapshots):
'''Merge several snapshots into one, prepare for later deploy.'''
if not name.startswith(PREFIX_MERGE):
self.logger.error('%s did not start with %s, Failed.' % (name, PREFIX_MERGE))
return False
package_refs = []
source_snapshots = [x.strip() for x in source_snapshots if x.strip() != '']
snap_list = self.aptly.snapshots.list()
for snapshot in source_snapshots:
snap_exist = False
for snap in snap_list:
if snap.name == snapshot:
snap_exist = True
package_list = self.aptly.snapshots.list_packages(snap.name, with_deps=False, detailed=False)
for package in package_list:
package_refs.append(package.key)
break
if not snap_exist:
self.logger.error('snapshot %s does not exist, merge failed.' % snapshot)
return False
# Remove a same name publish if exists
# For exist snapshot called NAME, we will:
# 1, rename it to backup-NAME
# 2, Create a new snapshot: NAME
# 3, delete snapshot backup-name
backup_name = None
publish_list = self.aptly.publish.list()
for publish in publish_list:
if publish.prefix == name:
task = self.aptly.publish.drop(prefix=name, distribution=publish.distribution, force_delete=True)
self.aptly.tasks.wait_for_task_by_id(task.id)
if self.aptly.tasks.show(task.id).state != 'SUCCEEDED':
self.logger.warning('Drop publication failed %s : %s' % (name, self.aptly.tasks.show(task.id).state))
return False
# Remove the backup snapshot if it exists
snap_list = self.aptly.snapshots.list()
for snap in snap_list:
if snap.name == 'backup-' + name:
backup_name = 'backup-' + name
task = self.aptly.snapshots.delete(snapshotname=backup_name, force=True)
if self.aptly.tasks.show(task.id).state != 'SUCCEEDED':
self.logger.warning('Drop snapshot failed %s : %s' % (backup_name, self.aptly.tasks.show(task.id).state))
return False
# Rename the snapshot if it exists
for snap in snap_list:
if snap.name == name:
backup_name = 'backup-' + name
self.aptly.tasks.wait_for_task_by_id(self.aptly.snapshots.update(name, backup_name).id)
# crate a snapshot with package_refs. Duplicate package_refs is harmless.
# Note: The key is "package_refs" instead of "source_snapshots", for function
# "create_from_packages", paramter "source_snapshots" almost has no means.
task = None
task = self.aptly.snapshots.create_from_packages(name, source_snapshots=source_snapshots, package_refs=package_refs)
self.aptly.tasks.wait_for_task_by_id(task.id)
if self.aptly.tasks.show(task.id).state != 'SUCCEEDED':
if backup_name:
self.aptly.tasks.wait_for_task_by_id(self.aptly.snapshots.update(backup_name, name).id)
self.logger.warning('Snapshot for %s creation failed: %s. ' % (name, self.aptly.tasks.show(task.id).state))
return False
# Remove the backup snapshot if it is created above
if backup_name:
task = self.aptly.snapshots.delete(snapshotname=backup_name, force=True)
self.aptly.tasks.wait_for_task_by_id(task.id)
if self.aptly.tasks.show(task.id).state != 'SUCCEEDED':
self.logger.warning('Drop snapshot failed %s : %s' % (backup_name, self.aptly.tasks.show(task.id).state))
return True
# Create a snapshot based on "name" with same name
# local: True ==> local_repo False ==> remote_mirror
# Return False if failed
@ -418,7 +492,7 @@ class Deb_aptly():
# pkg_name: package name
# architecture: Architecture of the package, now, only check 'source' or not
# pkg_version: the version of the package, None means version insensitive
def pkg_exist(self, repo_list, pkg_name, architecture, pkg_version: Optional[str] = None):
def pkg_exist(self, repo_list, pkg_name, architecture, pkg_version=None):
'''Search a package in a bundle of repositories including local repo and remote one.'''
for repo_name in repo_list:
if repo_name.startswith(PREFIX_LOCAL):
@ -447,6 +521,19 @@ class Deb_aptly():
return True
return False
# Merge several repositories into a new one(just snapshot and publish)
# name: the name of the new build snapshot/publish
# source_snapshots: list, snapshots to be merged
def merge_repos(self, name, source_snapshots):
'''Merge several repositories into a new publish.'''
if not name.startswith(PREFIX_MERGE):
self.logger.warning('The name should started with %s.', PREFIX_MERGE)
return None
if self.__merge_snapshot(name, source_snapshots):
ret = self.__publish_snap(name)
return ret
# deploy a loacl repository
# Input: the name of the local repository
# Output: None or DebAptDistributionResponse

20
build-tools/stx/repo_manage.py
View File

@ -372,6 +372,12 @@ class RepoMgr():
shutil.rmtree(self.workdir)
self.logger.info('local repo can be accessed through: %s ' % repo_str)
# Merge all packages of several repositories into a new publication(aptly)
# NOTE: aptly only. Not find similar feature in pulp...
def merge(self, name, source_snapshots):
'''Merge several repositories into a new aptly publication.'''
self.repo.merge_repos(name, source_snapshots.split(','))
# Construct a repository mirror to an upstream Debian repository
# kwargs:url: URL of the upstream repo (http://deb.debian.org/debian)
# kwargs:distribution: the distribution of the repo (bullseye)
@ -627,6 +633,11 @@ def _handleMirror(args):
repomgr.mirror(args.name, **kwargs)
def _handleMerge(args):
repomgr = RepoMgr('aptly', REPOMGR_URL, '/tmp', applogger)
repomgr.merge(args.name, args.repo_list)
def _handleUploadPkg(args):
repomgr = RepoMgr('aptly', REPOMGR_URL, '/tmp', applogger)
repomgr.upload_pkg(args.repository, args.package)
@ -722,6 +733,14 @@ def subcmd_mirror(subparsers):
mirror_parser.set_defaults(handle=_handleMirror)
def subcmd_merge(subparsers):
merge_parser = subparsers.add_parser('merge',
help='Merge several repositories into a new publication.\n\n')
merge_parser.add_argument('--name', '-n', help='Name of the new merged publication', required=True)
merge_parser.add_argument('--repo_list', '-l', help='a set of repos, seperate by comma', required=True)
merge_parser.set_defaults(handle=_handleMerge)
def main():
# command line arguments
parser = argparse.ArgumentParser(add_help=False,
@ -735,6 +754,7 @@ def main():
subcmd_download(subparsers)
subcmd_sync(subparsers)
subcmd_mirror(subparsers)
subcmd_merge(subparsers)
clean_parser = subparsers.add_parser('clean', help='Clear all aptly repos.\n\n')
clean_parser.set_defaults(handle=_handleClean)