starlingx/root
Code Issues Proposed changes

build-image: Merge the local repositories and feed to debootstrap

Browse Source 
The function of aptly snapshot merging is used to merge
the local repositories 'deb-local-binary' and
'deb-local-build' to a published unified snapshot.
This snapshot will be feeded to 'debootstrap' in LAT
and to do the rootfs.

In order to meet this changes, the corresponding changes
are also done in build-pkgs and downloader.

It should be noted that this commit can not fix the
package conflicting issues, the purpose is to replace
the upstream official repo for debootstrap with local
repositories, so the developer can fix the package
conflicit issues through modifying the local repositories.

Test Plan:
Pass: The replacement is not be triggered until
      'deb-merge-all' is set to 'debootstrap-mirror' in
      base-bullseye.yaml or in base-initramfs-bullseye.yaml
Pass: If triggered, an aptly snapshot
      'http://<REPOMGR_DEPLOY_URL>/deb-merge-all' is published
      and consumed by debootstrap in LAT, the debootstrap works as normal

Pass: The messages "WARNING: Drop duplicate package" reports by
      repo_manage is used to sort out the packages list

Depends-On: https://review.opendev.org/c/starlingx/root/+/826685

Story: 2008846
Task: 44359

Signed-off-by: hbai <haiqing.bai@windriver.com>
Change-Id: I4038135e4148277a3065397a571e25901086798a
This commit is contained in:
hbai 2022-01-29 17:15:46 +08:00
parent c1d3406c4f
commit 9f344ff475
3 changed files with 80 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

67
build-tools/stx/build-image
View File

@ -25,6 +25,7 @@ import time
import utils
import yaml
REPO_ALL = 'deb-merge-all'
REPO_BINARY = 'deb-local-binary'
REPO_BUILD = 'deb-local-build'
DEB_CONFIG_DIR = 'stx-tools/debian-mirror-tools/config/'
@ -34,6 +35,50 @@ img_pkgs = []
logger = logging.getLogger('build-image')
utils.set_logger(logger)
def merge_local_repos(repomgr):
logger.debug('Calls repo manager to create/udpate the snapshot %s which is merged from local repositories', REPO_ALL)
# REPO_BUILD is higher priority than REPO_BINARY for repomgr to select package
try:
pubname = repomgr.merge(REPO_ALL, ','.join([REPO_BUILD, REPO_BINARY]))
except Exception as e:
logger.error(str(e))
logger.error('Exception when repo_manager creates/updates snapshot %s', REPO_ALL)
return False
if pubname:
logger.debug('repo manager successfully created/updated snapshot %s', REPO_ALL)
else:
logger.debug('repo manager failed to create/update snapshot %s', REPO_ALL)
return False
return True
def update_debootstrap_mirror(img_yaml):
repomgr_url = os.environ.get('REPOMGR_DEPLOY_URL')
if not repomgr_url:
logger.error('REPOMGR_URL is not in current sys ENV')
return False
repo_all_url = '/'.join([repomgr_url, REPO_ALL])
try:
with open(img_yaml) as f:
yaml_doc = yaml.safe_load(f)
if not yaml_doc['debootstrap-mirror']:
logger.warning("There is not debootstrap-mirror in %s", img_yaml)
else:
mirror = yaml_doc['debootstrap-mirror']
if mirror == REPO_ALL:
yaml_doc['debootstrap-mirror'] = repo_all_url
with open(img_yaml, 'w') as f:
yaml.safe_dump(yaml_doc, f, default_flow_style=False, sort_keys=False)
logger.debug('Updating %s, setting debootstrap_mirror to %s', img_yaml, repo_all_url)
return True
except IOError as e:
logger.error(str(e))
logger.debug('Failed to update %s, could not set debootstrap_mirror to %s', img_yaml, repo_all_url)
return False
def update_ostree_osname(img_yaml):
ostree_osname = os.environ.get('OSTREE_OSNAME')
@ -53,6 +98,7 @@ def update_ostree_osname(img_yaml):
logger.debug(' '.join(['Update', img_yaml, 'to update the ostree_osname']))
return True
def feed_lat_src_repos(img_yaml, repo_url):
if not os.path.exists(img_yaml):
logger.error(' '.join(['LAT yaml file', img_yaml, 'does not exist']))
@ -97,7 +143,7 @@ def check_base_os_binaries(repomgr):
'does not exist']))
return False
results = query_repo(repomgr, REPO_BINARY, base_bins_list)
results = verify_pkgs_in_repo(repomgr, REPO_BINARY, base_bins_list)
if results:
logger.error("====OS binaries checking fail:")
for deb in results:
@ -117,7 +163,7 @@ def check_stx_binaries(repomgr, btype='std'):
# Assume no such list here means ok
return True
results = query_repo(repomgr, REPO_BINARY, stx_bins_list)
results = verify_pkgs_in_repo(repomgr, REPO_BINARY, stx_bins_list)
if results:
logger.error("====STX binaries checking fail:")
for deb in results:
@ -136,7 +182,7 @@ def check_stx_patched(repomgr, btype='std'):
'does not exist']))
return False
results = query_repo(repomgr, REPO_BUILD, stx_patched_list)
results = verify_pkgs_in_repo(repomgr, REPO_BUILD, stx_patched_list)
if results:
logger.error("====STX patched packages checking fail:")
for deb in results:
@ -147,7 +193,7 @@ def check_stx_patched(repomgr, btype='std'):
return True
def query_repo(repomgr, repo_name, pkg_list_path):
def verify_pkgs_in_repo(repomgr, repo_name, pkg_list_path):
failed_pkgs = []
with open(pkg_list_path, 'r') as flist:
lines = list(line for line in (lpkg.strip() for lpkg in flist) if line)
@ -169,7 +215,7 @@ def query_repo(repomgr, repo_name, pkg_list_path):
'is missing in local binary repo']))
failed_pkgs.append(pkg_name)
else:
if repomgr.search_pkg(repo_name, name):
if repomgr.search_pkg(repo_name, name, None, True):
img_pkgs.append(name)
logger.debug(''.join(['Found package with name:', name]))
else:
@ -205,8 +251,10 @@ if __name__ == "__main__":
logger.error(e.msg)
usage()
rmg_logger = logging.getLogger('repo_manager')
utils.set_logger(rmg_logger)
repo_manager = repo_manage.RepoMgr('aptly', os.environ.get('REPOMGR_URL'),
'/tmp/', logger)
'/tmp/', rmg_logger)
repo_manager.upload_pkg(REPO_BUILD, None)
repo_manager.upload_pkg(REPO_BINARY, None)
@ -242,9 +290,16 @@ if __name__ == "__main__":
shutil.copyfile(base_yaml, lat_yaml)
shutil.copyfile(base_initramfs_yaml, lat_initramfs_yaml)
except IOError as e:
logger.error(str(e))
logger.error('Fail to copy image yaml files to /localdisk/deploy')
sys.exit(1)
if merge_local_repos(repo_manager):
if update_debootstrap_mirror(lat_yaml):
logger.debug("Debootstrap switches to mirror %s in %s", REPO_ALL, lat_yaml)
if update_debootstrap_mirror(lat_initramfs_yaml):
logger.debug("Debootstrap switches to mirror %s in %s", REPO_ALL, lat_initramfs_yaml)
binary_repo_url = ''.join(['deb [trusted=yes] ',
os.environ.get('REPOMGR_DEPLOY_URL'),
REPO_BINARY, ' bullseye main'])

7
build-tools/stx/build-pkgs
View File

@ -450,12 +450,9 @@ class BuildController():
msg = ''.join(['package ', pkg_item[0], '(', pkg_item[1], ')'])
logger.info(' '.join(['Searching for binary', msg, 'in repository', REPO_BUILD]))
if self.kits['repo_mgr'].search_pkg(REPO_BUILD, pkg_item[0],
pkg_item[1]):
if self.kits['repo_mgr'].search_pkg(REPO_BUILD, pkg_item[0], None, True):
logger.info('Found binary %s in repository %s', msg, REPO_BUILD)
if self.kits['repo_mgr'].delete_pkg(REPO_BUILD,
pkg_item[0], 'binary',
pkg_item[1]):
if self.kits['repo_mgr'].delete_pkg(REPO_BUILD, pkg_item[0], 'binary', None):
logger.info('Successfully deleted binary %s from repository %s',
msg, REPO_BUILD)
else:

13
build-tools/stx/downloader
View File

@ -126,8 +126,10 @@ class BaseDownloader():
self.dl_need = []
self.dl_success = []
self.dl_failed = []
rlogger = logging.getLogger('repo_manager')
utils.set_logger(rlogger)
self.repomgr = repo_manage.RepoMgr('aptly', os.environ.get('REPOMGR_URL'),
'/tmp/', logger)
'/tmp/', rlogger)
self.repomgr.upload_pkg(REPO_BIN, None)
def clean(self):
@ -177,6 +179,7 @@ class DebDownloader(BaseDownloader):
self.repomgr.upload_pkg(REPO_BIN, None)
def download(self, _name, _version):
try:
package = self.apt_cache[_name]
candidate = package.versions.get(_version)
if not candidate:
@ -186,12 +189,12 @@ class DebDownloader(BaseDownloader):
return None
package.candidate = candidate
try:
ret = package.candidate.fetch_binary(self.dl_dir)
if ret:
return ret
except apt.package.FetchError:
except Exception as e:
logger.debug("Fail to fetch binray %s_%s", _name, _version)
logger.debug(str(e))
return None
def reports(self):
@ -246,7 +249,9 @@ class DebDownloader(BaseDownloader):
for deb in self.need_upload:
name, ver, arch = deb.split('_')
if not self.repomgr.search_pkg(REPO_BIN, name, ver):
if not self.repomgr.search_pkg(REPO_BIN, name, ver, True):
if name and ver:
logger.debug('Package %s-%s not found in %s', name, ver, REPO_BIN)
if self.repomgr.upload_pkg(REPO_BIN, os.path.join(stx_bin_mirror, deb)):
logger.info(' '.join([os.path.join(stx_bin_mirror, deb),
'is uploaded to', REPO_BIN]))