starlingx/root
Code Issues Proposed changes

downloader: Initial created to download packages

Browse Source 
This downloader merges the downloading functions both for
debian binary packages and STX source recipes.

Two local mirrors will be created according with the ENV
variable 'STX_MIRROR' which is the volume path mounted by
the container builder.
Below are the default things:
STX_MIRROR: /import/mirrors/starlingx
mirror for STX source: /import/mirrors/starlingx/sources
mirror for binary packages: /import/mirrors/starlingx/binaries

Test Plan:

Pass: successfully downloaded available STX source recipes
Pass: successfully downloaded binary packages listed in
      base_bullseye.lst

Story: 2008846
Task: 44220

Signed-off-by: hbai <haiqing.bai@windriver.com>
Depends-On: I5f877b44c32e31d93651cf98c789a6a248e0a530
Change-Id: I7a2738fae44b1d0b9cd252ac83cc503c300d9c45
This commit is contained in:
hbai 2021-12-22 16:25:56 +08:00
parent 390efe87cd
commit 1a8cb7e7e8
1 changed files with 401 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

401
build-tools/stx/downloader Executable file
View File

@ -0,0 +1,401 @@
#!/usr/bin/python3
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Copyright (C) 2021 Wind River Systems,Inc
import apt
import argparse
import debrepack
import logging
import os
import repo_manage
import shutil
import signal
import sys
import utils
DEFAULT_ARCH = 'amd64'
REPO_SOURCE = 'deb-local-source'
REPO_BIN = 'deb-local-binary'
mirror_root = os.environ.get('STX_MIRROR')
stx_src_mirror = os.path.join(mirror_root, 'sources')
stx_bin_mirror = os.path.join(mirror_root, 'binaries')
all_binary_lists = ['base-bullseye.lst', 'os-std.lst', 'os-rt.lst']
types_pkg_dirs = ['debian_pkg_dirs', 'debian_pkg_dirs_rt', 'debian_pkg_dirs_installer']
logger = logging.getLogger('downloader')
utils.set_logger(logger)
def get_downloaded(dl_dir, dl_type):
"""
Browse and get the already downloaded binary or source
packages in dl_dir
dl_dir: mirror dir
dl_type: binary or source
return: list of downloaded targets
"""
dl_list = []
if not os.path.exists(dl_dir):
return []
if dl_type == 'source':
logger.debug('debrepack will check the whole source mirror')
for file in os.listdir(dl_dir):
if dl_type == 'binary' and file.endswith('.deb'):
dl_list.append(file)
return dl_list
def get_pkgs_from_list(root_dir, list_file):
"""
Read each lines in debian_pkg_dirs_<type> and add it
to the map, for example:
entries { 'dhcp': '<path to>/stx/integ/base/dhcp',
'tsconfig': '<path to>/config/tsconfig'}
"""
entries = {}
try:
with open(list_file, 'r') as flist:
lines = list(line for line in (p.strip() for p in flist) if line)
except Exception as e:
logger.error(str(e))
else:
for entry in lines:
entry = entry.strip()
if entry.startswith('#'):
continue
entries[os.path.basename(entry)] = os.path.join(root_dir, entry)
return entries
def get_all_stx_pkgs():
"""
Scan all STX source layers to get all buildable packages
Params: None
Return:
Map of all STX buildable packages and path to debian folder
"""
pkgs = {}
stx_root = os.path.join(os.environ.get('MY_REPO_ROOT_DIR'), 'cgcs-root/stx')
for root, dirs, files in os.walk(stx_root):
if dirs:
pass
for r in files:
# Find all types of package dirs?
if r in types_pkg_dirs:
pkgs_file = os.path.join(root, r)
pkgs.update(get_pkgs_from_list(root, pkgs_file))
return pkgs
def get_all_binary_list():
"""
Return all binary packages listed in base-bullseye.lst, os-std.lst,os-rt.lst
"""
bin_list = []
stx_config = os.path.join(os.environ.get('MY_REPO_ROOT_DIR'),
'stx-tools/debian-mirror-tools/config/debian')
for root, dirs, files in os.walk(stx_config):
if dirs:
pass
for r in files:
if r in all_binary_lists:
bin_list.append(os.path.join(root, r))
return bin_list
class BaseDownloader():
def __init__(self, arch, _dl_dir, clean):
self.dl_dir = _dl_dir
self.arch = arch
self.clean_mirror = clean
self.dl_need = []
self.dl_success = []
self.dl_failed = []
self.repomgr = repo_manage.RepoMgr('aptly', os.environ.get('REPOMGR_URL'),
'/tmp/', logger)
self.repomgr.upload_pkg(REPO_BIN, None)
def clean(self):
if os.path.exists(self.dl_dir):
if self.clean_mirror:
try:
shutil.rmtree(self.dl_dir)
except Exception as e:
logger.error(str(e))
logger.critical("Failed to clean mirror %s", self.dl_dir)
sys.exit(1)
else:
logger.debug("Successfully cleaned mirror %s", self.dl_dir)
os.makedirs(self.dl_dir, exist_ok=True)
def reports(self):
ret = 0
if len(self.dl_need):
logger.info("++++++++++++++++++++++++++++++++++++++++++++++++++")
logger.info("All packages need to be downloaded: %d", len(self.dl_need))
if len(self.dl_success):
logger.info("++++++++++++++++++++++++++++++++++++++++++++++++++")
logger.info("Successfully downloaded packages: %d", len(self.dl_success))
for dlobj in sorted(self.dl_success):
logger.info(dlobj.strip())
failed_list = list(set(self.dl_need) - set(self.dl_success))
if len(failed_list):
logger.error("+++++++++++++++++++++++++++++++++++++++++++++++++")
logger.error("Failed to download packages %d", len(failed_list))
ret = 1
for dlobj in sorted(failed_list):
logger.error(dlobj.strip())
return ret
class DebDownloader(BaseDownloader):
def __init__(self, arch, _dl_dir, force, _bin_lists):
super(DebDownloader, self).__init__(arch, _dl_dir, force)
self.need_download = []
self.downloaded = []
self.need_upload = []
self.bin_lists = _bin_lists
self.apt_cache = apt.cache.Cache()
if self.repomgr:
self.repomgr.upload_pkg(REPO_BIN, None)
def download(self, _name, _version):
package = self.apt_cache[_name]
candidate = package.versions.get(_version)
if not candidate:
logger.error(' '.join(['Fail to download', _name,
'with wrong version', _version, '?']))
logger.error('May need to update the package list file')
return None
package.candidate = candidate
try:
ret = package.candidate.fetch_binary(self.dl_dir)
if ret:
return ret
except apt.package.FetchError:
logger.debug("Fail to fetch binray %s_%s", _name, _version)
return None
def reports(self):
if len(self.bin_lists):
logger.info("All binary lists are:")
for blist in self.bin_lists:
logger.info(blist)
logger.info("Show result for binary download:")
return super(DebDownloader, self).reports()
def download_list(self, list_file):
if not os.path.exists(list_file):
return
self.downloaded = get_downloaded(self.dl_dir, 'binary')
with open(list_file) as flist:
lines = list(line for line in (lpkg.strip() for lpkg in flist) if line)
for pkg in lines:
pkg = pkg.strip()
if pkg.startswith('#'):
continue
pkg_name_array = pkg.split()
pkg_name = pkg_name_array[0]
if len(pkg_name_array) == 1:
logger.error("The package version of %s should be defined", pkg_name)
logger.error("Please update the list file %s", list_file)
sys.exit(1)
# strip epoch
pkg_ver = pkg_name_array[1].split(":")[-1]
# current default arch is 'amd64'
pname_arch = '_'.join([pkg_name, pkg_ver, self.arch]) + '.deb'
pname_all = ''.join([pkg_name, '_', pkg_ver, '_all.deb'])
self.dl_need.append(pkg_name + '_' + pkg_ver)
if self.downloaded and pname_arch in self.downloaded:
logger.debug(''.join([pkg_name, '_', pkg_ver,
' has been downloaded, skip']))
self.dl_success.append(pkg_name + '_' + pkg_ver)
self.need_upload.append(pname_arch)
else:
if self.downloaded and pname_all in self.downloaded:
logger.debug(''.join([pkg_name, '_', pkg_ver,
' has been downloaded, skip']))
self.need_upload.append(pname_all)
self.dl_success.append(pkg_name + '_' + pkg_ver)
else:
# Tests show that the 'epoch' should be taken when
# fetch the package with 'apt' module, there is not 'epoch'
# in the dowloaded package name. This also requires the 'epoch'
# should be defined in the package list file with ':'
self.need_download.append(pkg_name + '_' + pkg_name_array[1])
for deb in self.need_upload:
name, ver, arch = deb.split('_')
if not self.repomgr.search_pkg(REPO_BIN, name, ver):
if self.repomgr.upload_pkg(REPO_BIN, os.path.join(stx_bin_mirror, deb)):
logger.info(' '.join([os.path.join(stx_bin_mirror, deb),
'is uploaded to', REPO_BIN]))
else:
logger.info(' '.join([os.path.join(stx_bin_mirror, deb),
'fail to uploaded to', REPO_BIN]))
for deb in self.need_download:
logger.debug(' '.join(['package', deb, 'is need to be downloaded']))
debnames = deb.split('_')
ret = self.download(debnames[0], debnames[1])
if ret:
logger.info(''.join([debnames[0], '_', debnames[1], ' download ok']))
# strip epoch
deb_ver = debnames[1].split(":")[-1]
self.dl_success.append('_'.join([debnames[0], deb_ver]))
if self.repomgr.upload_pkg(REPO_BIN, ret):
logger.info(''.join([debnames[0], '_', debnames[1], ' is uploaded to ', REPO_BIN]))
else:
logger.error(''.join([debnames[0], '_', debnames[1], ' fail to upload to ', REPO_BIN]))
else:
self.dl_failed.append(deb)
def start(self):
"""Here define:
the complete set of binaries = base_bullseye.lst
+ <layer>/os-std.lst
+ <layer>/os-rt.lst
"""
super(DebDownloader, self).clean()
if len(self.bin_lists):
for bin_list in self.bin_lists:
self.download_list(bin_list)
else:
logger.error("There are no lists of binary packages found")
sys.exit(1)
class SrcDownloader(BaseDownloader):
def __init__(self, arch, _dl_dir, force):
super(SrcDownloader, self).__init__(arch, _dl_dir, force)
self.parser = None
def prepare(self):
build_dir = os.path.join(os.environ.get('MY_BUILD_PKG_DIR'))
os.makedirs(build_dir, exist_ok=True)
recipes_dir = os.path.join(os.environ.get('MY_BUILD_PKG_DIR'), 'recipes')
os.makedirs(recipes_dir, exist_ok=True)
if not self.parser:
try:
self.parser = debrepack.Parser(build_dir,
recipes_dir, 'debug')
except Exception as e:
logger.error(str(e))
logger.error("Failed to create debrepack parser")
return False
return True
def download_pkg_src(self, _pkg_path):
if not self.parser:
return False
try:
self.parser.download(_pkg_path, self.dl_dir)
except Exception as e:
logger.error(str(e))
logger.error("Failed to download source with %s", _pkg_path)
return False
return True
def download_all(self):
pkgs_list = []
pkgs_all = get_all_stx_pkgs()
for pkg in pkgs_all.keys():
pkgs_list.append(pkg)
self.dl_need.append(pkg)
if not len(pkgs_list):
logger.info("All source packages are already in mirror")
else:
logger.info("Start to download source packages: %d", len(pkgs_list))
logger.info("%s", sorted(pkgs_list))
for pkg in sorted(pkgs_list):
if self.download_pkg_src(pkgs_all[pkg]):
self.dl_success.append(pkg)
else:
self.dl_failed.append(pkg)
def start(self):
# stx package source downloading
super(SrcDownloader, self).clean()
if self.prepare():
self.download_all()
else:
logger.error("Failed to initialize source downloader")
sys.exit(1)
def dl_signal_handler(signum, frame):
src_ret = 0
bin_ret = 0
logger.info("Received signal of keyboard interrupt")
if binary_dl:
bin_ret = binary_dl.reports()
if source_dl:
src_ret = source_dl.reports()
sys.exit(src_ret + bin_ret)
def dl_register_signal_handler():
signal.signal(signal.SIGINT, dl_signal_handler)
signal.signal(signal.SIGHUP, dl_signal_handler)
signal.signal(signal.SIGTERM, dl_signal_handler)
if __name__ == "__main__":
binary_dl = None
source_dl = None
binary_ret = 0
source_ret = 0
parser = argparse.ArgumentParser(description="downloader helper")
parser.add_argument('-b', '--download_binary', help="download binary debs",
action='store_true')
parser.add_argument('-s', '--download_source', help="download stx source",
action='store_true')
parser.add_argument('-c', '--clean_mirror', help="clean the whole mirror and download again, be careful to use",
action='store_true')
args = parser.parse_args()
clean_mirror = args.clean_mirror
if args.download_binary:
all_binary_lists = get_all_binary_list()
binary_dl = DebDownloader(DEFAULT_ARCH, stx_bin_mirror, clean_mirror, all_binary_lists)
if args.download_source:
source_dl = SrcDownloader(DEFAULT_ARCH, stx_src_mirror, clean_mirror)
dl_register_signal_handler()
if binary_dl:
binary_dl.start()
if source_dl:
source_dl.start()
if binary_dl:
binary_ret = binary_dl.reports()
if source_dl:
logger.info('Show the download result for source packages:')
source_ret = source_dl.reports()
logger.info("Downloader done")
sys.exit(binary_ret + source_ret)