starlingx/root
Code Issues Proposed changes

Merge "debian: build-pkgs: Make the build stats complete"

Browse Source
This commit is contained in:
Zuul 2021-12-15 22:34:34 +00:00 committed by Gerrit Code Review
commit 58e57c0efd
1 changed files with 43 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

57
build-tools/stx/build-pkgs
View File

@ -215,7 +215,8 @@ def get_all_packages():
if r == 'debian_pkg_dirs': if r == 'debian_pkg_dirs':
pkgs_file = os.path.join(root, r) pkgs_file = os.path.join(root, r)
pkgs.extend(bc_safe_fetch(pkgs_file, pkgdirs_entry_handler)) pkgs.extend(bc_safe_fetch(pkgs_file, pkgdirs_entry_handler))
return pkgs # Remove duplication
return list(set(pkgs))
def fetch_debian_folder(package): def fetch_debian_folder(package):
@ -273,7 +274,8 @@ class BuildController():
self.lists = { self.lists = {
'success': [], 'success': [],
'fail': [], 'fail': [],
'uploaded': [], 'build-needed': [],
'uploaded': []
} }
self.pkgs_digests = {} self.pkgs_digests = {}
if not self.kits['repo_mgr']: if not self.kits['repo_mgr']:
@ -327,7 +329,7 @@ class BuildController():
return True return True
def stop(self): def stop(self):
self.show_build_stats() return self.show_build_stats()
def clean(self): def clean(self):
""" """
@ -593,6 +595,7 @@ class BuildController():
self.kits['dsc_cache'].set_package_digest(package, pkg_md5) self.kits['dsc_cache'].set_package_digest(package, pkg_md5)
else: else:
logger.info(' '.join(['Failed to build', package, str(p)])) logger.info(' '.join(['Failed to build', package, str(p)]))
self.lists['fail'].append(package)
self.req_stop_task() self.req_stop_task()
if self.attrs['exit_on_fail']: if self.attrs['exit_on_fail']:
return return
@ -610,7 +613,8 @@ class BuildController():
def build_all(self): def build_all(self):
packages = get_all_packages() packages = get_all_packages()
if packages: if packages:
logger.debug(''.join(['All packages:', total_pkgs = len(packages)
logger.debug(''.join(['All packages(', str(total_pkgs), '):',
','.join(packages)])) ','.join(packages)]))
self.build_packages(packages) self.build_packages(packages)
else: else:
@ -620,6 +624,8 @@ class BuildController():
if not layers: if not layers:
logger.error('Failed to get layers') logger.error('Failed to get layers')
return return
# remove duplication
layers = list(set(layers))
for layer in layers: for layer in layers:
if layer not in STX_LAYERS: if layer not in STX_LAYERS:
logger.error(' '.join([layer, 'is not a valid layer'])) logger.error(' '.join([layer, 'is not a valid layer']))
@ -640,9 +646,12 @@ class BuildController():
return return
def build_packages(self, packages): def build_packages(self, packages):
# remove duplication
packages = list(set(packages))
fdsc_file = None fdsc_file = None
packages_dscs = {} packages_dscs = {}
self.lists['fail'] = packages self.lists['build-needed'] = packages
build_dir = os.path.join(BUILD_ROOT, self.attrs['type']) build_dir = os.path.join(BUILD_ROOT, self.attrs['type'])
os.makedirs(build_dir, exist_ok=True) os.makedirs(build_dir, exist_ok=True)
@ -685,32 +694,51 @@ class BuildController():
logger.info("No debian dsc files found") logger.info("No debian dsc files found")
def show_build_stats(self): def show_build_stats(self):
if len(self.lists['success']) > 0: """
logger.info("Successfully built:") Since all packages are put into self.lists['build-needed']
at the begining of building, we know how many
packages want to build
"""
ret_val = 0
logger.info("Total packages needing to be built: %d", len(self.lists['build-needed']))
success_number = len(self.lists['success'])
if success_number > 0:
logger.info("Successfully built: %d", success_number)
for deb in sorted(self.lists['success']): for deb in sorted(self.lists['success']):
logger.info(deb) logger.info(deb)
failed_pkgs = list(set(self.lists['fail']) - set(self.lists['success'])) # failed_pkgs is the universal set of failed packages for various reasons
if len(failed_pkgs) > 0: failed_pkgs = list(set(self.lists['build-needed']) - set(self.lists['success']))
logger.error("Failed to build:") failed_number = len(failed_pkgs)
if failed_number > 0:
ret_val = 1
logger.error("Failed to build: %d", failed_number)
for deb in sorted(failed_pkgs): for deb in sorted(failed_pkgs):
logger.error(deb) logger.error(deb)
# self.lists['fail'] is the subset of failed_pkgs
# particularly refer to those failed packages reported by pkgbuilder
if len(self.lists['fail']) > 0:
logger.info("List of failed packages:")
for deb in sorted(self.lists['fail']):
logger.error(deb)
logger.info("For the failure reason, you can check with:") logger.info("For the failure reason, you can check with:")
logger.info("\'cat /localdisk/builder.log | grep ERROR\' or") logger.info("\'cat /localdisk/builder.log | grep ERROR\' or")
logger.info("\'cat ${MY_WORKSPACE}/<std or rt>/<Failed package>/*.build\'") logger.info("\'cat ${MY_WORKSPACE}/<std or rt>/<Failed package>/*.build\'")
return ret_val
def bc_signal_handler(signum, frame): def bc_signal_handler(signum, frame):
ret_val = 0
if not build_controller: if not build_controller:
return sys.exit(1)
if frame: if frame:
logger.debug(' '.join(['Signal', str(signum), 'got'])) logger.debug(' '.join(['Signal', str(signum), 'got']))
logger.debug('Request to stop building tasks') logger.debug('Request to stop building tasks')
build_controller.req_stop_task() build_controller.req_stop_task()
build_controller.show_build_stats() ret_val = build_controller.show_build_stats()
logger.debug('Exit for user interruption') logger.debug('Exit for user interruption')
sys.exit(1) sys.exit(ret_val)
def bc_reg_signal_handler(): def bc_reg_signal_handler():
@ -781,6 +809,7 @@ if __name__ == "__main__":
sys.exit(1) sys.exit(1)
build_controller.build_route(build_port, build_data) build_controller.build_route(build_port, build_data)
build_controller.stop() ret_value = build_controller.stop()
logger.info("Build controller done") logger.info("Build controller done")
sys.exit(ret_value)