From 08b1999b5be176e07963f02746ad1db7dfbf330f Mon Sep 17 00:00:00 2001 From: Haiqing Bai Date: Fri, 6 Jan 2023 21:14:05 +0800 Subject: [PATCH] downloader: Removed the corrupted downloaded deb file Sometimes corrupted deb file is kept in download directory though exception is raised by candidate.fetch_binary() in the function download(). This corrupted deb file will be uploaded to the remote binary repository. The below was one of the cases we once had which caused this issue: INFO: trace-cmd_2.9.1-1 is uploaded to deb-local-binary DEBUG: package usb.ids_2022.05.20-0+deb11u1 needs to be downloaded DEBUG: Fail to fetch binary usb.ids_2022.05.20-0+deb11u1 DEBUG: The item 'starlingx/binaries/usb.ids_2022.05.20-0+deb11u1_all.deb' could not be fetched: 404 Not Found [IP: 10.101.144.137 80] This commit is to remove the corrupted deb file and prevent it uploading to the remote repository when exception got. Test Plan: Pass: a. Since this is not a issue that recurs every time, below code is added to raise exception to test the exception handling: between line "package.candidate = candidate" and "ret = package.candidate.fetch_binary(self.dl_dir)": "if deb_name == 'usb.ids_2022.05.20-0+deb11u1': os.system('touch ' + os.path.join(self.dl_dir, deb_name + '_all.deb')) raise Exception("Fake exception to test exception handling")" b. Make sure that there is not 'usb.ids*' in '/import/mirrors/starlingx/binaries' in builder container c. Run 'repo_manage.py list_pkgs -r deb-local-binary' and check the '/localdisk/builder.log' to make sure there is not 'usb.ids*.deb' in the remote repository 'deb-local-binary' d. $downloader -b e. downloader should print on screen: downloader - ERROR: Failed to download packages: 1 downloader - ERROR: usb.ids_2022.05.20-0+deb11u1 f. Check the log file '/localdisk/builder.log', there should be below messages: downloader - DEBUG: package usb.ids_2022.05.20-0+deb11u1 needs to be downloaded downloader - DEBUG: Fail to fetch binary usb.ids_2022.05.20-0+deb11u1 downloader - DEBUG: Fake exception to test exception handling g. Make sure that there is not 'usb.ids_2022.05.20-0+deb11u1_all.deb' in '/import/mirrors/starlingx/binaries'. h. Check that there is not 'usb.ids_2022.05.20-0+deb11u1*.deb' in deb-local-binary: $repo_manage.py list_pkgs -r deb-local-binary Story: 2008846 Task: 47123 Signed-off-by: Haiqing Bai Change-Id: I5923d72040abd8dee29be5cb24aa7a7f95a6edac --- build-tools/stx/downloader | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/build-tools/stx/downloader b/build-tools/stx/downloader index f432d15d..771372b3 100755 --- a/build-tools/stx/downloader +++ b/build-tools/stx/downloader @@ -271,8 +271,14 @@ class DebDownloader(BaseDownloader): if ret: return ret except Exception as e: - logger.debug("Fail to fetch binray %s_%s", _name, _version) + deb_name = _name + '_' + _version + logger.debug("Fail to fetch binary %s", deb_name) logger.debug(str(e)) + ''' + Sometimes the target deb is created in dl_dir, but actually it is + corrupted file. It should not be uploaded to binary repo. + ''' + os.system('rm -f ' + os.path.join(self.dl_dir, deb_name + '*.deb')) return None def reports(self):