starlingx/root
Code Issues Proposed changes

cengn reference removal

Browse Source 
mirror.starlingx.cengn.ca no longer exists. CENGN is kindly forwarding
requests to the new location mirror.starlingx.windriver.com for now, but
that will only last a few months. We need to replace all the references
with the new URL.

I will also remove as many 'cengn' references as possible, replacing them
with 'stx_mirror'

Partial-Bug: 2033555
Signed-off-by: Scott Little <scott.little@windriver.com>
Signed-off-by: Davlet Panech <davlet.panech@windriver.com>
Change-Id: I9d1396d603dcd3fc54bd4496fd332de278aa70da
This commit is contained in:
Scott Little 2023-11-07 15:34:54 -05:00 committed by Davlet Panech
parent 6adb3e8c72
commit 130c43619b
4 changed files with 57 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
build-tools/stx/debrepack.py
View File

@ -32,8 +32,22 @@ import yaml
RELEASENOTES = " ".join([os.environ.get('PROJECT'), os.environ.get('MY_RELEASE'), "distribution"])
DIST = os.environ.get('STX_DIST')
# The CENGN_STRATEGY and CENGNURL references is retained for backward
# compatability with pre-existing build environments.
CENGN_BASE = os.path.join(os.environ.get('CENGNURL'), "debian")
CENGN_STRATEGY = os.environ.get('CENGN_STRATEGY')
STX_MIRROR_BASE = os.path.join(os.environ.get('STX_MIRROR_URL'), "debian")
STX_MIRROR_STRATEGY = os.environ.get('STX_MIRROR_STRATEGY')
if STX_MIRROR_BASE is None:
STX_MIRROR_BASE = CENGN_BASE
if STX_MIRROR_STRATEGY is None:
STX_MIRROR_STRATEGY = CENGN_STRATEGY
if STX_MIRROR_STRATEGY == "cengn":
STX_MIRROR_STRATEGY = "stx_mirror"
if STX_MIRROR_STRATEGY == "cengn_first":
STX_MIRROR_STRATEGY = "stx_mirror_first"
BTYPE = "@KERNEL_TYPE@"
@ -167,10 +181,10 @@ class Parser():
if not self.logger.handlers:
utils.set_logger(self.logger, log_level=log_level)
self.strategy = "cengn_first"
if CENGN_STRATEGY is not None:
self.strategy = CENGN_STRATEGY
# dry run to check the value of CENGN_STRATEGY
self.strategy = "stx_mirror_first"
if STX_MIRROR_STRATEGY is not None:
self.strategy = STX_MIRROR_STRATEGY
# dry run to check the value of STX_MIRROR_STRATEGY
get_download_url("https://testurl/tarball.tgz", self.strategy)[0]
if not os.path.isdir(basedir):

13
build-tools/stx/downloader
View File

@ -45,7 +45,16 @@ ALL_DISTROS = discovery.get_all_distros()
ALL_LAYERS = discovery.get_all_layers(distro=STX_DEFAULT_DISTRO)
ALL_BUILD_TYPES = discovery.get_all_build_types(distro=STX_DEFAULT_DISTRO)
# The CENGN_STRATEGY reference is retained for backward compatability
# with pre-existing build environments.
CENGN_STRATEGY = os.environ.get('CENGN_STRATEGY')
STX_MIRROR_STRATEGY = os.environ.get('STX_MIRROR_STRATEGY')
if not STX_MIRROR_STRATEGY:
STX_MIRROR_STRATEGY = CENGN_STRATEGY
if STX_MIRROR_STRATEGY == "cengn":
STX_MIRROR_STRATEGY = "stx_mirror"
elif STX_MIRROR_STRATEGY == "cengn_first":
STX_MIRROR_STRATEGY = "stx_mirror_first"
def get_downloaded(dl_dir, dl_type):
"""
@ -267,7 +276,7 @@ class DebDownloader(BaseDownloader):
ret = os.path.join(self.dl_dir, dl_file)
tmp_file = ".".join([ret, "tmp"])
utils.run_shell_cmd(["rm", "-rf", tmp_file], logger)
(dl_url, alt_dl_url) = utils.get_download_url(url, CENGN_STRATEGY)
(dl_url, alt_dl_url) = utils.get_download_url(url, STX_MIRROR_STRATEGY)
for i in range(1,retries+1):
if alt_dl_url:
try:
@ -373,7 +382,7 @@ class DebDownloader(BaseDownloader):
if len(pkg_name_array) == 3:
url = pkg_name_array[2]
url_dict = utils.deb_file_name_to_dict(os.path.basename(url).replace("%2B", "+"))
logger.debug("dkg_data: name=%s, ver=%s, url=%s, url_dict=%s, file=%s", pkg_name, pkg_ver, url,url, str(url_dict), list_file)
logger.debug("dkg_data: name=%s, ver=%s, url=%s, url_dict=%s, file=%s", pkg_name, pkg_ver, url, str(url_dict), list_file)
if url_dict['ver'] and url_dict['ver'] != pkg_ver:
logger.warning("Package version mismatch for package %s, $s vs %s, in file %s", pkg_name, pkg_ver, url_dict['ver'], list_file)
pkg_ver = url_dict['ver']

37
build-tools/stx/utils.py
View File

@ -21,7 +21,16 @@ import subprocess
import urllib.parse
import urllib.request
CENGN_BASE = os.path.join(os.environ.get('CENGNURL'), "debian")
# The CENGNURL reference is retained for backward compatability
# with pre-existing build environmnets.
CENGNURL = os.environ.get('CENGNURL')
if CENGNURL:
CENGN_BASE = os.path.join(CENGNURL, "debian")
STX_MIRROR_URL = os.environ.get('STX_MIRROR_URL')
if STX_MIRROR_URL:
STX_MIRROR_BASE = os.path.join(STX_MIRROR_URL, "debian")
if not STX_MIRROR_BASE:
STX_MIRROR_BASE=CENGN_BASE
log_levels = {
'debug': logging.DEBUG,
@ -165,7 +174,7 @@ def run_shell_cmd(cmd, logger):
return outs.strip()
def url_to_cengn(url):
def url_to_stx_mirror(url):
url_change = urllib.parse.urlparse(url)
url_path = pathlib.Path(url_change.path)
@ -175,23 +184,23 @@ def url_to_cengn(url):
path = url_path
# FIXME: the ":" in a path is converted to "%25", after
# uploading to CENGN, the "%25" in the path is converted
# uploading to STX_MIRROR, the "%25" in the path is converted
# to "%2525".
return os.path.join(CENGN_BASE, path).replace("%25", "%2525")
return os.path.join(STX_MIRROR_BASE, path).replace("%25", "%2525")
def get_download_url(url, strategy):
alt_rt_url = None
cengn_url = url_to_cengn(url)
if strategy == "cengn":
rt_url = cengn_url
stx_mirror_url = url_to_stx_mirror(url)
if strategy == "stx_mirror":
rt_url = stx_mirror_url
elif strategy == "upstream":
rt_url = url
elif strategy == "cengn_first":
rt_url = urstx_mirror
elif strategy == "stx_mirror_first":
try:
urllib.request.urlopen(cengn_url)
rt_url = cengn_url
urllib.request.urlopen(stx_mirror_url)
rt_url = stx_mirror_url
alt_rt_url = url
except:
rt_url = url
@ -199,11 +208,11 @@ def get_download_url(url, strategy):
try:
urllib.request.urlopen(url)
rt_url = url
alt_rt_url = cengn_url
alt_rt_url = stx_mirror_url
except:
rt_url = cengn_url
rt_url = stx_mirror_url
else:
raise Exception(f'Invalid value "{strategy}" of CENGN_STRATEGY')
raise Exception(f'Invalid value "{strategy}" of STX_MIRROR_STRATEGY')
return (rt_url, alt_rt_url)

10
build-tools/url_utils.sh
View File

@ -234,10 +234,10 @@ repo_url_to_sub_path () {
return 0
}
CENGN_PROTOCOL="http"
CENGN_HOST="mirror.starlingx.cengn.ca"
CENGN_PORT="80"
CENGN_URL_ROOT="mirror"
STX_MIRROR_PROTOCOL="https"
STX_MIRROR_HOST="mirror.starlingx.windriver.com"
STX_MIRROR_PORT="80"
STX_MIRROR_URL_ROOT="mirror"
url_to_stx_mirror_url () {
local URL="$1"
@ -262,7 +262,7 @@ url_to_stx_mirror_url () {
return 1
fi
echo "$CENGN_PROTOCOL://$CENGN_HOST:$CENGN_PORT/$CENGN_URL_ROOT/$DISTRO/$URL_PATH"
echo "$STX_MIRROR_PROTOCOL://$STX_MIRROR_HOST:$STX_MIRROR_PORT/$STX_MIRROR_URL_ROOT/$DISTRO/$URL_PATH"
return 0
}