starlingx/tools
Code Issues Proposed changes

cengn reference removal - debian

Browse Source 
mirror.starlingx.cengn.ca no longer exists. CENGN is kindly forwarding
requests to the new location mirror.starlingx.windriver.com for now, but
that will only last a few months. We need to replace all the references
with the new URL.

I will also remove as many 'cengn' references as possible, replacing
them with 'stx_mirror'

TESTS
=====
Run "stx config --upgrade" with various combinations of the old,
  new & missing values.
Run "./stx-init-env --rebuild"  and verify stx-localrc is updated
  as well.

Partial-Bug: 2033555
Signed-off-by: Scott Little <scott.little@windriver.com>
Change-Id: I18491fb5aa0087029d51bb35c236613c1dc092bc
This commit is contained in:
Scott Little 2023-09-12 16:01:10 -04:00
parent de0f734d47
commit 2684e9e3fd
10 changed files with 68 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
release/README.rst
View File

@ -25,7 +25,7 @@ the lastest green build context at the time:
release/build-context.py \ release/build-context.py \
--remote starlingx \ --remote starlingx \
--context http://mirror.starlingx.cengn.ca/mirror/starlingx/master/centos/latest_green_build/outputs/CONTEXT.sh \ --context https://mirror.starlingx.windriver.com/mirror/starlingx/master/centos/latest_green_build/outputs/CONTEXT.sh \
https://opendev.org/starlingx/manifest/raw/branch/master/default.xml | \ https://opendev.org/starlingx/manifest/raw/branch/master/default.xml | \
release/branch-repo.sh -b r/stx.2.0 -t v2.0.0.rc0 release/branch-repo.sh -b r/stx.2.0 -t v2.0.0.rc0

8
stx.conf.sample
View File

@ -15,9 +15,9 @@ debian_version = 11.3
# We will append debian_snapshot_timestamp to each of them when creating # We will append debian_snapshot_timestamp to each of them when creating
# apt.sources lists # apt.sources lists
#debian_snapshot_base = http://snapshot.debian.org/archive/debian #debian_snapshot_base = http://snapshot.debian.org/archive/debian
debian_snapshot_base = http://mirror.starlingx.cengn.ca/mirror/debian/debian/snapshot.debian.org/archive/debian debian_snapshot_base = https://mirror.starlingx.windriver.com/mirror/debian/debian/snapshot.debian.org/archive/debian
#debian_security_snapshot_base = http://snapshot.debian.org/archive/debian-security #debian_security_snapshot_base = http://snapshot.debian.org/archive/debian-security
debian_security_snapshot_base = http://mirror.starlingx.cengn.ca/mirror/debian/debian/snapshot.debian.org/archive/debian-security debian_security_snapshot_base = https://mirror.starlingx.windriver.com/mirror/debian/debian/snapshot.debian.org/archive/debian-security
debian_snapshot_timestamp = 20220331T000000Z debian_snapshot_timestamp = 20220331T000000Z
[builder] [builder]
@ -34,8 +34,8 @@ mode = local
[repomgr] [repomgr]
type = aptly type = aptly
cengnurl = http://mirror.starlingx.cengn.ca:80/mirror stx_mirror_url = https://mirror.starlingx.windriver.com:80/mirror
cengnstrategy = cengn_first stx_mirror_strategy = stx_mirror_first
sourceslist = sourceslist =
deblist = deblist =
dsclist = dsclist =

2
stx/dockerfiles/stx-lat-tool.Dockerfile
View File

@ -16,7 +16,7 @@ FROM debian:bullseye
MAINTAINER Chen Qi <Qi.Chen@windriver.com> MAINTAINER Chen Qi <Qi.Chen@windriver.com>
ARG LAT_BINARY_RESOURCE_PATH=http://mirror.starlingx.cengn.ca/mirror/lat-sdk/lat-sdk-20230510 ARG LAT_BINARY_RESOURCE_PATH=https://mirror.starlingx.windriver.com/mirror/lat-sdk/lat-sdk-20230510
# Update certificates # Update certificates
RUN apt-get -y update && apt-get -y install --no-install-recommends ca-certificates && update-ca-certificates RUN apt-get -y update && apt-get -y install --no-install-recommends ca-certificates && update-ca-certificates

43
stx/lib/stx/stx_configparser.py
View File

@ -214,33 +214,54 @@ class STXConfigParser:
# delete old key # delete old key
self.__delete_key('project', 'debian_security_snapshot') self.__delete_key('project', 'debian_security_snapshot')
# Change debian_snapshot_* to CENGN # Change debian_snapshot_* to STX_MIRROR
if not self.__upgrade_id_exists('debian_snapshot_cengn'): if not self.__upgrade_id_exists('debian_snapshot_stx_mirror'):
debian_snapshot_cengn_upgraded = False debian_snapshot_stx_mirror_upgraded = False
# debian_snapshot_base # debian_snapshot_base
old_value = 'http://snapshot.debian.org/archive/debian' old_value = 'http://snapshot.debian.org/archive/debian'
new_value = 'http://mirror.starlingx.cengn.ca/mirror/debian/debian/snapshot.debian.org/archive/debian' old_value2 = 'http://mirror.starlingx.cengn.ca/mirror/debian/debian/snapshot.debian.org/archive/debian'
new_value = 'https://mirror.starlingx.windriver.com/mirror/debian/debian/snapshot.debian.org/archive/debian'
current_value = self.__get('project', 'debian_snapshot_base') current_value = self.__get('project', 'debian_snapshot_base')
if current_value == old_value: if current_value == old_value or current_value == old_value2:
self.__upgrade_nonempty_key('project', 'debian_snapshot_base', new_value) self.__upgrade_nonempty_key('project', 'debian_snapshot_base', new_value)
debian_snapshot_cengn_upgraded = True debian_snapshot_stx_mirror_upgraded = True
# debian_security_snapshot_base # debian_security_snapshot_base
old_value = 'http://snapshot.debian.org/archive/debian-security' old_value = 'http://snapshot.debian.org/archive/debian-security'
new_value = 'http://mirror.starlingx.cengn.ca/mirror/debian/debian/snapshot.debian.org/archive/debian-security' old_value2 = 'http://mirror.starlingx.cengn.ca/mirror/debian/debian/snapshot.debian.org/archive/debian-security'
new_value = 'https://mirror.starlingx.windriver.com/mirror/debian/debian/snapshot.debian.org/archive/debian-security'
current_value = self.__get('project', 'debian_security_snapshot_base') current_value = self.__get('project', 'debian_security_snapshot_base')
if current_value == old_value: if current_value == old_value:
self.__upgrade_nonempty_key('project', 'debian_security_snapshot_base', new_value) self.__upgrade_nonempty_key('project', 'debian_security_snapshot_base', new_value)
debian_snapshot_cengn_upgraded = True debian_snapshot_stx_mirror_upgraded = True
if debian_snapshot_cengn_upgraded: if debian_snapshot_stx_mirror_upgraded:
self.__save_upgrade_id('debian_snapshot_cengn') self.__save_upgrade_id('debian_snapshot_stx_mirror')
need_restart = True need_restart = True
# Convert cengnurl => stx_mirror_url
if self.cf.has_option('repomgr', 'cengnurl'):
oldstring = self.cf.get('repomgr', 'cengnurl')
newstring = re.sub('http://mirror.starlingx.cengn.ca',
r'https://mirror.starlingx.windriver.com',
oldstring)
self.__upgrade_nonempty_key('repomgr', 'stx_mirror_url', newstring)
# delete old key
self.__delete_key('repomgr', 'cengnurl')
need_restart = True
# Convert cengnstrategy => stx_mirror_strategy
if self.cf.has_option('repomgr', 'cengnstrategy'):
oldstring = self.cf.get('repomgr', 'cengnstrategy')
newstring = re.sub('cengn', 'stx_mirror', oldstring)
self.__upgrade_nonempty_key('repomgr', 'stx_mirror_strategy', newstring)
# delete old key
self.__delete_key('repomgr', 'cengnstrategy')
# Save changes # Save changes
self.syncConfigFile() self.syncConfigFile()
if need_restart: if need_restart:
logger.warning("One or more configuration keeys have been upgraded") logger.warning("One or more configuration keys have been upgraded")
logger.warning("These changes will take effect after you restart your builder containers") logger.warning("These changes will take effect after you restart your builder containers")

8
stx/lib/stx/stx_control.py
View File

@ -87,8 +87,8 @@ class HandleControlTask:
proxyport = self.config.get('project', 'proxyport') proxyport = self.config.get('project', 'proxyport')
buildbranch = self.config.get('project', 'buildbranch') buildbranch = self.config.get('project', 'buildbranch')
manifest = self.config.get('project', 'manifest') manifest = self.config.get('project', 'manifest')
cengnurl = self.config.get('repomgr', 'cengnurl') stx_mirror_url = self.config.get('repomgr', 'stx_mirror_url')
cengnstrategy = self.config.get('repomgr', 'cengnstrategy') stx_mirror_strategy = self.config.get('repomgr', 'stx_mirror_strategy')
sourceslist = self.config.get('repomgr', 'sourceslist') sourceslist = self.config.get('repomgr', 'sourceslist')
deblist = self.config.get('repomgr', 'deblist') deblist = self.config.get('repomgr', 'deblist')
dsclist = self.config.get('repomgr', 'dsclist') dsclist = self.config.get('repomgr', 'dsclist')
@ -171,8 +171,8 @@ stx-pkgbuilder/configmap/')
line = line.replace("@BUILDBRANCH@", buildbranch) line = line.replace("@BUILDBRANCH@", buildbranch)
line = line.replace("@MANIFEST@", manifest) line = line.replace("@MANIFEST@", manifest)
line = line.replace("@HOSTUSERNAME@", hostusername) line = line.replace("@HOSTUSERNAME@", hostusername)
line = line.replace("@CENGNURL@", cengnurl) line = line.replace("@STX_MIRROR_URL@", stx_mirror_url)
line = line.replace("@CENGNSTRATEGY@", cengnstrategy) line = line.replace("@STX_MIRROR_STRATEGY@", stx_mirror_strategy)
line = line.replace("@OSTREE_OSNAME@", ostree_osname) line = line.replace("@OSTREE_OSNAME@", ostree_osname)
line = line.replace("@DEBIAN_SNAPSHOT@", debian_snapshot) line = line.replace("@DEBIAN_SNAPSHOT@", debian_snapshot)
line = line.replace("@DEBIAN_SECURITY_SNAPSHOT@", debian_security_snapshot) line = line.replace("@DEBIAN_SECURITY_SNAPSHOT@", debian_security_snapshot)

6
stx/stx-build-tools-chart/stx-builder/configmap/localrc.sample
View File

@ -36,10 +36,10 @@ export MANIFEST=@MANIFEST@
export HOSTUSERNAME=@HOSTUSERNAME@ export HOSTUSERNAME=@HOSTUSERNAME@
export CENGNURL=@CENGNURL@ export STX_MIRROR_URL=@STX_MIRROR_URL@
# CENGN_STRATEGY value: [ cengn|cengn_first|upstream|upstream_first ] # STX_MIRROR_STRATEGY value: [ stx_mirror|stx_mirror_first|upstream|upstream_first ]
export CENGN_STRATEGY=@CENGNSTRATEGY@ export STX_MIRROR_STRATEGY=@STX_MIRROR_STRATEGY@
if [[ x"@fetch@" == x"true" ]];then if [[ x"@fetch@" == x"true" ]];then
export SOURCESLIST=/usr/local/bin/stx/@SOURCESLIST@ export SOURCESLIST=/usr/local/bin/stx/@SOURCESLIST@

8
stx/stx-build-tools-chart/stx-builder/configmap/stx-sources.list
View File

@ -1,4 +1,4 @@
deb [trusted=yes] http://mirror.starlingx.cengn.ca/mirror/debian/debian/ftp.ca.debian.org/debian/ bullseye main deb [trusted=yes] https://mirror.starlingx.windriver.com/mirror/debian/debian/ftp.ca.debian.org/debian/ bullseye main
deb-src [trusted=yes] http://mirror.starlingx.cengn.ca/mirror/debian/debian/ftp.ca.debian.org/debian/ bullseye main deb-src [trusted=yes] https://mirror.starlingx.windriver.com/mirror/debian/debian/ftp.ca.debian.org/debian/ bullseye main
deb [trusted=yes] http://mirror.starlingx.cengn.ca/mirror/debian/debian/ftp.ca.debian.org/debian/ buster main deb [trusted=yes] https://mirror.starlingx.windriver.com/mirror/debian/debian/ftp.ca.debian.org/debian/ buster main
deb-src [trusted=yes] http://mirror.starlingx.cengn.ca/mirror/debian/debian/ftp.ca.debian.org/debian/ buster main deb-src [trusted=yes] https://mirror.starlingx.windriver.com/mirror/debian/debian/ftp.ca.debian.org/debian/ buster main

8
stx/toCOPY/builder/finishSetup.sh
View File

@ -4,7 +4,7 @@
REPOMGR=aptly REPOMGR=aptly
if [ "$REPOMGR" == "aptly" ]; then if [ "$REPOMGR" == "aptly" ]; then
CENGN_MIRROR="${CENGNURL}/debian/debian/deb.debian.org/debian/${DEBIAN_DISTRIBUTION}-${DEBIAN_VERSION} ${DEBIAN_DISTRIBUTION} main" STX_MIRROR="${STX_MIRROR_URL}/debian/debian/deb.debian.org/debian/${DEBIAN_DISTRIBUTION}-${DEBIAN_VERSION} ${DEBIAN_DISTRIBUTION} main"
REPO_SNAPSHOT="[check-valid-until=no] ${DEBIAN_SNAPSHOT} ${DEBIAN_DISTRIBUTION} main" REPO_SNAPSHOT="[check-valid-until=no] ${DEBIAN_SNAPSHOT} ${DEBIAN_DISTRIBUTION} main"
REPO_BIN="deb [trusted=yes] ${REPOMGR_DEPLOY_URL}deb-local-binary ${DEBIAN_DISTRIBUTION} main" REPO_BIN="deb [trusted=yes] ${REPOMGR_DEPLOY_URL}deb-local-binary ${DEBIAN_DISTRIBUTION} main"
REPO_SRC="deb-src [trusted=yes] ${REPOMGR_DEPLOY_URL}deb-local-source ${DEBIAN_DISTRIBUTION} main" REPO_SRC="deb-src [trusted=yes] ${REPOMGR_DEPLOY_URL}deb-local-source ${DEBIAN_DISTRIBUTION} main"
@ -21,10 +21,10 @@ if [ "$REPOMGR" == "aptly" ]; then
sed -i "1i\\deb ${REPO_SNAPSHOT}" /etc/apt/sources.list sed -i "1i\\deb ${REPO_SNAPSHOT}" /etc/apt/sources.list
sed -i "1i\\deb-src ${REPO_SNAPSHOT}" /etc/apt/sources.list sed -i "1i\\deb-src ${REPO_SNAPSHOT}" /etc/apt/sources.list
fi fi
ret=`grep ${CENGNURL} /etc/apt/sources.list` ret=`grep ${STX_MIRROR_URL} /etc/apt/sources.list`
if [ "x$ret" == "x" ]; then if [ "x$ret" == "x" ]; then
sed -i "1i\\deb ${CENGN_MIRROR}" /etc/apt/sources.list sed -i "1i\\deb ${STX_MIRROR}" /etc/apt/sources.list
sed -i "1i\\deb-src ${CENGN_MIRROR}" /etc/apt/sources.list sed -i "1i\\deb-src ${STX_MIRROR}" /etc/apt/sources.list
fi fi
fi fi

4
stx/toCOPY/pkgbuilder/debbuilder.conf
View File

@ -36,8 +36,8 @@ $external_commands = {
'cp /etc/apt/sources.list tmp.list', 'cp /etc/apt/sources.list tmp.list',
'cat tmp.list', 'cat tmp.list',
'sed -i "1 i\deb [trusted=yes] http://stx-stx-repomgr:80/deb-local-binary @DEBIAN_DISTRIBUTION@ main" tmp.list', 'sed -i "1 i\deb [trusted=yes] http://stx-stx-repomgr:80/deb-local-binary @DEBIAN_DISTRIBUTION@ main" tmp.list',
'echo "deb @CENGNURL@/debian/debian/deb.debian.org/debian/@DEBIAN_DISTRIBUTION@-@DEBIAN_VERSION@ @DEBIAN_DISTRIBUTION@ main" >> tmp.list', 'echo "deb @STX_MIRROR_URL@/debian/debian/deb.debian.org/debian/@DEBIAN_DISTRIBUTION@-@DEBIAN_VERSION@ @DEBIAN_DISTRIBUTION@ main" >> tmp.list',
'echo "deb-src @CENGNURL@/debian/debian/deb.debian.org/debian/@DEBIAN_DISTRIBUTION@-@DEBIAN_VERSION@ @DEBIAN_DISTRIBUTION@ main" >> tmp.list', 'echo "deb-src @STX_MIRROR_URL@/debian/debian/deb.debian.org/debian/@DEBIAN_DISTRIBUTION@-@DEBIAN_VERSION@ @DEBIAN_DISTRIBUTION@ main" >> tmp.list',
'awk \'!a[$0]++\' tmp.list > new.list && mv -f new.list /etc/apt/sources.list', 'awk \'!a[$0]++\' tmp.list > new.list && mv -f new.list /etc/apt/sources.list',
'cat /etc/apt/sources.list', 'cat /etc/apt/sources.list',
'echo "Package: *" > /etc/apt/preferences.d/local_repos', 'echo "Package: *" > /etc/apt/preferences.d/local_repos',

14
stx/toCOPY/pkgbuilder/debbuilder.py
View File

@ -23,7 +23,7 @@ STORE_ROOT = '/localdisk/pkgbuilder'
BUILD_ENGINE = 'sbuild' BUILD_ENGINE = 'sbuild'
STX_LOCALRC = '/usr/local/bin/stx/stx-localrc' STX_LOCALRC = '/usr/local/bin/stx/stx-localrc'
SBUILD_CONF = '/etc/sbuild/sbuild.conf' SBUILD_CONF = '/etc/sbuild/sbuild.conf'
ENVIRON_VARS = ['OSTREE_OSNAME', 'CENGNURL', 'DEBIAN_DISTRIBUTION', 'DEBIAN_VERSION'] ENVIRON_VARS = ['OSTREE_OSNAME', 'STX_MIRROR_URL', 'DEBIAN_DISTRIBUTION', 'DEBIAN_VERSION']
REPO_BUILD = 'deb-local-build' REPO_BUILD = 'deb-local-build'
@ -83,6 +83,18 @@ class Debbuilder:
return return
self.logger.debug("%s does exist", STX_LOCALRC) self.logger.debug("%s does exist", STX_LOCALRC)
# backward compatability with CENGNURL
self.logger.debug("Fixing CENGNURL references in stx-localrc")
cmd = "sed -i 's#CENGNURL#STX_MIRROR_URL#' %s" % (STX_LOCALRC)
self.logger.debug('The substitution command is %s', cmd)
try:
outs = subprocess.check_output(cmd, shell=True).decode()
except Exception as e:
self.logger.error(str(e))
self.logger.error("Failed to substitute %s in %s", "CENGNURL", STX_LOCALRC)
return
# end of backward compatibility
for var in ENVIRON_VARS: for var in ENVIRON_VARS:
self.logger.debug("Fetching %s from stx-localrc", var) self.logger.debug("Fetching %s from stx-localrc", var)
cmd = "grep '^export *%s=.*' %s | cut -d \\= -f 2" % (var, STX_LOCALRC) cmd = "grep '^export *%s=.*' %s | cut -d \\= -f 2" % (var, STX_LOCALRC)