starlingx/tools
Code Issues Proposed changes

cengn reference removal - debian

Browse Source 
mirror.starlingx.cengn.ca no longer exists. CENGN is kindly forwarding
requests to the new location mirror.starlingx.windriver.com for now, but
that will only last a few months. We need to replace all the references
with the new URL.

I will also remove as many 'cengn' references as possible, replacing
them with 'stx_mirror'

TESTS
=====
Run "stx config --upgrade" with various combinations of the old,
  new & missing values.
Run "./stx-init-env --rebuild"  and verify stx-localrc is updated
  as well.

Partial-Bug: 2033555
Signed-off-by: Scott Little <scott.little@windriver.com>
Change-Id: I18491fb5aa0087029d51bb35c236613c1dc092bc
This commit is contained in:
Scott Little 2023-09-12 16:01:10 -04:00
parent de0f734d47
commit 2684e9e3fd
10 changed files with 68 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
release/README.rst
View File

@ -25,7 +25,7 @@ the lastest green build context at the time:
release/build-context.py \
--remote starlingx \
--context http://mirror.starlingx.cengn.ca/mirror/starlingx/master/centos/latest_green_build/outputs/CONTEXT.sh \
--context https://mirror.starlingx.windriver.com/mirror/starlingx/master/centos/latest_green_build/outputs/CONTEXT.sh \
https://opendev.org/starlingx/manifest/raw/branch/master/default.xml | \
release/branch-repo.sh -b r/stx.2.0 -t v2.0.0.rc0

8
stx.conf.sample
View File

@ -15,9 +15,9 @@ debian_version = 11.3
# We will append debian_snapshot_timestamp to each of them when creating
# apt.sources lists
#debian_snapshot_base = http://snapshot.debian.org/archive/debian
debian_snapshot_base = http://mirror.starlingx.cengn.ca/mirror/debian/debian/snapshot.debian.org/archive/debian
debian_snapshot_base = https://mirror.starlingx.windriver.com/mirror/debian/debian/snapshot.debian.org/archive/debian
#debian_security_snapshot_base = http://snapshot.debian.org/archive/debian-security
debian_security_snapshot_base = http://mirror.starlingx.cengn.ca/mirror/debian/debian/snapshot.debian.org/archive/debian-security
debian_security_snapshot_base = https://mirror.starlingx.windriver.com/mirror/debian/debian/snapshot.debian.org/archive/debian-security
debian_snapshot_timestamp = 20220331T000000Z
[builder]
@ -34,8 +34,8 @@ mode = local
[repomgr]
type = aptly
cengnurl = http://mirror.starlingx.cengn.ca:80/mirror
cengnstrategy = cengn_first
stx_mirror_url = https://mirror.starlingx.windriver.com:80/mirror
stx_mirror_strategy = stx_mirror_first
sourceslist =
deblist =
dsclist =

2
stx/dockerfiles/stx-lat-tool.Dockerfile
View File

@ -16,7 +16,7 @@ FROM debian:bullseye
MAINTAINER Chen Qi <Qi.Chen@windriver.com>
ARG LAT_BINARY_RESOURCE_PATH=http://mirror.starlingx.cengn.ca/mirror/lat-sdk/lat-sdk-20230510
ARG LAT_BINARY_RESOURCE_PATH=https://mirror.starlingx.windriver.com/mirror/lat-sdk/lat-sdk-20230510
# Update certificates
RUN apt-get -y update && apt-get -y install --no-install-recommends ca-certificates && update-ca-certificates

43
stx/lib/stx/stx_configparser.py
View File

@ -214,33 +214,54 @@ class STXConfigParser:
# delete old key
self.__delete_key('project', 'debian_security_snapshot')
# Change debian_snapshot_* to CENGN
if not self.__upgrade_id_exists('debian_snapshot_cengn'):
debian_snapshot_cengn_upgraded = False
# Change debian_snapshot_* to STX_MIRROR
if not self.__upgrade_id_exists('debian_snapshot_stx_mirror'):
debian_snapshot_stx_mirror_upgraded = False
# debian_snapshot_base
old_value = 'http://snapshot.debian.org/archive/debian'
new_value = 'http://mirror.starlingx.cengn.ca/mirror/debian/debian/snapshot.debian.org/archive/debian'
old_value2 = 'http://mirror.starlingx.cengn.ca/mirror/debian/debian/snapshot.debian.org/archive/debian'
new_value = 'https://mirror.starlingx.windriver.com/mirror/debian/debian/snapshot.debian.org/archive/debian'
current_value = self.__get('project', 'debian_snapshot_base')
if current_value == old_value:
if current_value == old_value or current_value == old_value2:
self.__upgrade_nonempty_key('project', 'debian_snapshot_base', new_value)
debian_snapshot_cengn_upgraded = True
debian_snapshot_stx_mirror_upgraded = True
# debian_security_snapshot_base
old_value = 'http://snapshot.debian.org/archive/debian-security'
new_value = 'http://mirror.starlingx.cengn.ca/mirror/debian/debian/snapshot.debian.org/archive/debian-security'
old_value2 = 'http://mirror.starlingx.cengn.ca/mirror/debian/debian/snapshot.debian.org/archive/debian-security'
new_value = 'https://mirror.starlingx.windriver.com/mirror/debian/debian/snapshot.debian.org/archive/debian-security'
current_value = self.__get('project', 'debian_security_snapshot_base')
if current_value == old_value:
self.__upgrade_nonempty_key('project', 'debian_security_snapshot_base', new_value)
debian_snapshot_cengn_upgraded = True
debian_snapshot_stx_mirror_upgraded = True
if debian_snapshot_cengn_upgraded:
self.__save_upgrade_id('debian_snapshot_cengn')
if debian_snapshot_stx_mirror_upgraded:
self.__save_upgrade_id('debian_snapshot_stx_mirror')
need_restart = True
# Convert cengnurl => stx_mirror_url
if self.cf.has_option('repomgr', 'cengnurl'):
oldstring = self.cf.get('repomgr', 'cengnurl')
newstring = re.sub('http://mirror.starlingx.cengn.ca',
r'https://mirror.starlingx.windriver.com',
oldstring)
self.__upgrade_nonempty_key('repomgr', 'stx_mirror_url', newstring)
# delete old key
self.__delete_key('repomgr', 'cengnurl')
need_restart = True
# Convert cengnstrategy => stx_mirror_strategy
if self.cf.has_option('repomgr', 'cengnstrategy'):
oldstring = self.cf.get('repomgr', 'cengnstrategy')
newstring = re.sub('cengn', 'stx_mirror', oldstring)
self.__upgrade_nonempty_key('repomgr', 'stx_mirror_strategy', newstring)
# delete old key
self.__delete_key('repomgr', 'cengnstrategy')
# Save changes
self.syncConfigFile()
if need_restart:
logger.warning("One or more configuration keeys have been upgraded")
logger.warning("One or more configuration keys have been upgraded")
logger.warning("These changes will take effect after you restart your builder containers")

8
stx/lib/stx/stx_control.py
View File

@ -87,8 +87,8 @@ class HandleControlTask:
proxyport = self.config.get('project', 'proxyport')
buildbranch = self.config.get('project', 'buildbranch')
manifest = self.config.get('project', 'manifest')
cengnurl = self.config.get('repomgr', 'cengnurl')
cengnstrategy = self.config.get('repomgr', 'cengnstrategy')
stx_mirror_url = self.config.get('repomgr', 'stx_mirror_url')
stx_mirror_strategy = self.config.get('repomgr', 'stx_mirror_strategy')
sourceslist = self.config.get('repomgr', 'sourceslist')
deblist = self.config.get('repomgr', 'deblist')
dsclist = self.config.get('repomgr', 'dsclist')
@ -171,8 +171,8 @@ stx-pkgbuilder/configmap/')
line = line.replace("@BUILDBRANCH@", buildbranch)
line = line.replace("@MANIFEST@", manifest)
line = line.replace("@HOSTUSERNAME@", hostusername)
line = line.replace("@CENGNURL@", cengnurl)
line = line.replace("@CENGNSTRATEGY@", cengnstrategy)
line = line.replace("@STX_MIRROR_URL@", stx_mirror_url)
line = line.replace("@STX_MIRROR_STRATEGY@", stx_mirror_strategy)
line = line.replace("@OSTREE_OSNAME@", ostree_osname)
line = line.replace("@DEBIAN_SNAPSHOT@", debian_snapshot)
line = line.replace("@DEBIAN_SECURITY_SNAPSHOT@", debian_security_snapshot)

6
stx/stx-build-tools-chart/stx-builder/configmap/localrc.sample
View File

@ -36,10 +36,10 @@ export MANIFEST=@MANIFEST@
export HOSTUSERNAME=@HOSTUSERNAME@
export CENGNURL=@CENGNURL@
export STX_MIRROR_URL=@STX_MIRROR_URL@
# CENGN_STRATEGY value: [ cengn|cengn_first|upstream|upstream_first ]
export CENGN_STRATEGY=@CENGNSTRATEGY@
# STX_MIRROR_STRATEGY value: [ stx_mirror|stx_mirror_first|upstream|upstream_first ]
export STX_MIRROR_STRATEGY=@STX_MIRROR_STRATEGY@
if [[ x"@fetch@" == x"true" ]];then
export SOURCESLIST=/usr/local/bin/stx/@SOURCESLIST@

8
stx/stx-build-tools-chart/stx-builder/configmap/stx-sources.list
View File

@ -1,4 +1,4 @@
deb [trusted=yes] http://mirror.starlingx.cengn.ca/mirror/debian/debian/ftp.ca.debian.org/debian/ bullseye main
deb-src [trusted=yes] http://mirror.starlingx.cengn.ca/mirror/debian/debian/ftp.ca.debian.org/debian/ bullseye main
deb [trusted=yes] http://mirror.starlingx.cengn.ca/mirror/debian/debian/ftp.ca.debian.org/debian/ buster main
deb-src [trusted=yes] http://mirror.starlingx.cengn.ca/mirror/debian/debian/ftp.ca.debian.org/debian/ buster main
deb [trusted=yes] https://mirror.starlingx.windriver.com/mirror/debian/debian/ftp.ca.debian.org/debian/ bullseye main
deb-src [trusted=yes] https://mirror.starlingx.windriver.com/mirror/debian/debian/ftp.ca.debian.org/debian/ bullseye main
deb [trusted=yes] https://mirror.starlingx.windriver.com/mirror/debian/debian/ftp.ca.debian.org/debian/ buster main
deb-src [trusted=yes] https://mirror.starlingx.windriver.com/mirror/debian/debian/ftp.ca.debian.org/debian/ buster main

8
stx/toCOPY/builder/finishSetup.sh
View File

@ -4,7 +4,7 @@
REPOMGR=aptly
if [ "$REPOMGR" == "aptly" ]; then
CENGN_MIRROR="${CENGNURL}/debian/debian/deb.debian.org/debian/${DEBIAN_DISTRIBUTION}-${DEBIAN_VERSION} ${DEBIAN_DISTRIBUTION} main"
STX_MIRROR="${STX_MIRROR_URL}/debian/debian/deb.debian.org/debian/${DEBIAN_DISTRIBUTION}-${DEBIAN_VERSION} ${DEBIAN_DISTRIBUTION} main"
REPO_SNAPSHOT="[check-valid-until=no] ${DEBIAN_SNAPSHOT} ${DEBIAN_DISTRIBUTION} main"
REPO_BIN="deb [trusted=yes] ${REPOMGR_DEPLOY_URL}deb-local-binary ${DEBIAN_DISTRIBUTION} main"
REPO_SRC="deb-src [trusted=yes] ${REPOMGR_DEPLOY_URL}deb-local-source ${DEBIAN_DISTRIBUTION} main"
@ -21,10 +21,10 @@ if [ "$REPOMGR" == "aptly" ]; then
sed -i "1i\\deb ${REPO_SNAPSHOT}" /etc/apt/sources.list
sed -i "1i\\deb-src ${REPO_SNAPSHOT}" /etc/apt/sources.list
fi
ret=`grep ${CENGNURL} /etc/apt/sources.list`
ret=`grep ${STX_MIRROR_URL} /etc/apt/sources.list`
if [ "x$ret" == "x" ]; then
sed -i "1i\\deb ${CENGN_MIRROR}" /etc/apt/sources.list
sed -i "1i\\deb-src ${CENGN_MIRROR}" /etc/apt/sources.list
sed -i "1i\\deb ${STX_MIRROR}" /etc/apt/sources.list
sed -i "1i\\deb-src ${STX_MIRROR}" /etc/apt/sources.list
fi
fi

4
stx/toCOPY/pkgbuilder/debbuilder.conf
View File

@ -36,8 +36,8 @@ $external_commands = {
'cp /etc/apt/sources.list tmp.list',
'cat tmp.list',
'sed -i "1 i\deb [trusted=yes] http://stx-stx-repomgr:80/deb-local-binary @DEBIAN_DISTRIBUTION@ main" tmp.list',
'echo "deb @CENGNURL@/debian/debian/deb.debian.org/debian/@DEBIAN_DISTRIBUTION@-@DEBIAN_VERSION@ @DEBIAN_DISTRIBUTION@ main" >> tmp.list',
'echo "deb-src @CENGNURL@/debian/debian/deb.debian.org/debian/@DEBIAN_DISTRIBUTION@-@DEBIAN_VERSION@ @DEBIAN_DISTRIBUTION@ main" >> tmp.list',
'echo "deb @STX_MIRROR_URL@/debian/debian/deb.debian.org/debian/@DEBIAN_DISTRIBUTION@-@DEBIAN_VERSION@ @DEBIAN_DISTRIBUTION@ main" >> tmp.list',
'echo "deb-src @STX_MIRROR_URL@/debian/debian/deb.debian.org/debian/@DEBIAN_DISTRIBUTION@-@DEBIAN_VERSION@ @DEBIAN_DISTRIBUTION@ main" >> tmp.list',
'awk \'!a[$0]++\' tmp.list > new.list && mv -f new.list /etc/apt/sources.list',
'cat /etc/apt/sources.list',
'echo "Package: *" > /etc/apt/preferences.d/local_repos',

14
stx/toCOPY/pkgbuilder/debbuilder.py
View File

@ -23,7 +23,7 @@ STORE_ROOT = '/localdisk/pkgbuilder'
BUILD_ENGINE = 'sbuild'
STX_LOCALRC = '/usr/local/bin/stx/stx-localrc'
SBUILD_CONF = '/etc/sbuild/sbuild.conf'
ENVIRON_VARS = ['OSTREE_OSNAME', 'CENGNURL', 'DEBIAN_DISTRIBUTION', 'DEBIAN_VERSION']
ENVIRON_VARS = ['OSTREE_OSNAME', 'STX_MIRROR_URL', 'DEBIAN_DISTRIBUTION', 'DEBIAN_VERSION']
REPO_BUILD = 'deb-local-build'
@ -83,6 +83,18 @@ class Debbuilder:
return
self.logger.debug("%s does exist", STX_LOCALRC)
# backward compatability with CENGNURL
self.logger.debug("Fixing CENGNURL references in stx-localrc")
cmd = "sed -i 's#CENGNURL#STX_MIRROR_URL#' %s" % (STX_LOCALRC)
self.logger.debug('The substitution command is %s', cmd)
try:
outs = subprocess.check_output(cmd, shell=True).decode()
except Exception as e:
self.logger.error(str(e))
self.logger.error("Failed to substitute %s in %s", "CENGNURL", STX_LOCALRC)
return
# end of backward compatibility
for var in ENVIRON_VARS:
self.logger.debug("Fetching %s from stx-localrc", var)
cmd = "grep '^export *%s=.*' %s | cut -d \\= -f 2" % (var, STX_LOCALRC)