starlingx/tools
Code Issues Proposed changes

stx tools: Create initial Dockerfile of Debian package build container

Browse Source 
Debian based container to build Debian packages,
manages the chroots and most of tools used to build
source or binary packages are installed

Story: 2008846
Task: 43004

Signed-off-by: hbai <haiqing.bai@windriver.com>
Change-Id: I419de9112b0259e6c75044afc3faaf977d89bfd5
This commit is contained in:
hbai 2021-08-12 11:23:06 -04:00
parent 0448ac722f
commit 327e887f9f
4 changed files with 448 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
stx/dockerfiles/stx-pkgbuilder.Dockerfile Normal file
View File

@ -0,0 +1,52 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Copyright (C) 2021 Wind River Systems,Inc.
#
FROM debian:bullseye
RUN echo "deb-src http://deb.debian.org/debian bullseye main" >> /etc/apt/sources.list
# Download required dependencies by mirror/build processes.
ARG DEBIAN_FRONTEND=noninteractive
RUN apt-get update && apt-get install --no-install-recommends -y \
build-essential \
live-build \
pbuilder \
debootstrap \
devscripts \
schroot \
debmake \
dpkg-dev \
apt-utils \
sbuild \
osc \
python3-pip \
git \
wget \
curl \
vim \
sudo \
emacs \
tini \
procps && \
apt-get clean && \
rm -rf /var/lib/apt/lists/* && \
pip3 install Flask && \
sudo sbuild-adduser root
COPY stx/toCOPY/pkgbuilder/app.py /opt/
COPY stx/toCOPY/pkgbuilder/debbuilder.py /opt/
COPY stx/toCOPY/pkgbuilder/debbuilder.conf /etc/sbuild/sbuild.conf
ENTRYPOINT ["/usr/bin/tini", "--"]
WORKDIR /opt
CMD ["python3", "app.py"]

152
stx/toCOPY/pkgbuilder/app.py Normal file
View File

@ -0,0 +1,152 @@
#!/usr/bin/python3
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Copyright (C) 2021 Wind River Systems,Inc
#
from debbuilder import Debbuilder
from flask import Flask
from flask import jsonify
from flask import request
import logging
app = Flask(__name__)
log = logging.getLogger('pkgbuilder')
dbuilder = Debbuilder('private', log)
@app.route('/pkgbuilder/state', methods=['GET'])
def get_state():
response = {
'status': dbuilder.state,
'msg': ''
}
return jsonify(response)
@app.route('/pkgbuilder/loadchroot', methods=['GET'])
def load_chroot():
attrs = ['user', 'project']
if all(t in request.form for t in attrs):
user = request.form['user']
project = request.form['project']
response = dbuilder.load_chroot(user, project)
log.info("Reply to load chroot, response=%s", str(response))
else:
response = {
'status': 'fail',
'msg': 'invalid request, missing parameter'
}
return jsonify(response)
@app.route('/pkgbuilder/savechroot', methods=['GET'])
def save_chroot():
attrs = ['user', 'project']
if all(t in request.form for t in attrs):
user = request.form['user']
project = request.form['project']
response = dbuilder.save_chroot(user, project)
log.info("Reply to save chroot, response=%s", str(response))
else:
response = {
'status': 'fail',
'msg': 'invalid request, missing parameter'
}
return jsonify(response)
@app.route('/pkgbuilder/addchroot', methods=['GET'])
def add_chroot():
if not request.form or 'user' not in request.form:
log.error("Invalid request to add user chroot")
response = {
'status': 'fail',
'msg': 'invalid request'
}
else:
user = request.form['user']
project = request.form['project']
if 'mirror' in request.form:
response = dbuilder.add_chroot(user, project,
request.form['mirror'])
else:
response = dbuilder.add_chroot(user, project)
log.info("Reply to add user chroot, response=%s", str(response))
return jsonify(response)
@app.route('/pkgbuilder/addtask', methods=['GET'])
def add_task():
response = {}
attrs = ['user', 'project', 'dsc', 'type', 'name', 'mode']
if not all(t in request.form for t in attrs):
log.error("Invalid request to add task")
response['status'] = 'fail'
response['msg'] = 'invalid request'
else:
dbuilder.mode = request.form['mode']
user = request.form['user']
project = request.form['project']
task_info = {'package': request.form['name'],
'dsc': request.form['dsc'], 'type': request.form['type']}
response = dbuilder.add_task(user, project, task_info)
log.info("Reply to add task, response=%s", str(response))
return jsonify(response)
@app.route('/pkgbuilder/killtask', methods=['GET'])
def clean_env():
response = {}
attrs = ['user', 'owner']
if all(t in request.form for t in attrs):
user = request.form['user']
owner = request.form['owner']
response = dbuilder.kill_task(user, owner)
log.info("Reply to kill task, response=%s", str(response))
else:
log.error("Invalid request to kill task")
response = {
'status': 'fail',
'msg': 'invalid request'
}
return jsonify(response)
@app.route('/pkgbuilder/stoptask', methods=['GET'])
def stop_task():
response = {}
attrs = ['user']
if all(t in request.form for t in attrs):
user = request.form['user']
response = dbuilder.stop_task(user)
log.info("Reply to stop task, response=%s", str(response))
else:
log.error("Invalid request to stop task")
response = {
'status': 'fail',
'msg': 'invalid request'
}
return jsonify(response)
if __name__ == '__main__':
app.debug = True
handler = logging.FileHandler('/localdisk/pkgbuilder.log',
encoding='UTF-8')
logging.basicConfig(level=logging.DEBUG)
log_format = logging.Formatter('pkgbuilder: %(levelname)s %(message)s')
handler.setFormatter(log_format)
log.addHandler(handler)
app.run(host='0.0.0.0', port=80, debug=True)

12
stx/toCOPY/pkgbuilder/debbuilder.conf Normal file
View File

@ -0,0 +1,12 @@
$build_arch_all = undef;
$build_arch_any = 1;
$build_source = 1;
$run_autopkgtest = 0;
$run_lintian = 0;
$run_piuparts = 0;
$purge_build_deps = 'never';
$purge_build_directory = 'successful';
$extra_repositories = [];
$log_colour = 1;
1;

232
stx/toCOPY/pkgbuilder/debbuilder.py Normal file
View File

@ -0,0 +1,232 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Copyright (C) 2021 Wind River Systems,Inc
#
import os
import shutil
import subprocess
BUILD_ROOT = '/localdisk/loadbuild/'
BUILD_ENGINE = 'sbuild'
DEBDIST = 'bullseye'
class Debbuilder:
"""
Debbuilder querys/creates/saves/restores the schroot for sbuild
The default name of schroot is '<Debian DIST>-amd64-<USER>'
it takes USER as suffix, per user per schroot and the multiple
build instances launched on the same schroot will be queued.
Debuilder starts/stops the build instances for USER, it also
cleans the scene and handles the USER's abort/terminate commands
to build instance. The whole build log will be displayed
on front end console including the detailed build stats.
For these key result status like success,fail or give-back,
please refer to the document of Debian sbuild.
Debbuiler allows to customize the build configuration for sbuild
engine by updating debbuilder.conf
Debuilder is created by python3 application 'app.py' which runs in
python Flask server to provide Restful APIs to offload the build tasks.
"""
def __init__(self, mode, logger):
self._state = 'idle'
self._mode = mode
self.logger = logger
self.chroot_processes = {}
self.sbuild_processes = {}
self.ctlog = None
@property
def state(self):
return self._state
@property
def mode(self):
return self._mode
@mode.setter
def mode(self, mode):
self._mode = mode
def has_chroot(self, chroot):
chroots = os.popen('schroot -l')
for line in chroots:
if chroot in line.strip():
self.logger.info("chroot %s exists" % chroot)
return True
return False
def add_chroot(self, user='builder', project='stx', mirror=None):
response = {}
if user == 'builder':
self._mode = 'private'
else:
self._mode = 'public'
self.logger.debug("Current chroot mode=%s" % self._mode)
chroot = ''.join([DEBDIST, '-amd64-', user])
if self.has_chroot(chroot):
self.logger.warn("chroot %s has already exists" % chroot)
response['status'] = 'exists'
response['msg'] = 'chroot exists'
return response
user_dir = os.path.join(BUILD_ROOT, user, project)
user_chroots_dir = os.path.join(user_dir, 'chroots')
if not os.path.exists(user_chroots_dir):
os.makedirs(user_chroots_dir)
self.logger.debug("User's chroot dir=%s" % user_chroots_dir)
user_chroot = os.path.join(user_chroots_dir, chroot)
if os.path.exists(user_chroot):
self.logger.debug("Invalid chroot %s, clean it" % user_chroot)
shutil.rmtree(user_chroot)
self.ctlog = open(os.path.join(user_chroots_dir, 'chroot.log'), 'w')
chroot_suffix = '--chroot-suffix=-' + user
chroot_cmd = ' '.join(['sbuild-createchroot', chroot_suffix,
'--include=eatmydata', DEBDIST, user_chroot])
if mirror:
chroot_cmd = ' '.join([chroot_cmd, mirror])
self.logger.debug("Command to creat chroot:%s" % chroot_cmd)
p = subprocess.Popen(chroot_cmd, shell=True, stdout=self.ctlog,
stderr=self.ctlog)
self.chroot_processes.setdefault(user, []).append(p)
response['status'] = 'creating'
response['msg'] = ' '.join(['please check',
user_chroots_dir + '/chroot.log'])
return response
def load_chroot(self, user, project):
response = {}
user_dir = os.path.join(BUILD_ROOT, user, project)
user_chroots = os.path.join(user_dir, 'chroots/chroot.d')
if not os.path.exists(user_chroots):
self.logger.warn("Not find chroots %s" % user_chroots)
response['status'] = 'success'
response['msg'] = ' '.join(['External chroot', user_chroots,
'does not exist'])
else:
target_dir = '/etc/schroot/chroot.d'
if os.path.exists(target_dir):
shutil.rmtree(target_dir)
shutil.copytree(user_chroots, target_dir)
response['status'] = 'success'
response['msg'] = 'Load external chroot config ok'
self.logger.debug("Load chroots %s" % response['status'])
return response
def save_chroot(self, user, project):
response = {}
user_dir = os.path.join(BUILD_ROOT, user, project)
user_chroots = os.path.join(user_dir, 'chroots/chroot.d')
if os.path.exists(user_chroots):
shutil.rmtree(user_chroots)
sys_schroots = '/etc/schroot/chroot.d'
shutil.copytree(sys_schroots, user_chroots)
response['status'] = 'success'
response['msg'] = 'Save chroots config to external'
self.logger.debug("Save chroots config %s" % response['status'])
return response
def add_task(self, user, proj, task_info):
response = {}
chroot = ''.join([DEBDIST, '-amd64-', user])
if not self.has_chroot(chroot):
self.logger.critical("The chroot %s does not exist" % chroot)
response['status'] = 'fail'
response['msg'] = ' '.join(['chroot', chroot, 'does not exist'])
return response
project = os.path.join(BUILD_ROOT, user, proj)
build_dir = os.path.join(project, task_info['type'],
task_info['package'])
if not os.path.isdir(build_dir):
self.logger.critical("%s does not exist" % build_dir)
response['status'] = 'fail'
response['msg'] = build_dir + ' does not exist'
return response
# make sure the dsc file exists
dsc_target = os.path.join(build_dir, task_info['dsc'])
if not os.path.isfile(dsc_target):
self.logger.error("%s does not exist" % dsc_target)
response['status'] = 'fail'
response['msg'] = dsc_target + ' does not exist'
return response
bcommand = ' '.join([BUILD_ENGINE, '-d', DEBDIST, '-c', chroot,
'--build-dir', build_dir, dsc_target])
self.logger.debug("Build command: %s" % bcommand)
self._state = 'works'
p = subprocess.Popen(bcommand, shell=True)
self.sbuild_processes.setdefault(user, []).append(p)
response['status'] = 'success'
response['msg'] = 'sbuild package building task launched'
return response
def kill_task(self, user, owner):
response = {}
if owner in ['sbuild', 'all']:
if self.sbuild_processes and self.sbuild_processes[user]:
for p in self.sbuild_processes[user]:
self.logger.debug("Terminating package build process")
p.terminate()
p.wait()
self.logger.debug("Package build process terminated")
del self.sbuild_processes[user]
if owner in ['chroot', 'all']:
if self.ctlog:
self.ctlog.close()
if self.chroot_processes and self.chroot_processes[user]:
for p in self.chroot_processes[user]:
self.logger.debug("Terminating chroot process")
p.terminate()
p.wait()
self.logger.debug("Chroot process terminated")
del self.chroot_processes[user]
response['status'] = 'success'
response['msg'] = 'killed all build related tasks'
return response
def stop_task(self, user):
response = {}
# check whether the need schroot exists
chroot = ''.join([DEBDIST, '-amd64-', user])
if 'public' in self.mode:
self.logger.debug("Public mode, chroot:%s" % chroot)
else:
self.logger.debug("Private mode, chroot:%s" % chroot)
if not self.has_chroot(chroot):
self.logger.critical("No required chroot %s" % chroot)
self.kill_task(user, 'all')
os.system('sbuild_abort')
response['status'] = 'success'
response['msg'] = 'Stop current build tasks'
return response