From 18b8fc07016fc2d31600e90291fe4be603a651a5 Mon Sep 17 00:00:00 2001 From: Zhixiong Chi Date: Mon, 13 Sep 2021 17:12:24 +0800 Subject: [PATCH] stx tool: Create initial Dockerfile of LAT(Linux Assembly Tool) container Create the initial Dockerfile of STX LAT tool container. Now LAT source code has been open sourced, but it didn't include the binary. Once the binary is open sourced, we will replace the ${LAT_BINARY_RESOURCE_PATH} in the stx-lat-tool.dockerfile file. Story: 2008846 Task: 43292 Signed-off-by: Zhixiong Chi Change-Id: Ibfcc05855b56bedb91392d5958c105ca005f2146 --- stx/dockerfiles/stx-lat-tool.Dockerfile | 41 +++++ stx/toCOPY/lat-tool/README | 101 +++++++++++ stx/toCOPY/lat-tool/lat/latc | 101 +++++++++++ stx/toCOPY/lat-tool/lat/latd | 226 ++++++++++++++++++++++++ stx/toCOPY/lat-tool/lat/utils.py | 64 +++++++ stx/toCOPY/lat-tool/lat/volume.py | 126 +++++++++++++ 6 files changed, 659 insertions(+) create mode 100644 stx/dockerfiles/stx-lat-tool.Dockerfile create mode 100644 stx/toCOPY/lat-tool/README create mode 100644 stx/toCOPY/lat-tool/lat/latc create mode 100755 stx/toCOPY/lat-tool/lat/latd create mode 100644 stx/toCOPY/lat-tool/lat/utils.py create mode 100644 stx/toCOPY/lat-tool/lat/volume.py diff --git a/stx/dockerfiles/stx-lat-tool.Dockerfile b/stx/dockerfiles/stx-lat-tool.Dockerfile new file mode 100644 index 00000000..493dd336 --- /dev/null +++ b/stx/dockerfiles/stx-lat-tool.Dockerfile @@ -0,0 +1,41 @@ +# Copyright (c) 2021 Wind River Systems, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM debian:bullseye + +MAINTAINER Chen Qi + +# Install necessary packages +RUN apt-get -y update && apt-get --no-install-recommends -y install \ + python3 \ + xz-utils \ + file \ + bzip2 \ + locales-all \ + python3-yaml && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* && \ + mkdir -p /opt/LAT/SDK + +# Prepare executables +COPY stx/toCOPY/lat-tool/lat/ /opt/LAT/lat +# Prepare LAT SDK. +# Current URL is an internal one for development. ${LAT_BINARY_RESOURCE_PATH} +# needs to be replaced by a public one, once the lat sdk binary was open +# sourced, and it's coming... +ADD ${LAT_BINARY_RESOURCE_PATH}/lat-sdk.sh /opt/LAT/AppSDK.sh +RUN chmod +x /opt/LAT/AppSDK.sh +RUN /opt/LAT/AppSDK.sh -d /opt/LAT/SDK -y + +ENTRYPOINT ["/opt/LAT/lat/latd"] diff --git a/stx/toCOPY/lat-tool/README b/stx/toCOPY/lat-tool/README new file mode 100644 index 00000000..9a08ac30 --- /dev/null +++ b/stx/toCOPY/lat-tool/README @@ -0,0 +1,101 @@ +# Copyright (c) 2021 Wind River Systems, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +latd +--- + +This is a daemon expected to run inside the LAT container as the entry point. +It accepts requests from clients and issue correspoing commands to generate the image. + +Functionalities: +* Accept request to give client an example yaml file to start with + latc getyaml +* Accept build request to build image, with a yaml file supplied. + latc build --file stx.yaml +* Accept request to stop the previous build + latc stop +* Accept request to clean things up, including all build results + latc clean +* Accept status query request, return meaningful contents + latc status + e.g. + lat_status: idle/busy + latest_build_status: running/succeed/fail/not-started +* Accept logs requrest, return log information about the latest build + latc logs + +latc(builder container client) +--- + +A dummy implementation to only show how to make use of volume.py to communicate with latd. +e.g. +latc status/stop/clean/logs/build + +volume communication implementation +--- + +* channel/c-2-s.msg + + Client to server message. Convey info about what to do. + + action: build/status/stop/logs/clean/getyaml + yaml_file: /mnt/workspace/stx.yaml + +* channel/c-2-s.done + + A file to watch, when it appears, latd read c-2-s.msg and act accordingly. + +* channel/s-2-c.msg + + Server to client message. Same format as c-2-s.msg + +* channel/s-2-c.done + + A file to watch, when it appears, latc reads s-2-c.msg and act accordingly. + +* channel/invalid_message + + A file which, if exists, indicates the an invalid client request. + +* channel/status.lat + + File containing LAT container status. + +* log/log.appsdk + + Hold information about the appsdk debug output. + +* client_message_history + + Hold message history from client. + +* hack/lat-genimage-cmd + + A hack file, which replace the 'appsdk genimage ...' command with the contents inside it. + e.g. + echo "sleep 10" > hack/lat-genimage-cmd + + This is ONLY for debugging purpose. This mechanism should be removed in formal release. + +* latd workflow + + Watch c-2-s.done, when it appears, read c-2-s.msg, delete c-2-s.done file. + Fork a subprocess to do the actual work if needed, record its PID; otherwise, perform some action. Send result to client. + Note that the result only means whether the client request is handled by latd or not. + + +Assumption +--- + +* At any time, at most one build is run inside LAT container. diff --git a/stx/toCOPY/lat-tool/lat/latc b/stx/toCOPY/lat-tool/lat/latc new file mode 100644 index 00000000..c04db879 --- /dev/null +++ b/stx/toCOPY/lat-tool/lat/latc @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2021 Wind River Systems, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys +import time +import logging +import subprocess +import argparse +import signal +import yaml + +import utils +import volume as channel + +logger = logging.getLogger('latc') +utils.set_logger(logger) + +WORKSPACE_DIR = channel.workspace_dir +CHANNEL_DIR = channel.channel_dir + +def main(): + parser = argparse.ArgumentParser( + description="LAT Daemon which runs inside LAT container", + epilog="Use %(prog)s --help to get help") + parser.add_argument("-d", "--debug", + help = "Enable debug output", + action="store_const", const=logging.DEBUG, dest="loglevel", + default=logging.INFO) + parser.add_argument("-q", "--quiet", + help = "Hide all output except error messages", + action="store_const", const=logging.ERROR, dest="loglevel") + parser.add_argument('action', metavar='action', type=str, nargs=1, + help = 'Action to take. e.g. getyaml/build/status/stop/clean/logs') + parser.add_argument('--file', required=False, + help = 'Yaml file path which is fed to LAT') + + args = parser.parse_args() + logger.setLevel(args.loglevel) + + client_action = args.action[0] + + msg = {} + if client_action == 'getyaml': + msg['action'] = 'getyaml' + elif client_action == 'build': + if not args.file: + logger.error('latc build --file /path/to/some.yaml') + sys.exit(1) + msg['action'] = 'build' + if not os.path.exists(args.file): + logger.error("Yaml file does not exist: %s" % args.file) + sys.exit(1) + msg['yaml_file'] = os.path.abspath(args.file) + elif client_action == 'status': + msg['action'] = 'status' + elif client_action == 'stop': + msg['action'] = 'stop' + elif client_action == 'clean': + msg['action'] = 'clean' + elif client_action == 'logs': + msg['action'] = 'logs' + else: + logger.error("Action supported: getyaml/build/status/stop/clean/logs") + sys.exit(1) + + channel.send_message_to_server(msg) + smsg = channel.get_server_message() + if smsg['result'] == 'fail': + fail_reason = smsg['fail_reason'] if 'fail_reason' in smsg else 'Unknown' + logger.error("%s failed: %s" % (client_action, fail_reason)) + else: + logger.info("%s request handled by server" % client_action) + for key in ['status_file', 'log_file_path', 'yaml_file_path']: + if key in smsg: + logger.info("%s: %s" % (key, smsg[key])) + subprocess.check_call('cat %s' % smsg[key], shell=True) + + +if __name__ == "__main__": + try: + ret = main() + except Exception as esc: + ret = 1 + import traceback + traceback.print_exc() + + sys.exit(ret) diff --git a/stx/toCOPY/lat-tool/lat/latd b/stx/toCOPY/lat-tool/lat/latd new file mode 100755 index 00000000..5114c3bc --- /dev/null +++ b/stx/toCOPY/lat-tool/lat/latd @@ -0,0 +1,226 @@ +#!/usr/bin/env python3 + +import os +import sys +import time +import logging +import subprocess +import argparse +import signal +import yaml + +import utils +import volume as channel + +logger = logging.getLogger('latd') +utils.set_logger(logger) + +LAT_SDK_DIR = "/opt/LAT/SDK" +WORKSPACE_DIR = channel.workspace_dir +CHANNEL_DIR = channel.channel_dir + +class LATD(object): + + def __init__(self): + self.appsdk_dir = LAT_SDK_DIR + self.workspace_dir = WORKSPACE_DIR + self.child_pid = 0 + self.latest_build_status = 'not-started' + + def update_build_status(self): + """ + Update the build status + """ + if self.child_pid > 0: + c_pid, c_st = os.waitpid(self.child_pid, os.WNOHANG) + if c_pid == 0: + self.latest_build_status = 'running' + else: + if c_st == 0: + self.latest_build_status = 'succeed' + else: + self.latest_build_status = 'fail' + self.child_pid = 0 + + def run(self): + """ + Listen for client requests and act accordingly + """ + def record_message(hist_file, msg): + with open(hist_file, 'a') as f: + f.write("%s: %s\n" % (time.time(), msg)) + + logger.info("workspace_dir: %s" % self.workspace_dir) + env_script = self.appsdk_dir + '/environment-setup-corei7-64-wrs-linux' + channel.server_init_channel() + while(True): + logger.info("latd waiting for client request") + message = channel.get_client_message() + logger.info("Got client message: %s" % message) + # record the message into history + msg_hist = CHANNEL_DIR + "/client_message_history" + record_message(msg_hist, message) + if message['action'] == 'build': + msg_latd = {} + msg_latd['action'] = 'build' + + if 'yaml_file' in message: + yaml_file = message['yaml_file'] + if os.path.exists(yaml_file): + channel.mark_client_message_valid(True) + else: + logger.info("Yaml file does not exist: %s" % yaml_file) + channel.mark_client_message_valid(False) + msg_latd['result'] = 'fail' + msg_latd['fail_reason'] = 'Configuration yaml file (%s) does not exist' % yaml_file + channel.send_message_to_client(msg_latd) + continue + else: + logger.info('No yaml file contents: %s' % yaml_file) + channel.mark_client_message_valid(False) + msg_latd['result'] = 'fail' + msg_latd['fail_reason'] = 'No configuration yaml file contents' + channel.send_message_to_client(msg_latd) + continue + + # check if a build is still running + if self.child_pid > 0: + c_pid, c_st = os.waitpid(self.child_pid, os.WNOHANG) + if c_pid == 0: + msg_latd['result'] = 'fail' + msg_latd['fail_reason'] = 'A previous build is still running' + channel.send_message_to_client(msg_latd) + continue + else: + self.child_pid = 0 + # fork a child process to do the build + try: + pid = os.fork() + except OSError: + logger.warning("Failed to fork a child process to do the build") + msg_latd['result'] = 'fail' + msg_latd['fail_reason'] = 'Failed to fork a child process to do the build' + channel.send_message_to_client(msg_latd) + continue + if pid > 0: + # parent process + msg_latd['result'] = 'succeed' + self.child_pid = pid + channel.send_message_to_client(msg_latd) + else: + # child process + # use genimage command do the build and then exit + gencmd = 'genimage' + with open(yaml_file) as f: + contents = yaml.safe_load(f) + if 'image_type' in contents: + if 'initramfs' in contents['image_type']: + gencmd = 'geninitramfs' + cmd = '. %s; appsdk --log-dir log %s %s' % (env_script, gencmd, yaml_file) + # hack to change genimage command + if os.path.exists(self.workspace_dir + '/hack/lat-genimage-cmd'): + with open(self.workspace_dir + '/hack/lat-genimage-cmd', 'r') as f: + lines = f.readlines() + cmd = ' '.join(lines) + logger.info("latd running command: %s" % cmd) + subprocess.check_call(cmd, shell=True, cwd=self.workspace_dir) + sys.exit(0) + elif message['action'] == 'status': + channel.mark_client_message_valid(True) + status_file = CHANNEL_DIR + '/status.lat' + st = {} + self.update_build_status() + if self.latest_build_status == 'running': + st['lat_status'] = 'busy' + else: + st['lat_status'] = 'idle' + st['build_status'] = self.latest_build_status + with open(status_file, 'w') as f: + yaml.safe_dump(st, f) + # status action should always succeed + msg_latd = {} + msg_latd['action'] = 'status' + msg_latd['result'] = 'succeed' + msg_latd['status_file'] = status_file + channel.send_message_to_client(msg_latd) + elif message['action'] == 'stop': + channel.mark_client_message_valid(True) + # stop action should always succeed + if self.child_pid > 0: + os.kill(self.child_pid, signal.SIGKILL) + self.child_pid = 0 + self.latest_build_status = "fail" + msg_latd = {} + msg_latd['action'] = 'stop' + msg_latd['result'] = 'succeed' + channel.send_message_to_client(msg_latd) + elif message['action'] == 'logs': + channel.mark_client_message_valid(True) + log_file = self.workspace_dir + '/log/log.appsdk' + msg_latd = {} + msg_latd['action'] = 'logs' + if os.path.exists(log_file): + msg_latd['result'] = 'succeed' + msg_latd['log_file_path'] = log_file + else: + msg_latd['result'] = 'fail' + msg_latd['fail_reason'] = 'No log file found' + channel.send_message_to_client(msg_latd) + elif message['action'] == 'clean': + channel.mark_client_message_valid(True) + self.update_build_status() + msg_latd = {} + if self.latest_build_status == 'running': + msg_latd['action'] = 'clean' + msg_latd['result'] = 'fail' + msg_latd['fail_reason'] = 'A build is still running' + else: + cmd = 'rm -rf deploy exampleyamls workdir log' + subprocess.check_call(cmd, shell=True, cwd=self.workspace_dir) + msg_latd['action'] = 'clean' + msg_latd['result'] = 'succeed' + channel.send_message_to_client(msg_latd) + elif message['action'] == 'getyaml': + channel.mark_client_message_valid(True) + cmd = '. %s; appsdk exampleyamls --pkg-type external-debian' % env_script + subprocess.check_call(cmd, shell=True, cwd=self.workspace_dir) + example_yaml = self.workspace_dir + '/exampleyamls/debian-image-base-intel-x86-64.yaml' + msg_latd = {} + msg_latd['action'] = 'getyaml' + msg_latd['result'] = 'succeed' + msg_latd['yaml_file_path'] = example_yaml + channel.send_message_to_client(msg_latd) + else: + logger.warning("Invalid action request from client: %s" % message['action']) + channel.mark_client_message_valid(False) + + +def main(): + parser = argparse.ArgumentParser( + description="LAT Daemon which runs inside LAT container", + epilog="Use %(prog)s --help to get help") + parser.add_argument("-d", "--debug", + help = "Enable debug output", + action="store_const", const=logging.DEBUG, dest="loglevel", default=logging.INFO) + parser.add_argument("-q", "--quiet", + help = "Hide all output except error messages", + action="store_const", const=logging.ERROR, dest="loglevel") + + args = parser.parse_args() + logger.setLevel(args.loglevel) + + logger.info("Started") + latd = LATD() + latd.run() + + +if __name__ == "__main__": + try: + ret = main() + except Exception as esc: + ret = 1 + import traceback + traceback.print_exc() + + sys.exit(ret) + diff --git a/stx/toCOPY/lat-tool/lat/utils.py b/stx/toCOPY/lat-tool/lat/utils.py new file mode 100644 index 00000000..0a5a523e --- /dev/null +++ b/stx/toCOPY/lat-tool/lat/utils.py @@ -0,0 +1,64 @@ +# Copyright (c) 2021 Wind River Systems, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + + +def set_logger(logger): + logger.setLevel(logging.DEBUG) + + class ColorFormatter(logging.Formatter): + FORMAT = ("$BOLD%(name)-s$RESET - %(levelname)s: %(message)s") + + BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = list(range(8)) + + RESET_SEQ = "\033[0m" + COLOR_SEQ = "\033[1;%dm" + BOLD_SEQ = "\033[1m" + + COLORS = { + 'WARNING': YELLOW, + 'INFO': GREEN, + 'DEBUG': BLUE, + 'ERROR': RED + } + + def formatter_msg(self, msg, use_color=True): + if use_color: + msg = msg.replace("$RESET", + self.RESET_SEQ).replace("$BOLD", + self.BOLD_SEQ) + else: + msg = msg.replace("$RESET", "").replace("$BOLD", "") + return msg + + def __init__(self, use_color=True): + msg = self.formatter_msg(self.FORMAT, use_color) + logging.Formatter.__init__(self, msg) + self.use_color = use_color + + def format(self, record): + levelname = record.levelname + if self.use_color and levelname in self.COLORS: + fore_color = 30 + self.COLORS[levelname] + levelname_color = self.COLOR_SEQ % fore_color + levelname + levelname_color += self.RESET_SEQ + record.levelname = levelname_color + return logging.Formatter.format(self, record) + + # create console handler and set level to debug + ch = logging.StreamHandler() + ch.setLevel(logging.DEBUG) + ch.setFormatter(ColorFormatter()) + logger.addHandler(ch) diff --git a/stx/toCOPY/lat-tool/lat/volume.py b/stx/toCOPY/lat-tool/lat/volume.py new file mode 100644 index 00000000..d825a354 --- /dev/null +++ b/stx/toCOPY/lat-tool/lat/volume.py @@ -0,0 +1,126 @@ +# Copyright (c) 2021 Wind River Systems, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +volume module implements shared volume communication between STX builder +container and LAT container +""" + +import logging +import os +import subprocess +import time +import yaml + +logger = logging.getLogger('latd.volume') + +workspace_dir = "/localdisk" if 'WORKSPACE_DIR' not in os.environ \ + else os.environ['WORKSPACE_DIR'] + +channel_dir = workspace_dir + "/channel" + +client_message_watch_file = channel_dir + "/c-2-s.done" +client_message_content_file = channel_dir + "/c-2-s.msg" +server_message_watch_file = channel_dir + '/s-2-c.done' +server_message_content_file = channel_dir + '/s-2-c.msg' + + +def server_init_channel(): + """ + Init channel + """ + if not os.path.exists(channel_dir): + subprocess.check_call('mkdir -p %s' % channel_dir, shell=True) + rm_cmd = 'rm -f %s %s %s %s' % (client_message_watch_file, + client_message_content_file, + server_message_watch_file, + server_message_content_file) + subprocess.check_call(rm_cmd, shell=True) + + +def get_client_message(): + """ + Get client message. + Return a dict of {'action': '', : xxx} + + If the message is related to file contents, store the contents into a LAT + container local file, return dict containing the local file path. + """ + # As we are using shared volume, the file contents could be shared, + # so extra need for storing local files. + while True: + if os.path.exists(client_message_watch_file): + with open(client_message_content_file) as f: + msg = yaml.safe_load(f) + os.unlink(client_message_watch_file) + return msg + + time.sleep(0.1) + + +def get_server_message(): + """ + Get server message. Return a dict of {'action': '', + 'result': xxx, : xxx} + + If the message is related to file contents, store the contents into a STX + build container local file, return dict containing the local file path. + """ + # As we are using shared volume, the file contents could be shared, + # so extra need for storing local files. + while True: + if os.path.exists(server_message_watch_file): + with open(server_message_content_file) as f: + msg = yaml.safe_load(f) + os.unlink(server_message_watch_file) + return msg + + time.sleep(0.1) + + +def mark_client_message_valid(is_valid): + """ + Mark client message valid or not + """ + invalid_message_file = channel_dir + 'invalid_message' + if is_valid: + subprocess.check_call('rm -f %s' % invalid_message_file, shell=True) + else: + subprocess.check_call('touch %s' % invalid_message_file, shell=True) + + +def send_message_to_client(msg): + """ + Send message to client. + msg is a dict. + """ + # According to different action in msg, it should behave differently. + # For example, for messages which contain file path, the contents might + # need to read and send out in other backend like REST. + # But for shared volume backend, no need to do so, just letting the + # client read the file is OK. + + with open(server_message_content_file, 'w') as f: + yaml.safe_dump(msg, f) + subprocess.check_call('touch %s' % server_message_watch_file, shell=True) + + +def send_message_to_server(msg): + """ + Send message to server. + msg is a dict. + """ + with open(client_message_content_file, 'w') as f: + yaml.safe_dump(msg, f) + subprocess.check_call('touch %s' % client_message_watch_file, shell=True)