diff --git a/stx/dockerfiles/stx-lat-tool.Dockerfile b/stx/dockerfiles/stx-lat-tool.Dockerfile new file mode 100644 index 00000000..493dd336 --- /dev/null +++ b/stx/dockerfiles/stx-lat-tool.Dockerfile @@ -0,0 +1,41 @@ +# Copyright (c) 2021 Wind River Systems, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM debian:bullseye + +MAINTAINER Chen Qi + +# Install necessary packages +RUN apt-get -y update && apt-get --no-install-recommends -y install \ + python3 \ + xz-utils \ + file \ + bzip2 \ + locales-all \ + python3-yaml && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* && \ + mkdir -p /opt/LAT/SDK + +# Prepare executables +COPY stx/toCOPY/lat-tool/lat/ /opt/LAT/lat +# Prepare LAT SDK. +# Current URL is an internal one for development. ${LAT_BINARY_RESOURCE_PATH} +# needs to be replaced by a public one, once the lat sdk binary was open +# sourced, and it's coming... +ADD ${LAT_BINARY_RESOURCE_PATH}/lat-sdk.sh /opt/LAT/AppSDK.sh +RUN chmod +x /opt/LAT/AppSDK.sh +RUN /opt/LAT/AppSDK.sh -d /opt/LAT/SDK -y + +ENTRYPOINT ["/opt/LAT/lat/latd"] diff --git a/stx/toCOPY/lat-tool/README b/stx/toCOPY/lat-tool/README new file mode 100644 index 00000000..9a08ac30 --- /dev/null +++ b/stx/toCOPY/lat-tool/README @@ -0,0 +1,101 @@ +# Copyright (c) 2021 Wind River Systems, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +latd +--- + +This is a daemon expected to run inside the LAT container as the entry point. +It accepts requests from clients and issue correspoing commands to generate the image. + +Functionalities: +* Accept request to give client an example yaml file to start with + latc getyaml +* Accept build request to build image, with a yaml file supplied. + latc build --file stx.yaml +* Accept request to stop the previous build + latc stop +* Accept request to clean things up, including all build results + latc clean +* Accept status query request, return meaningful contents + latc status + e.g. + lat_status: idle/busy + latest_build_status: running/succeed/fail/not-started +* Accept logs requrest, return log information about the latest build + latc logs + +latc(builder container client) +--- + +A dummy implementation to only show how to make use of volume.py to communicate with latd. +e.g. +latc status/stop/clean/logs/build + +volume communication implementation +--- + +* channel/c-2-s.msg + + Client to server message. Convey info about what to do. + + action: build/status/stop/logs/clean/getyaml + yaml_file: /mnt/workspace/stx.yaml + +* channel/c-2-s.done + + A file to watch, when it appears, latd read c-2-s.msg and act accordingly. + +* channel/s-2-c.msg + + Server to client message. Same format as c-2-s.msg + +* channel/s-2-c.done + + A file to watch, when it appears, latc reads s-2-c.msg and act accordingly. + +* channel/invalid_message + + A file which, if exists, indicates the an invalid client request. + +* channel/status.lat + + File containing LAT container status. + +* log/log.appsdk + + Hold information about the appsdk debug output. + +* client_message_history + + Hold message history from client. + +* hack/lat-genimage-cmd + + A hack file, which replace the 'appsdk genimage ...' command with the contents inside it. + e.g. + echo "sleep 10" > hack/lat-genimage-cmd + + This is ONLY for debugging purpose. This mechanism should be removed in formal release. + +* latd workflow + + Watch c-2-s.done, when it appears, read c-2-s.msg, delete c-2-s.done file. + Fork a subprocess to do the actual work if needed, record its PID; otherwise, perform some action. Send result to client. + Note that the result only means whether the client request is handled by latd or not. + + +Assumption +--- + +* At any time, at most one build is run inside LAT container. diff --git a/stx/toCOPY/lat-tool/lat/latc b/stx/toCOPY/lat-tool/lat/latc new file mode 100644 index 00000000..c04db879 --- /dev/null +++ b/stx/toCOPY/lat-tool/lat/latc @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2021 Wind River Systems, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys +import time +import logging +import subprocess +import argparse +import signal +import yaml + +import utils +import volume as channel + +logger = logging.getLogger('latc') +utils.set_logger(logger) + +WORKSPACE_DIR = channel.workspace_dir +CHANNEL_DIR = channel.channel_dir + +def main(): + parser = argparse.ArgumentParser( + description="LAT Daemon which runs inside LAT container", + epilog="Use %(prog)s --help to get help") + parser.add_argument("-d", "--debug", + help = "Enable debug output", + action="store_const", const=logging.DEBUG, dest="loglevel", + default=logging.INFO) + parser.add_argument("-q", "--quiet", + help = "Hide all output except error messages", + action="store_const", const=logging.ERROR, dest="loglevel") + parser.add_argument('action', metavar='action', type=str, nargs=1, + help = 'Action to take. e.g. getyaml/build/status/stop/clean/logs') + parser.add_argument('--file', required=False, + help = 'Yaml file path which is fed to LAT') + + args = parser.parse_args() + logger.setLevel(args.loglevel) + + client_action = args.action[0] + + msg = {} + if client_action == 'getyaml': + msg['action'] = 'getyaml' + elif client_action == 'build': + if not args.file: + logger.error('latc build --file /path/to/some.yaml') + sys.exit(1) + msg['action'] = 'build' + if not os.path.exists(args.file): + logger.error("Yaml file does not exist: %s" % args.file) + sys.exit(1) + msg['yaml_file'] = os.path.abspath(args.file) + elif client_action == 'status': + msg['action'] = 'status' + elif client_action == 'stop': + msg['action'] = 'stop' + elif client_action == 'clean': + msg['action'] = 'clean' + elif client_action == 'logs': + msg['action'] = 'logs' + else: + logger.error("Action supported: getyaml/build/status/stop/clean/logs") + sys.exit(1) + + channel.send_message_to_server(msg) + smsg = channel.get_server_message() + if smsg['result'] == 'fail': + fail_reason = smsg['fail_reason'] if 'fail_reason' in smsg else 'Unknown' + logger.error("%s failed: %s" % (client_action, fail_reason)) + else: + logger.info("%s request handled by server" % client_action) + for key in ['status_file', 'log_file_path', 'yaml_file_path']: + if key in smsg: + logger.info("%s: %s" % (key, smsg[key])) + subprocess.check_call('cat %s' % smsg[key], shell=True) + + +if __name__ == "__main__": + try: + ret = main() + except Exception as esc: + ret = 1 + import traceback + traceback.print_exc() + + sys.exit(ret) diff --git a/stx/toCOPY/lat-tool/lat/latd b/stx/toCOPY/lat-tool/lat/latd new file mode 100755 index 00000000..5114c3bc --- /dev/null +++ b/stx/toCOPY/lat-tool/lat/latd @@ -0,0 +1,226 @@ +#!/usr/bin/env python3 + +import os +import sys +import time +import logging +import subprocess +import argparse +import signal +import yaml + +import utils +import volume as channel + +logger = logging.getLogger('latd') +utils.set_logger(logger) + +LAT_SDK_DIR = "/opt/LAT/SDK" +WORKSPACE_DIR = channel.workspace_dir +CHANNEL_DIR = channel.channel_dir + +class LATD(object): + + def __init__(self): + self.appsdk_dir = LAT_SDK_DIR + self.workspace_dir = WORKSPACE_DIR + self.child_pid = 0 + self.latest_build_status = 'not-started' + + def update_build_status(self): + """ + Update the build status + """ + if self.child_pid > 0: + c_pid, c_st = os.waitpid(self.child_pid, os.WNOHANG) + if c_pid == 0: + self.latest_build_status = 'running' + else: + if c_st == 0: + self.latest_build_status = 'succeed' + else: + self.latest_build_status = 'fail' + self.child_pid = 0 + + def run(self): + """ + Listen for client requests and act accordingly + """ + def record_message(hist_file, msg): + with open(hist_file, 'a') as f: + f.write("%s: %s\n" % (time.time(), msg)) + + logger.info("workspace_dir: %s" % self.workspace_dir) + env_script = self.appsdk_dir + '/environment-setup-corei7-64-wrs-linux' + channel.server_init_channel() + while(True): + logger.info("latd waiting for client request") + message = channel.get_client_message() + logger.info("Got client message: %s" % message) + # record the message into history + msg_hist = CHANNEL_DIR + "/client_message_history" + record_message(msg_hist, message) + if message['action'] == 'build': + msg_latd = {} + msg_latd['action'] = 'build' + + if 'yaml_file' in message: + yaml_file = message['yaml_file'] + if os.path.exists(yaml_file): + channel.mark_client_message_valid(True) + else: + logger.info("Yaml file does not exist: %s" % yaml_file) + channel.mark_client_message_valid(False) + msg_latd['result'] = 'fail' + msg_latd['fail_reason'] = 'Configuration yaml file (%s) does not exist' % yaml_file + channel.send_message_to_client(msg_latd) + continue + else: + logger.info('No yaml file contents: %s' % yaml_file) + channel.mark_client_message_valid(False) + msg_latd['result'] = 'fail' + msg_latd['fail_reason'] = 'No configuration yaml file contents' + channel.send_message_to_client(msg_latd) + continue + + # check if a build is still running + if self.child_pid > 0: + c_pid, c_st = os.waitpid(self.child_pid, os.WNOHANG) + if c_pid == 0: + msg_latd['result'] = 'fail' + msg_latd['fail_reason'] = 'A previous build is still running' + channel.send_message_to_client(msg_latd) + continue + else: + self.child_pid = 0 + # fork a child process to do the build + try: + pid = os.fork() + except OSError: + logger.warning("Failed to fork a child process to do the build") + msg_latd['result'] = 'fail' + msg_latd['fail_reason'] = 'Failed to fork a child process to do the build' + channel.send_message_to_client(msg_latd) + continue + if pid > 0: + # parent process + msg_latd['result'] = 'succeed' + self.child_pid = pid + channel.send_message_to_client(msg_latd) + else: + # child process + # use genimage command do the build and then exit + gencmd = 'genimage' + with open(yaml_file) as f: + contents = yaml.safe_load(f) + if 'image_type' in contents: + if 'initramfs' in contents['image_type']: + gencmd = 'geninitramfs' + cmd = '. %s; appsdk --log-dir log %s %s' % (env_script, gencmd, yaml_file) + # hack to change genimage command + if os.path.exists(self.workspace_dir + '/hack/lat-genimage-cmd'): + with open(self.workspace_dir + '/hack/lat-genimage-cmd', 'r') as f: + lines = f.readlines() + cmd = ' '.join(lines) + logger.info("latd running command: %s" % cmd) + subprocess.check_call(cmd, shell=True, cwd=self.workspace_dir) + sys.exit(0) + elif message['action'] == 'status': + channel.mark_client_message_valid(True) + status_file = CHANNEL_DIR + '/status.lat' + st = {} + self.update_build_status() + if self.latest_build_status == 'running': + st['lat_status'] = 'busy' + else: + st['lat_status'] = 'idle' + st['build_status'] = self.latest_build_status + with open(status_file, 'w') as f: + yaml.safe_dump(st, f) + # status action should always succeed + msg_latd = {} + msg_latd['action'] = 'status' + msg_latd['result'] = 'succeed' + msg_latd['status_file'] = status_file + channel.send_message_to_client(msg_latd) + elif message['action'] == 'stop': + channel.mark_client_message_valid(True) + # stop action should always succeed + if self.child_pid > 0: + os.kill(self.child_pid, signal.SIGKILL) + self.child_pid = 0 + self.latest_build_status = "fail" + msg_latd = {} + msg_latd['action'] = 'stop' + msg_latd['result'] = 'succeed' + channel.send_message_to_client(msg_latd) + elif message['action'] == 'logs': + channel.mark_client_message_valid(True) + log_file = self.workspace_dir + '/log/log.appsdk' + msg_latd = {} + msg_latd['action'] = 'logs' + if os.path.exists(log_file): + msg_latd['result'] = 'succeed' + msg_latd['log_file_path'] = log_file + else: + msg_latd['result'] = 'fail' + msg_latd['fail_reason'] = 'No log file found' + channel.send_message_to_client(msg_latd) + elif message['action'] == 'clean': + channel.mark_client_message_valid(True) + self.update_build_status() + msg_latd = {} + if self.latest_build_status == 'running': + msg_latd['action'] = 'clean' + msg_latd['result'] = 'fail' + msg_latd['fail_reason'] = 'A build is still running' + else: + cmd = 'rm -rf deploy exampleyamls workdir log' + subprocess.check_call(cmd, shell=True, cwd=self.workspace_dir) + msg_latd['action'] = 'clean' + msg_latd['result'] = 'succeed' + channel.send_message_to_client(msg_latd) + elif message['action'] == 'getyaml': + channel.mark_client_message_valid(True) + cmd = '. %s; appsdk exampleyamls --pkg-type external-debian' % env_script + subprocess.check_call(cmd, shell=True, cwd=self.workspace_dir) + example_yaml = self.workspace_dir + '/exampleyamls/debian-image-base-intel-x86-64.yaml' + msg_latd = {} + msg_latd['action'] = 'getyaml' + msg_latd['result'] = 'succeed' + msg_latd['yaml_file_path'] = example_yaml + channel.send_message_to_client(msg_latd) + else: + logger.warning("Invalid action request from client: %s" % message['action']) + channel.mark_client_message_valid(False) + + +def main(): + parser = argparse.ArgumentParser( + description="LAT Daemon which runs inside LAT container", + epilog="Use %(prog)s --help to get help") + parser.add_argument("-d", "--debug", + help = "Enable debug output", + action="store_const", const=logging.DEBUG, dest="loglevel", default=logging.INFO) + parser.add_argument("-q", "--quiet", + help = "Hide all output except error messages", + action="store_const", const=logging.ERROR, dest="loglevel") + + args = parser.parse_args() + logger.setLevel(args.loglevel) + + logger.info("Started") + latd = LATD() + latd.run() + + +if __name__ == "__main__": + try: + ret = main() + except Exception as esc: + ret = 1 + import traceback + traceback.print_exc() + + sys.exit(ret) + diff --git a/stx/toCOPY/lat-tool/lat/utils.py b/stx/toCOPY/lat-tool/lat/utils.py new file mode 100644 index 00000000..0a5a523e --- /dev/null +++ b/stx/toCOPY/lat-tool/lat/utils.py @@ -0,0 +1,64 @@ +# Copyright (c) 2021 Wind River Systems, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + + +def set_logger(logger): + logger.setLevel(logging.DEBUG) + + class ColorFormatter(logging.Formatter): + FORMAT = ("$BOLD%(name)-s$RESET - %(levelname)s: %(message)s") + + BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = list(range(8)) + + RESET_SEQ = "\033[0m" + COLOR_SEQ = "\033[1;%dm" + BOLD_SEQ = "\033[1m" + + COLORS = { + 'WARNING': YELLOW, + 'INFO': GREEN, + 'DEBUG': BLUE, + 'ERROR': RED + } + + def formatter_msg(self, msg, use_color=True): + if use_color: + msg = msg.replace("$RESET", + self.RESET_SEQ).replace("$BOLD", + self.BOLD_SEQ) + else: + msg = msg.replace("$RESET", "").replace("$BOLD", "") + return msg + + def __init__(self, use_color=True): + msg = self.formatter_msg(self.FORMAT, use_color) + logging.Formatter.__init__(self, msg) + self.use_color = use_color + + def format(self, record): + levelname = record.levelname + if self.use_color and levelname in self.COLORS: + fore_color = 30 + self.COLORS[levelname] + levelname_color = self.COLOR_SEQ % fore_color + levelname + levelname_color += self.RESET_SEQ + record.levelname = levelname_color + return logging.Formatter.format(self, record) + + # create console handler and set level to debug + ch = logging.StreamHandler() + ch.setLevel(logging.DEBUG) + ch.setFormatter(ColorFormatter()) + logger.addHandler(ch) diff --git a/stx/toCOPY/lat-tool/lat/volume.py b/stx/toCOPY/lat-tool/lat/volume.py new file mode 100644 index 00000000..d825a354 --- /dev/null +++ b/stx/toCOPY/lat-tool/lat/volume.py @@ -0,0 +1,126 @@ +# Copyright (c) 2021 Wind River Systems, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +volume module implements shared volume communication between STX builder +container and LAT container +""" + +import logging +import os +import subprocess +import time +import yaml + +logger = logging.getLogger('latd.volume') + +workspace_dir = "/localdisk" if 'WORKSPACE_DIR' not in os.environ \ + else os.environ['WORKSPACE_DIR'] + +channel_dir = workspace_dir + "/channel" + +client_message_watch_file = channel_dir + "/c-2-s.done" +client_message_content_file = channel_dir + "/c-2-s.msg" +server_message_watch_file = channel_dir + '/s-2-c.done' +server_message_content_file = channel_dir + '/s-2-c.msg' + + +def server_init_channel(): + """ + Init channel + """ + if not os.path.exists(channel_dir): + subprocess.check_call('mkdir -p %s' % channel_dir, shell=True) + rm_cmd = 'rm -f %s %s %s %s' % (client_message_watch_file, + client_message_content_file, + server_message_watch_file, + server_message_content_file) + subprocess.check_call(rm_cmd, shell=True) + + +def get_client_message(): + """ + Get client message. + Return a dict of {'action': '', : xxx} + + If the message is related to file contents, store the contents into a LAT + container local file, return dict containing the local file path. + """ + # As we are using shared volume, the file contents could be shared, + # so extra need for storing local files. + while True: + if os.path.exists(client_message_watch_file): + with open(client_message_content_file) as f: + msg = yaml.safe_load(f) + os.unlink(client_message_watch_file) + return msg + + time.sleep(0.1) + + +def get_server_message(): + """ + Get server message. Return a dict of {'action': '', + 'result': xxx, : xxx} + + If the message is related to file contents, store the contents into a STX + build container local file, return dict containing the local file path. + """ + # As we are using shared volume, the file contents could be shared, + # so extra need for storing local files. + while True: + if os.path.exists(server_message_watch_file): + with open(server_message_content_file) as f: + msg = yaml.safe_load(f) + os.unlink(server_message_watch_file) + return msg + + time.sleep(0.1) + + +def mark_client_message_valid(is_valid): + """ + Mark client message valid or not + """ + invalid_message_file = channel_dir + 'invalid_message' + if is_valid: + subprocess.check_call('rm -f %s' % invalid_message_file, shell=True) + else: + subprocess.check_call('touch %s' % invalid_message_file, shell=True) + + +def send_message_to_client(msg): + """ + Send message to client. + msg is a dict. + """ + # According to different action in msg, it should behave differently. + # For example, for messages which contain file path, the contents might + # need to read and send out in other backend like REST. + # But for shared volume backend, no need to do so, just letting the + # client read the file is OK. + + with open(server_message_content_file, 'w') as f: + yaml.safe_dump(msg, f) + subprocess.check_call('touch %s' % server_message_watch_file, shell=True) + + +def send_message_to_server(msg): + """ + Send message to server. + msg is a dict. + """ + with open(client_message_content_file, 'w') as f: + yaml.safe_dump(msg, f) + subprocess.check_call('touch %s' % client_message_watch_file, shell=True)