starlingx/tools
Code Issues Proposed changes

Add new pod for accessing local files over HTTP

Browse Source 
After this change we can access any files under
/localdist/{designer,loadbuild} using an environment variable for the
URL:

  curl $BUILDER_FILES_URL/localdisk/designer/some/file

This change is required by docker build scripts in starlingx/root
for projects that use Loci and have MIRROR_LOCAL set to "yes".

This change requires one to restart the environment (stx-init-env).

CHANGES
=======

* New k8s deployment "stx-builder-files-http". Includes a single pod
  based on the open source "nginx" image. Serves all files under
  /localdisk/{designer,loadbuild} on port 8088.

* k8s.py: fixed problem with distinguishing a pod name that is a prefix
  of another pod name, eg "builder" vs "builder-files-http"

TESTS
=====

* Re-build & re-deploy the helm chart using "stx-init-env"
* Create a dummy docker image recipe that uses Loci and MIRROR_LOCAL and
  make sure it does not fail when cloning

Story: 2010294
Task: 47307

Signed-off-by: Davlet Panech <davlet.panech@windriver.com>
Change-Id: I04e5c6237930fb75a8acbfd65a6479bc06fa6194
This commit is contained in:
Davlet Panech 2023-02-07 12:46:56 -05:00
parent 4fdc291540
commit 16dd9d0a78
14 changed files with 352 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
stx/lib/stx/config.py
View File

@ -23,7 +23,7 @@ from stx import utils
logger = logging.getLogger('STX-Config')
utils.set_logger(logger)
ALL_CONTAINER_NAMES = ['builder', 'pkgbuilder', 'lat', 'docker', 'repomgr']
ALL_CONTAINER_NAMES = ['builder', 'builder-files-http', 'pkgbuilder', 'lat', 'docker', 'repomgr']
def require_env(var):
@ -125,6 +125,10 @@ class Config:
"""Container network MTU value"""
return self._container_mtu
@property
def project_name(self):
return self.get('project', 'name')
def _init_kubectl_cmd(self):
# helm
self.helm_cmd = 'helm'

7
stx/lib/stx/k8s.py
View File

@ -54,8 +54,11 @@ class KubeHelper:
def get_pod_name(self, dockername):
'''get the detailed pod name from the four pods.'''
cmd = self.config.kubectl() + ' get pods | grep Running | ' + \
'grep stx-' + dockername + ' | awk \'{print $1}\' '
selector = 'app.kubernetes.io/instance=%s,app.kubernetes.io/name=%s' \
% (self.config.project_name, 'stx-' + dockername)
cmd = self.config.kubectl() + f" get pods --selector '{selector}'" + \
" | tail -n +2 | awk '{print $1}'"
logger.info('Running: %s', cmd)
output = subprocess.check_output(cmd, shell=True)
podname = str(output.decode('utf8').strip())

7
stx/lib/stx/stx_main.py
View File

@ -64,8 +64,9 @@ task.\t\teg: [start|enter|stop|status|upgrade]')
containers.\n\n')
control_subparser.add_argument('--dockername',
help='[ builder|pkgbuilder|repomgr|' +
'lat|docker ]: container name to ' +
'enter, default: builder\n\n',
'lat|docker|builder-files-http ]: ' +
'container name to enter, ' +
'default: builder\n\n',
required=False)
control_subparser.set_defaults(handle=self.handlecontrol.handleControl)
@ -157,7 +158,7 @@ remove_repo|search_pkg|upload_pkg|delete_pkg ]')
action='store_const', const=True)
shell_subparser.add_argument(
'--container',
metavar='builder|pkgbuilder|lat|repomgr|docker',
metavar='builder|pkgbuilder|lat|repomgr|docker|builder-files-http',
help='Container name (default: builder)')
shell_subparser.set_defaults(handle=self.handleshell.cmd_shell)

3
stx/stx-build-tools-chart/stx-builder/Chart.yaml
View File

@ -36,3 +36,6 @@ dependencies:
- name: stx-docker
version: "0.1.0"
repository: "file://dependency_chart/stx-docker"
- name: stx-builder-files-http
version: "0.1.0"
repository: "file://dependency_chart/stx-builder-files-http"

2
stx/stx-build-tools-chart/stx-builder/configmap/localrc.sample
View File

@ -66,3 +66,5 @@ export DEBIAN_DISTRIBUTION="@DEBIAN_DISTRIBUTION@"
export DEBIAN_VERSION="@DEBIAN_VERSION@"
export MAX_CPUS=@MAX_CPUS@
export BUILDER_FILES_URL="http://@PROJECT@-stx-builder-files-http:8088"

23
stx/stx-build-tools-chart/stx-builder/dependency_chart/stx-builder-files-http/.helmignore Normal file
View File

@ -0,0 +1,23 @@
# Patterns to ignore when building packages.
# This supports shell glob matching, relative path matching, and
# negation (prefixed with !). Only one pattern per line.
.DS_Store
# Common VCS dirs
.git/
.gitignore
.bzr/
.bzrignore
.hg/
.hgignore
.svn/
# Common backup files
*.swp
*.bak
*.tmp
*.orig
*~
# Various IDEs
.project
.idea/
*.tmproj
.vscode/

24
stx/stx-build-tools-chart/stx-builder/dependency_chart/stx-builder-files-http/Chart.yaml Normal file
View File

@ -0,0 +1,24 @@
---
apiVersion: v2
name: stx-builder-files-http
description: A Helm chart for the HTTP server for acessing builder files
# A chart can be either an 'application' or a 'library' chart.
#
# Application charts are a collection of templates that can be packaged into versioned archives
# to be deployed.
#
# Library charts provide useful utilities or functions for the chart developer. They're included as
# a dependency of application charts to inject those utilities and functions into the rendering
# pipeline. Library charts do not define any templates and therefore cannot be deployed.
type: application
# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
# Versions are expected to follow Semantic Versioning (https://semver.org/)
version: 0.1.0
# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application. Versions are not expected to
# follow Semantic Versioning. They should reflect the version the application is using.
appVersion: 1.16.0

9
stx/stx-build-tools-chart/stx-builder/dependency_chart/stx-builder-files-http/configmap/entrypoint.sh Normal file
View File

@ -0,0 +1,9 @@
#!/bin/sh
set -ex
# Update/replace config files provided in the image
\cp -f -v /configmap/nginx-default.conf /etc/nginx/conf.d/default.conf
# Call entrypoint script provided by the image
exec /docker-entrypoint.sh "$@"

44
stx/stx-build-tools-chart/stx-builder/dependency_chart/stx-builder-files-http/configmap/nginx-default.conf Normal file
View File

@ -0,0 +1,44 @@
server {
listen 80;
server_name localhost;
#access_log /var/log/nginx/host.access.log main;
location / {
root /usr/share/nginx/html;
index index.html index.htm;
autoindex on;
}
error_page 404 /404.html;
# redirect server error pages to the static page /50x.html
#
error_page 500 502 503 504 /50x.html;
location = /50x.html {
root /usr/share/nginx/html;
}
# proxy the PHP scripts to Apache listening on 127.0.0.1:80
#
#location ~ \.php$ {
# proxy_pass http://127.0.0.1;
#}
# pass the PHP scripts to FastCGI server listening on 127.0.0.1:9000
#
#location ~ \.php$ {
# root html;
# fastcgi_pass 127.0.0.1:9000;
# fastcgi_index index.php;
# fastcgi_param SCRIPT_FILENAME /scripts$fastcgi_script_name;
# include fastcgi_params;
#}
# deny access to .htaccess files, if Apache's document root
# concurs with nginx's one
#
#location ~ /\.ht {
# deny all;
#}
}

63
stx/stx-build-tools-chart/stx-builder/dependency_chart/stx-builder-files-http/templates/_helpers.tpl Normal file
View File

@ -0,0 +1,63 @@
{{/* vim: set filetype=mustache: */}}
{{/*
Expand the name of the chart.
*/}}
{{- define "stx-builder-files-http.name" -}}
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
{{- end }}
{{/*
Create a default fully qualified app name.
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
If release name contains chart name it will be used as a full name.
*/}}
{{- define "stx-builder-files-http.fullname" -}}
{{- if .Values.fullnameOverride }}
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
{{- else }}
{{- $name := default .Chart.Name .Values.nameOverride }}
{{- if contains $name .Release.Name }}
{{- .Release.Name | trunc 63 | trimSuffix "-" }}
{{- else }}
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
{{- end }}
{{- end }}
{{- end }}
{{/*
Create chart name and version as used by the chart label.
*/}}
{{- define "stx-builder-files-http.chart" -}}
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
{{- end }}
{{/*
Common labels
*/}}
{{- define "stx-builder-files-http.labels" -}}
helm.sh/chart: {{ include "stx-builder-files-http.chart" . }}
{{ include "stx-builder-files-http.selectorLabels" . }}
{{- if .Chart.AppVersion }}
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
{{- end }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
{{- end }}
{{/*
Selector labels
*/}}
{{- define "stx-builder-files-http.selectorLabels" -}}
app.kubernetes.io/name: {{ include "stx-builder-files-http.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
{{- end }}
{{/*
Create the name of the service account to use
*/}}
{{- define "stx-builder-files-http.serviceAccountName" -}}
{{- if .Values.serviceAccount.create }}
{{- default (include "stx-builder-files-http.fullname" .) .Values.serviceAccount.name }}
{{- else }}
{{- default "default" .Values.serviceAccount.name }}
{{- end }}
{{- end }}

7
stx/stx-build-tools-chart/stx-builder/dependency_chart/stx-builder-files-http/templates/configmap.yaml Normal file
View File

@ -0,0 +1,7 @@
---
apiVersion: v1
kind: ConfigMap
metadata:
name: configmap
data:
{{ (.Files.Glob "configmap/*").AsConfig | indent 2 }}

78
stx/stx-build-tools-chart/stx-builder/dependency_chart/stx-builder-files-http/templates/deployment.yaml Normal file
View File

@ -0,0 +1,78 @@
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "stx-builder-files-http.fullname" . }}
labels:
{{- include "stx-builder-files-http.labels" . | nindent 4 }}
spec:
{{- if not .Values.autoscaling.enabled }}
replicas: {{ .Values.replicaCount }}
{{- end }}
selector:
matchLabels:
{{- include "stx-builder-files-http.selectorLabels" . | nindent 6 }}
template:
metadata:
{{- with .Values.podAnnotations }}
annotations:
{{- toYaml . | nindent 8 }}
{{- end }}
labels:
{{- include "stx-builder-files-http.selectorLabels" . | nindent 8 }}
spec:
{{- with .Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
securityContext:
{{- toYaml .Values.podSecurityContext | nindent 8 }}
containers:
- name: {{ .Chart.Name }}
securityContext:
{{- toYaml .Values.securityContext | nindent 12 }}
image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
imagePullPolicy: {{ .Values.image.pullPolicy }}
tty: true
env:
command: [ "/configmap/entrypoint.sh" ]
args: [ "nginx", "-g", "daemon off;" ]
volumeMounts:
- name: localdisk-designer
mountPath: {{ .Values.volumes.localdiskDesigner.mountPath }}
readOnly: true
- name: localdisk-loadbuild
mountPath: {{ .Values.volumes.localdiskLoadbuild.mountPath }}
readOnly: true
- name: configmap
mountPath: /configmap
readOnly: true
resources:
{{- toYaml .Values.resources | nindent 12 }}
dnsConfig:
{{- toYaml .Values.dnsConfig | nindent 8 }}
volumes:
- name: localdisk-designer
hostPath:
path: {{ .Values.global.hostDir }}{{ .Values.volumes.localdiskDesigner.hostPath }}
- name: localdisk-loadbuild
hostPath:
path: {{ .Values.global.hostDir }}{{ .Values.volumes.localdiskLoadbuild.hostPath }}
- name: configmap
configMap:
name: configmap
defaultMode: 0700
{{- with .Values.nodeSelector }}
nodeSelector:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.affinity }}
affinity:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.tolerations }}
tolerations:
{{- toYaml . | nindent 8 }}
{{- end }}

16
stx/stx-build-tools-chart/stx-builder/dependency_chart/stx-builder-files-http/templates/service.yaml Normal file
View File

@ -0,0 +1,16 @@
---
apiVersion: v1
kind: Service
metadata:
name: {{ include "stx-builder-files-http.fullname" . }}
labels:
{{- include "stx-builder-files-http.labels" . | nindent 4 }}
spec:
type: {{ .Values.services.builderFilesHttpService.type }}
ports:
- port: {{ .Values.services.builderFilesHttpService.port }}
targetPort: 80
protocol: TCP
name: http
selector:
{{- include "stx-builder-files-http.selectorLabels" . | nindent 4 }}

69
stx/stx-build-tools-chart/stx-builder/dependency_chart/stx-builder-files-http/values.yaml Normal file
View File

@ -0,0 +1,69 @@
---
replicaCount: 1
image:
repository: nginx
pullPolicy: IfNotPresent
# Overrides the image tag whose default is the chart appVersion.
tag: "1.23.3"
volumes:
localdiskDesigner:
mountPath: /usr/share/nginx/html/localdisk/designer
hostPath: /localdisk/designer
localdiskLoadbuild:
mountPath: /usr/share/nginx/html/localdisk/loadbuild
hostPath: /localdisk/loadbuild
services:
builderFilesHttpService:
type: ClusterIP
port: 8088
dnsConfig:
options:
- name: ndots
value: "1"
imagePullSecrets: []
nameOverride: ""
fullnameOverride: ""
podAnnotations: {}
podSecurityContext: {}
# fsGroup: 2000
securityContext:
privileged: true
# capabilities:
# drop:
# - ALL
# readOnlyRootFilesystem: true
# runAsNonRoot: true
# runAsUser: 1000
resources: {}
# We usually recommend not to specify default resources and to leave this as a conscious
# choice for the user. This also increases chances charts run on environments with little
# resources, such as Minikube. If you do want to specify resources, uncomment the following
# lines, adjust them as necessary, and remove the curly braces after 'resources:'.
# limits:
# cpu: 100m
# memory: 128Mi
# requests:
# cpu: 100m
# memory: 128Mi
autoscaling:
enabled: false
minReplicas: 1
maxReplicas: 100
targetCPUUtilizationPercentage: 80
# targetMemoryUtilizationPercentage: 80
nodeSelector: {}
tolerations: []
affinity: {}