Add new pod for accessing local files over HTTP

After this change we can access any files under
/localdist/{designer,loadbuild} using an environment variable for the
URL:

  curl $BUILDER_FILES_URL/localdisk/designer/some/file

This change is required by docker build scripts in starlingx/root
for projects that use Loci and have MIRROR_LOCAL set to "yes".

This change requires one to restart the environment (stx-init-env).

CHANGES
=======

* New k8s deployment "stx-builder-files-http". Includes a single pod
  based on the open source "nginx" image. Serves all files under
  /localdisk/{designer,loadbuild} on port 8088.

* k8s.py: fixed problem with distinguishing a pod name that is a prefix
  of another pod name, eg "builder" vs "builder-files-http"

TESTS
=====

* Re-build & re-deploy the helm chart using "stx-init-env"
* Create a dummy docker image recipe that uses Loci and MIRROR_LOCAL and
  make sure it does not fail when cloning

Story: 2010294
Task: 47307

Signed-off-by: Davlet Panech <davlet.panech@windriver.com>
Change-Id: I04e5c6237930fb75a8acbfd65a6479bc06fa6194

stx/lib/stx/config.py

@@ -23,7 +23,7 @@ from stx import utils
 logger = logging.getLogger('STX-Config')
 utils.set_logger(logger)
 
-ALL_CONTAINER_NAMES = ['builder', 'pkgbuilder', 'lat', 'docker', 'repomgr']
+ALL_CONTAINER_NAMES = ['builder', 'builder-files-http', 'pkgbuilder', 'lat', 'docker', 'repomgr']
 
 
 def require_env(var):
@@ -125,6 +125,10 @@ class Config:
         """Container network MTU value"""
         return self._container_mtu
 
+    @property
+    def project_name(self):
+        return self.get('project', 'name')
+
     def _init_kubectl_cmd(self):
         # helm
         self.helm_cmd = 'helm'

stx/lib/stx/k8s.py

@@ -54,8 +54,11 @@ class KubeHelper:
     def get_pod_name(self, dockername):
         '''get the detailed pod name from the four pods.'''
 
-        cmd = self.config.kubectl() + ' get pods | grep Running | ' + \
+        selector = 'app.kubernetes.io/instance=%s,app.kubernetes.io/name=%s' \
-            'grep stx-' + dockername + ' | awk \'{print $1}\' '
+            % (self.config.project_name, 'stx-' + dockername)
+        cmd = self.config.kubectl() + f" get pods --selector '{selector}'" + \
+            " | tail -n +2 | awk '{print $1}'"
+        logger.info('Running: %s', cmd)
         output = subprocess.check_output(cmd, shell=True)
         podname = str(output.decode('utf8').strip())
 

stx/lib/stx/stx_main.py

@@ -64,8 +64,9 @@ task.\t\teg: [start|enter|stop|status|upgrade]')
                                        containers.\n\n')
         control_subparser.add_argument('--dockername',
                                        help='[ builder|pkgbuilder|repomgr|' +
-                                       'lat|docker ]: container name to ' +
+                                       'lat|docker|builder-files-http ]: ' +
-                                       'enter, default: builder\n\n',
+                                       'container name to enter, ' +
+                                       'default: builder\n\n',
                                        required=False)
         control_subparser.set_defaults(handle=self.handlecontrol.handleControl)
 
@@ -157,7 +158,7 @@ remove_repo|search_pkg|upload_pkg|delete_pkg ]')
             action='store_const', const=True)
         shell_subparser.add_argument(
             '--container',
-            metavar='builder|pkgbuilder|lat|repomgr|docker',
+            metavar='builder|pkgbuilder|lat|repomgr|docker|builder-files-http',
             help='Container name (default: builder)')
         shell_subparser.set_defaults(handle=self.handleshell.cmd_shell)
 

stx/stx-build-tools-chart/stx-builder/Chart.yaml

@@ -36,3 +36,6 @@ dependencies:
   - name: stx-docker
     version: "0.1.0"
     repository: "file://dependency_chart/stx-docker"
+  - name: stx-builder-files-http
+    version: "0.1.0"
+    repository: "file://dependency_chart/stx-builder-files-http"

stx/stx-build-tools-chart/stx-builder/configmap/localrc.sample

@@ -66,3 +66,5 @@ export DEBIAN_DISTRIBUTION="@DEBIAN_DISTRIBUTION@"
 export DEBIAN_VERSION="@DEBIAN_VERSION@"
 
 export MAX_CPUS=@MAX_CPUS@
+
+export BUILDER_FILES_URL="http://@PROJECT@-stx-builder-files-http:8088"

stx/stx-build-tools-chart/stx-builder/dependency_chart/stx-builder-files-http/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

stx/stx-build-tools-chart/stx-builder/dependency_chart/stx-builder-files-http/Chart.yaml

@@ -0,0 +1,24 @@
+---
+apiVersion: v2
+name: stx-builder-files-http
+description: A Helm chart for the HTTP server for acessing builder files
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.1.0
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+appVersion: 1.16.0

stx/stx-build-tools-chart/stx-builder/dependency_chart/stx-builder-files-http/configmap/entrypoint.sh

@@ -0,0 +1,9 @@
+#!/bin/sh
+
+set -ex
+
+# Update/replace config files provided in the image
+\cp -f -v /configmap/nginx-default.conf /etc/nginx/conf.d/default.conf
+
+# Call entrypoint script provided by the image
+exec /docker-entrypoint.sh "$@"

stx/stx-build-tools-chart/stx-builder/dependency_chart/stx-builder-files-http/configmap/nginx-default.conf

@@ -0,0 +1,44 @@
+server {
+    listen       80;
+    server_name  localhost;
+
+    #access_log  /var/log/nginx/host.access.log  main;
+
+    location / {
+        root   /usr/share/nginx/html;
+        index  index.html index.htm;
+        autoindex on;
+    }
+
+    error_page  404              /404.html;
+
+    # redirect server error pages to the static page /50x.html
+    #
+    error_page   500 502 503 504  /50x.html;
+    location = /50x.html {
+        root   /usr/share/nginx/html;
+    }
+
+    # proxy the PHP scripts to Apache listening on 127.0.0.1:80
+    #
+    #location ~ \.php$ {
+    #    proxy_pass   http://127.0.0.1;
+    #}
+
+    # pass the PHP scripts to FastCGI server listening on 127.0.0.1:9000
+    #
+    #location ~ \.php$ {
+    #    root           html;
+    #    fastcgi_pass   127.0.0.1:9000;
+    #    fastcgi_index  index.php;
+    #    fastcgi_param  SCRIPT_FILENAME  /scripts$fastcgi_script_name;
+    #    include        fastcgi_params;
+    #}
+
+    # deny access to .htaccess files, if Apache's document root
+    # concurs with nginx's one
+    #
+    #location ~ /\.ht {
+    #    deny  all;
+    #}
+}

stx/stx-build-tools-chart/stx-builder/dependency_chart/stx-builder-files-http/templates/_helpers.tpl

@@ -0,0 +1,63 @@
+{{/* vim: set filetype=mustache: */}}
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "stx-builder-files-http.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "stx-builder-files-http.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "stx-builder-files-http.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "stx-builder-files-http.labels" -}}
+helm.sh/chart: {{ include "stx-builder-files-http.chart" . }}
+{{ include "stx-builder-files-http.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "stx-builder-files-http.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "stx-builder-files-http.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "stx-builder-files-http.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create }}
+{{- default (include "stx-builder-files-http.fullname" .) .Values.serviceAccount.name }}
+{{- else }}
+{{- default "default" .Values.serviceAccount.name }}
+{{- end }}
+{{- end }}

stx/stx-build-tools-chart/stx-builder/dependency_chart/stx-builder-files-http/templates/configmap.yaml

@@ -0,0 +1,7 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: configmap
+data:
+{{ (.Files.Glob "configmap/*").AsConfig | indent 2 }}

stx/stx-build-tools-chart/stx-builder/dependency_chart/stx-builder-files-http/templates/deployment.yaml

@@ -0,0 +1,78 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "stx-builder-files-http.fullname" . }}
+  labels:
+    {{- include "stx-builder-files-http.labels" . | nindent 4 }}
+spec:
+{{- if not .Values.autoscaling.enabled }}
+  replicas: {{ .Values.replicaCount }}
+{{- end }}
+  selector:
+    matchLabels:
+      {{- include "stx-builder-files-http.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+    {{- with .Values.podAnnotations }}
+      annotations:
+        {{- toYaml . | nindent 8 }}
+    {{- end }}
+      labels:
+        {{- include "stx-builder-files-http.selectorLabels" . | nindent 8 }}
+    spec:
+      {{- with .Values.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      securityContext:
+        {{- toYaml .Values.podSecurityContext | nindent 8 }}
+      containers:
+        - name: {{ .Chart.Name }}
+          securityContext:
+            {{- toYaml .Values.securityContext | nindent 12 }}
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          tty: true
+          env:
+          command: [ "/configmap/entrypoint.sh" ]
+          args: [ "nginx", "-g", "daemon off;" ]
+          volumeMounts:
+          - name: localdisk-designer
+            mountPath: {{ .Values.volumes.localdiskDesigner.mountPath }}
+            readOnly: true
+          - name: localdisk-loadbuild
+            mountPath: {{ .Values.volumes.localdiskLoadbuild.mountPath }}
+            readOnly: true
+          - name: configmap
+            mountPath: /configmap
+            readOnly: true
+          resources:
+            {{- toYaml .Values.resources | nindent 12 }}
+
+      dnsConfig:
+        {{- toYaml .Values.dnsConfig | nindent 8 }}
+
+      volumes:
+      - name: localdisk-designer
+        hostPath:
+          path: {{ .Values.global.hostDir }}{{ .Values.volumes.localdiskDesigner.hostPath }}
+      - name: localdisk-loadbuild
+        hostPath:
+          path: {{ .Values.global.hostDir }}{{ .Values.volumes.localdiskLoadbuild.hostPath }}
+      - name: configmap
+        configMap:
+          name: configmap
+          defaultMode: 0700
+      {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}

stx/stx-build-tools-chart/stx-builder/dependency_chart/stx-builder-files-http/templates/service.yaml

@@ -0,0 +1,16 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "stx-builder-files-http.fullname" . }}
+  labels:
+    {{- include "stx-builder-files-http.labels" . | nindent 4 }}
+spec:
+  type: {{ .Values.services.builderFilesHttpService.type }}
+  ports:
+    - port: {{ .Values.services.builderFilesHttpService.port }}
+      targetPort: 80
+      protocol: TCP
+      name: http
+  selector:
+    {{- include "stx-builder-files-http.selectorLabels" . | nindent 4 }}

stx/stx-build-tools-chart/stx-builder/dependency_chart/stx-builder-files-http/values.yaml

@@ -0,0 +1,69 @@
+---
+replicaCount: 1
+
+image:
+  repository: nginx
+  pullPolicy: IfNotPresent
+  # Overrides the image tag whose default is the chart appVersion.
+  tag: "1.23.3"
+
+volumes:
+  localdiskDesigner:
+    mountPath: /usr/share/nginx/html/localdisk/designer
+    hostPath: /localdisk/designer
+  localdiskLoadbuild:
+    mountPath: /usr/share/nginx/html/localdisk/loadbuild
+    hostPath: /localdisk/loadbuild
+
+services:
+  builderFilesHttpService:
+    type: ClusterIP
+    port: 8088
+
+dnsConfig:
+  options:
+    - name: ndots
+      value: "1"
+
+imagePullSecrets: []
+nameOverride: ""
+fullnameOverride: ""
+
+podAnnotations: {}
+
+podSecurityContext: {}
+# fsGroup: 2000
+
+securityContext:
+  privileged: true
+  # capabilities:
+  #   drop:
+  #   - ALL
+  # readOnlyRootFilesystem: true
+  # runAsNonRoot: true
+  # runAsUser: 1000
+
+resources: {}
+# We usually recommend not to specify default resources and to leave this as a conscious
+# choice for the user. This also increases chances charts run on environments with little
+# resources, such as Minikube. If you do want to specify resources, uncomment the following
+# lines, adjust them as necessary, and remove the curly braces after 'resources:'.
+# limits:
+#   cpu: 100m
+#   memory: 128Mi
+# requests:
+#   cpu: 100m
+#   memory: 128Mi
+
+autoscaling:
+  enabled: false
+  minReplicas: 1
+  maxReplicas: 100
+  targetCPUUtilizationPercentage: 80
+  # targetMemoryUtilizationPercentage: 80
+
+nodeSelector: {}
+
+tolerations: []
+
+affinity: {}