nebulous/optimiser-solver
Code Issues Proposed changes

Refine deployment process

Browse Source 
- Build, upload solver docker image.

- Remove deployment job: the solver is not deployed on the Nebulous
  cluster, but is deployed by SAL on each app cluster.

- Adapt helm chart; SAL might use it to deploy the solver.

- Add AMPL license file contents to nebulous secrets store, pass it in
  via helm charts.  Note that it's ok to run without a license file.

- For testing purposes, add a docker-compose.yaml file

- Pacify hadolint: when installing packages in Dockerfile, specify
  package versions.

- Pacify shellcheck: fix warnings in start-solver.sh

- Pacify podman: do not use heredocs for RUN.

- Remove Spring Boot demo application.

Change-Id: I757a440d09082b5824f36a81cb82d6e5c169a699
This commit is contained in:
Rudi Schlatte 2024-02-29 15:45:22 +01:00
parent 5c67c7c7e0
commit 72b2007bc4
15 changed files with 192 additions and 231 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
Dockerfile
View File

@ -1,50 +1,62 @@
# syntax=docker/dockerfile:1.3-labs
FROM fedora:39 AS builder
# To build:
# docker build -t nebulous/solver .
# To run, resulting in a terminal for further experiments:
# docker run -it nebulous/solver
#
# For a shell, to diagnose problems etc.:
# docker run --rm -it --entrypoint /bin/bash nebulous/solver
RUN mkdir -p /solver
WORKDIR /solver
COPY ./ /solver
RUN <<EOF
# Development framework
dnf --assumeyes install gcc-c++ make git-core boost boost-devel ccache qpid-proton-cpp qpid-proton-cpp-devel json-c json-devel json-glib jsoncpp jsoncpp-devel coin-or-Couenne wget
# Dependencies
git clone https://github.com/jarro2783/cxxopts.git CxxOpts
git clone https://github.com/GeirHo/TheronPlusPlus.git Theron++
# Development framework, dependencies
RUN dnf --assumeyes install gcc-c++-13.2.1-6.fc39 make-1:4.4.1-2.fc39 git-core-2.44.0-1.fc39 boost-devel-1.81.0-8.fc39 ccache-4.8.2-2.fc39 qpid-proton-cpp-devel-0.38.0-4.fc39 json-c-0.17-1.fc39 json-devel-3.11.2-3.fc39 json-glib-1.8.0-1.fc39 jsoncpp-1.9.5-5.fc39 jsoncpp-devel-1.9.5-5.fc39 coin-or-Couenne-0.5.8-12.fc39 wget-1.21.3-7.fc39 && \
dnf clean all && \
git clone https://github.com/jarro2783/cxxopts.git CxxOpts && \
git clone https://github.com/GeirHo/TheronPlusPlus.git Theron++ && \
mkdir Theron++/Bin
# Install AMPL library
wget --no-verbose https://portal.ampl.com/external/?url=https://portal.ampl.com/dl/amplce/ampl.linux64.tgz -O ampl.linux64.tgz
tar --file=ampl.linux64.tgz --extract
mv ampl.linux-intel64 AMPL
RUN wget --progress=dot:giga https://portal.ampl.com/external/?url=https://portal.ampl.com/dl/amplce/ampl.linux64.tgz -O ampl.linux64.tgz && \
tar --file=ampl.linux64.tgz --extract && \
mv ampl.linux-intel64 AMPL && \
rm ampl.linux64.tgz
EOF
# Make AMPL shared libraries findable
ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/solver/AMPL:/solver/AMPL/amplapi/lib"
COPY . /solver
# Build solver
RUN make SolverComponent -e THERON=./Theron++ AMPL_INCLUDE=./AMPL/amplapi/include AMPL_LIB=./AMPL/amplapi/lib CxxOpts_DIR=./CxxOpts/include
RUN make SolverComponent -e THERON=./Theron++ AMPL_INCLUDE=./AMPL/amplapi/include AMPL_LIB=./AMPL/amplapi/lib CxxOpts_DIR=./CxxOpts/include && \
make clean
# ============================================================
FROM fedora:39
WORKDIR /solver
RUN dnf --assumeyes install boost qpid-proton-cpp json-c json-glib jsoncpp coin-or-Couenne
RUN dnf --assumeyes install boost-1.81.0-8.fc39 qpid-proton-cpp-0.38.0-4.fc39 json-c-0.17-1.fc39 json-glib-1.8.0-1.fc39 jsoncpp-1.9.5-5.fc39 coin-or-Couenne-0.5.8-12.fc39 && \
dnf clean all
COPY --from=builder /solver /solver
ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/solver/AMPL:/solver/AMPL/amplapi/lib"
# The `SolverComponent` arguments `--ModelDir` and `--AMPLDir` are
# constant, the other arguments can be given on the command line or
# via environment variables:
#
# We set constant `--ModelDir` and `--AMPLDir`; the other arguments can be
# given on the command line, like so (note that `--Endpoint` is mandatory):
# -b amqbroker (variable ACTIVEMQ_HOST, default localhost)
# -P amqpport (variable ACTIVEMQ_PORT, default 5672)
# -u user (variable ACTIVEMQ_USER, default admin)
# -p amqppassword (variable ACTIVEMQ_PASSWORD)
# -e appid (variable APPLICATION_ID)
# -l license (variable AMPL_LICENSE)
#
# docker run nebulous/solver --Broker=https://somewhere.else/ --Endpoint=my-app-id
# The docker can be started with explicit parameters, environment
# variables or a mix of both. Parameters override variables.
#
# For a shell, to diagnose problems etc.:
# docker run --rm -it --entrypoint /bin/bash nebulous/solver
ENTRYPOINT ["/solver/SolverComponent", "--ModelDir=/tmp", "--AMPLDir=/solver/AMPL"]
# docker run -e APPLICATION_ID="my_app_id" nebulous/solver -b="https://amqp.example.com/" -p=s3kr1t
#
ENTRYPOINT ["/solver/start-solver.sh"]

31
charts/nebulous-optimiser-solver/templates/deployment.yaml
View File

@ -33,20 +33,26 @@ spec:
{{- toYaml .Values.securityContext | nindent 12 }}
image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
imagePullPolicy: {{ .Values.image.pullPolicy }}
ports:
- name: http
containerPort: 8080
protocol: TCP
livenessProbe:
httpGet:
path: /
port: http
readinessProbe:
httpGet:
path: /
port: http
resources:
{{- toYaml .Values.resources | nindent 12 }}
env:
- name: APPLICATION_ID
- name: ACTIVEMQ_HOST
value: {{ .Values.activemq.ACTIVEMQ_HOST }}
- name: ACTIVEMQ_PORT
value: {{ .Values.activemq.ACTIVEMQ_PORT }}
- name: ACTIVEMQ_USER
value: {{ .Values.activemq.ACTIVEMQ_USER }}
- name: ACTIVEMQ_PASSWORD
valueFrom:
secretKeyRef:
name: {{ .Values.activemqSecret.secretName }}
key: {{ .Values.activemqSecret.secretKey }}
- name: AMPL_LICENSE
valueFrom:
secretKeyRef:
name: {{ .Values.amplLicense.secretName }}
key: {{ .Values.amplLicense.secretKey }}
{{- with .Values.nodeSelector }}
nodeSelector:
{{- toYaml . | nindent 8 }}
@ -59,3 +65,4 @@ spec:
tolerations:
{{- toYaml . | nindent 8 }}
{{- end }}

19
charts/nebulous-optimiser-solver/values.yaml
View File

@ -5,7 +5,7 @@
replicaCount: 1
image:
repository: "quay.io/nebulous/optimiser-solver-java-spring-boot-demo"
repository: "quay.io/nebulous/optimiser-solver"
pullPolicy: IfNotPresent
# Overrides the image tag whose default is the chart appVersion.
tag: ""
@ -36,10 +36,6 @@ securityContext: {}
# runAsNonRoot: true
# runAsUser: 1000
service:
type: ClusterIP
port: 80
ingress:
enabled: false
className: ""
@ -80,3 +76,16 @@ nodeSelector: {}
tolerations: []
affinity: {}
activemq:
ACTIVEMQ_HOST: nebulous-activemq
ACTIVEMQ_PORT: 5672
ACTIVEMQ_USER: admin
activemqSecret:
secretName: "activeMqPassword"
secretKey: "ACTIVEMQ_PASSWORD"
amplLicense:
secretName: "ampl-license"
secretKey: "AMPL_LICENSE"

46
docker-compose.yaml Normal file
View File

@ -0,0 +1,46 @@
# This file tests starting the solver inside a container. Since the
# solver will not start without a connection to an ActiveMQ broker,
# start one alongside it.
# If we have an application id, run the solver like this:
#
# APPLICATION_ID="my_application_id" docker-compose up
#
services:
activemq:
image: apache/activemq-classic
ports:
- "61616:61616"
- "5672:5672"
- "8161:8161"
networks:
- nebulous-activemq
container_name: myActiveMQ
environment:
ACTIVEMQ_WEB_USER: admin
ACTIVEMQ_WEB_PASSWORD: admin
container_name: activemq
healthcheck:
test: curl -u admin:admin -H origin:localhost --silent --show-error "http://localhost:8161/api/jolokia/exec/org.apache.activemq:type=Broker,brokerName=localhost,service=Health/healthStatus" | tee /dev/tty | grep --silent -e 'Getting Worried' -e 'Good'
interval: 5s
timeout: 5s
retries: 5
solver:
image: nebulous/solver
depends_on:
activemq:
condition: service_healthy
links:
- activemq
networks:
- nebulous-activemq
environment:
ACTIVEMQ_HOST: activemq
APPLICATION_ID: ${APPLICATION_ID:-invalid-application-id}
# Note that an invalid license crashes the solver; no license lets it start
AMPL_LICENSE: ${AMPL_LICENSE}
container_name: solver
networks:
nebulous-activemq:

33
java-spring-boot-demo/.gitignore vendored
View File

@ -1,33 +0,0 @@
HELP.md
target/
!.mvn/wrapper/maven-wrapper.jar
!**/src/main/**/target/
!**/src/test/**/target/
### STS ###
.apt_generated
.classpath
.factorypath
.project
.settings
.springBeans
.sts4-cache
### IntelliJ IDEA ###
.idea
*.iws
*.iml
*.ipr
### NetBeans ###
/nbproject/private/
/nbbuild/
/dist/
/nbdist/
/.nb-gradle/
build/
!**/src/main/**/build/
!**/src/test/**/build/
### VS Code ###
.vscode/

15
java-spring-boot-demo/Dockerfile
View File

@ -1,15 +0,0 @@
#
# Build stage
#
FROM docker.io/library/maven:3.9.2-eclipse-temurin-17 AS build
COPY src /home/app/src
COPY pom.xml /home/app
RUN mvn -f /home/app/pom.xml clean package
#
# Package stage
#
FROM docker.io/library/eclipse-temurin:17-jre
COPY --from=build /home/app/target/demo-0.0.1-SNAPSHOT.jar /usr/local/lib/demo.jar
EXPOSE 8080
ENTRYPOINT ["java","-jar","/usr/local/lib/demo.jar"]

42
java-spring-boot-demo/pom.xml
View File

@ -1,42 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>3.1.0</version>
<relativePath/> <!-- lookup parent from repository -->
</parent>
<groupId>com.example</groupId>
<artifactId>demo</artifactId>
<version>0.0.1-SNAPSHOT</version>
<name>demo</name>
<description>Demo project for Spring Boot</description>
<properties>
<java.version>17</java.version>
</properties>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
</plugin>
</plugins>
</build>
</project>

13
java-spring-boot-demo/src/main/java/com/example/demo/DemoApplication.java
View File

@ -1,13 +0,0 @@
package com.example.demo;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
@SpringBootApplication
public class DemoApplication {
public static void main(String[] args) {
SpringApplication.run(DemoApplication.class, args);
}
}

14
java-spring-boot-demo/src/main/java/com/example/demo/DemoController.java
View File

@ -1,14 +0,0 @@
package com.example.demo;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
@RestController
public class DemoController {
@RequestMapping("/")
public Object root() {
return null;
}
}

0
java-spring-boot-demo/src/main/resources/application.properties
View File

13
java-spring-boot-demo/src/test/java/com/example/demo/DemoApplicationTests.java
View File

@ -1,13 +0,0 @@
package com.example.demo;
import org.junit.jupiter.api.Test;
import org.springframework.boot.test.context.SpringBootTest;
@SpringBootTest
class DemoApplicationTests {
@Test
void contextLoads() {
}
}

2
makefile
View File

@ -13,7 +13,7 @@
# json-devel # Niels Lohmann's JSON library
# coin-or-Couenne # The solver to be used by AMPL
#
# In addtition the problem is formuated using A Mathematical Programming
# In addition the problem is formulated using A Mathematical Programming
# Language (AMPL) and so it should be installed from
# https://portal.ampl.com/user/ampl/request/amplce/trial/new
#

38
start-solver.sh Executable file
View File

@ -0,0 +1,38 @@
#!/usr/bin/bash
# call bash directly since we know we're on Fedora
set -euf -o pipefail
amqpbroker=${ACTIVEMQ_HOST:-localhost}
amqpport=${ACTIVEMQ_PORT:-5672}
amqpuser=${ACTIVEMQ_USER:-admin}
amqppassword=${ACTIVEMQ_PASSWORD:-}
appid=${APPLICATION_ID:-}
license=${AMPL_LICENSE:-}
while getopts b:P:u:p:e:l: name
do
case $name in
b) amqpbroker="$OPTARG";;
p) amqppassword="$OPTARG";;
u) amqpuser="$OPTARG";;
P) amqpport="$OPTARG";;
e) appid="$OPTARG";;
l) license="$OPTARG";;
?) printf "Usage: %s [-b amqbroker] [-P amqpport] [-u user] [-p amqppassword] [-e appid] [-l license]\n -b amqpbroker overrides ACTIVEMQ_HOST (default localhost)\n -P amqpport overrides ACTIVEMQ_PORT (default 5672)\n -u user overrides ACTIVEMQ_USER (default admin)\n -p amqppassword overrides ACTIVEMQ_PASSWORD\n -e appid overrides APPLICATION_ID\n -l license overrides AMPL_LICENSE\n" "$0"
exit 2;;
esac
done
if [ -z "${appid}" ]; then
printf "Missing Application ID, unable to start the solver"
exit 1
fi
if [ -n "${license}" ] ; then
echo "creating license file /solver/AMPL/ampl.lic"
echo "$license" > /solver/AMPL/ampl.lic
fi
echo "Starting SolverComponent: app id='${appid}' broker='${amqpbroker}:${amqpport}'" && sync
exec ./SolverComponent --AMPLDir=/solver/AMPL --ModelDir=/tmp --Broker="$amqpbroker" --Port="$amqpport" --User="$amqpuser" --Pw="$amqppassword" --Endpoint="$appid"

27
zuul.d/jobs.yaml
View File

@ -8,15 +8,15 @@
- nebulous-optimiser-solver-container-images
description: Build the container images.
files: &image_files
- ^java-spring-boot-demo/
- ^/
vars: &image_vars
promote_container_image_job: nebulous-optimiser-solver-upload-container-images
container_images:
- context: java-spring-boot-demo
- context: .
registry: quay.io
repository: quay.io/nebulous/optimiser-solver-java-spring-boot-demo
repository: quay.io/nebulous/optimiser-solver
namespace: nebulous
repo_shortname: optimiser-solver-java-spring-boot-demo
repo_shortname: optimiser-solver
repo_description: ""
- job:
@ -44,7 +44,7 @@
description: Run Hadolint on Dockerfile(s).
vars:
dockerfiles:
- java-spring-boot-demo/Dockerfile
- Dockerfile
- job:
name: nebulous-optimiser-solver-helm-lint
@ -53,20 +53,3 @@
vars:
helm_charts:
- ./charts/nebulous-optimiser-solver
- job:
name: nebulous-optimiser-solver-apply-helm-charts
parent: nebulous-apply-helm-charts
dependencies:
- name: opendev-buildset-registry
soft: false
- name: nebulous-optimiser-solver-build-container-images
soft: true
- name: nebulous-optimiser-solver-upload-container-images
soft: true
requires:
- nebulous-optimiser-solver-container-images
description: Deploy a Kubernetes cluster and apply charts.
vars:
helm_charts:
nebulous-optimiser-solver: ./charts/nebulous-optimiser-solver

4
zuul.d/project.yaml
View File

@ -2,18 +2,14 @@
check:
jobs:
- opendev-buildset-registry
- nebulous-optimiser-solver-helm-lint
- nebulous-optimiser-solver-build-container-images
- nebulous-optimiser-solver-hadolint
- nebulous-platform-apply-helm-charts
- nox-linters
gate:
jobs:
- opendev-buildset-registry
- nebulous-optimiser-solver-helm-lint
- nebulous-optimiser-solver-upload-container-images
- nebulous-optimiser-solver-hadolint
- nebulous-platform-apply-helm-charts
- nox-linters
promote:
jobs: