nebulous/optimiser-solver
Code Issues Proposed changes

Merge "Add ci process to solver"

Browse Source
This commit is contained in:
Zuul 2024-03-22 09:55:56 +00:00 committed by Gerrit Code Review
commit f17b524c09
15 changed files with 167 additions and 202 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

65
Dockerfile
View File

@ -1,50 +1,63 @@
# syntax=docker/dockerfile:1.3-labs
FROM fedora:39 AS builder
# To build:
# docker build -t nebulous/solver .
# To run, resulting in a terminal for further experiments:
# docker run -it nebulous/solver
#
# For a shell, to diagnose problems etc.:
# docker run --rm -it --entrypoint /bin/bash nebulous/solver
RUN mkdir -p /solver
WORKDIR /solver
COPY ./ /solver
RUN <<EOF
# Development framework
dnf --assumeyes install gcc-c++ make git-core boost boost-devel ccache qpid-proton-cpp qpid-proton-cpp-devel json-c json-devel json-glib jsoncpp jsoncpp-devel coin-or-Couenne wget
# Dependencies
git clone https://github.com/jarro2783/cxxopts.git CxxOpts
git clone https://github.com/GeirHo/TheronPlusPlus.git Theron++
# Development framework, dependencies
RUN dnf --assumeyes install gcc-c++-13.2.1-7.fc39 make-1:4.4.1-2.fc39 git-core-2.44.0-1.fc39 boost-devel-1.81.0-8.fc39 ccache-4.8.2-2.fc39 qpid-proton-cpp-devel-0.38.0-4.fc39 json-c-0.17-1.fc39 json-devel-3.11.2-3.fc39 json-glib-1.8.0-1.fc39 jsoncpp-1.9.5-5.fc39 jsoncpp-devel-1.9.5-5.fc39 coin-or-Couenne-0.5.8-12.fc39 wget-1.21.3-7.fc39 && \
dnf clean all && \
git clone https://github.com/jarro2783/cxxopts.git CxxOpts && \
git clone https://github.com/GeirHo/TheronPlusPlus.git Theron++ && \
mkdir Theron++/Bin
# Install AMPL library
wget --no-verbose https://portal.ampl.com/external/?url=https://portal.ampl.com/dl/amplce/ampl.linux64.tgz -O ampl.linux64.tgz
tar --file=ampl.linux64.tgz --extract
mv ampl.linux-intel64 AMPL
# Install AMPL library
RUN wget --progress=dot:giga https://portal.ampl.com/external/?url=https://portal.ampl.com/dl/amplce/ampl.linux64.tgz -O ampl.linux64.tgz && \
tar --file=ampl.linux64.tgz --extract && \
mv ampl.linux-intel64 AMPL && \
rm ampl.linux64.tgz
EOF
# Make AMPL shared libraries findable
ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/solver/AMPL:/solver/AMPL/amplapi/lib"
COPY . /solver
# Build solver
RUN make SolverComponent -e THERON=./Theron++ AMPL_INCLUDE=./AMPL/amplapi/include AMPL_LIB=./AMPL/amplapi/lib CxxOpts_DIR=./CxxOpts/include
RUN make SolverComponent -e THERON=./Theron++ AMPL_INCLUDE=./AMPL/amplapi/include AMPL_LIB=./AMPL/amplapi/lib CxxOpts_DIR=./CxxOpts/include && \
make clean
# ============================================================
FROM fedora:39
WORKDIR /solver
RUN dnf --assumeyes install boost qpid-proton-cpp json-c json-glib jsoncpp coin-or-Couenne
RUN dnf --assumeyes install boost-1.81.0-8.fc39 qpid-proton-cpp-0.38.0-4.fc39 json-c-0.17-1.fc39 json-glib-1.8.0-1.fc39 jsoncpp-1.9.5-5.fc39 coin-or-Couenne-0.5.8-12.fc39 && \
dnf clean all
COPY --from=builder /solver /solver
ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/solver/AMPL:/solver/AMPL/amplapi/lib"
#
# We set constant `--ModelDir` and `--AMPLDir`; the other arguments can be
# given on the command line, like so (note that `--Endpoint` is mandatory):
#
# docker run nebulous/solver --Broker=https://somewhere.else/ --Endpoint=my-app-id
#
# For a shell, to diagnose problems etc.:
# docker run --rm -it --entrypoint /bin/bash nebulous/solver
ENTRYPOINT ["/solver/SolverComponent", "--ModelDir=/tmp", "--AMPLDir=/solver/AMPL"]
# The `SolverComponent` arguments `--ModelDir` and `--AMPLDir` are
# constant, the other arguments can be given on the command line or
# via environment variables:
#
# -b amqbroker (variable ACTIVEMQ_HOST, default localhost)
# -P amqpport (variable ACTIVEMQ_PORT, default 5672)
# -u user (variable ACTIVEMQ_USER, default admin)
# -p amqppassword (variable ACTIVEMQ_PASSWORD)
# -e appid (variable APPLICATION_ID)
# -l license (variable AMPL_LICENSE)
#
# The docker can be started with explicit parameters, environment
# variables or a mix of both. Parameters override variables.
#
# docker run -e APPLICATION_ID="my_app_id" nebulous/solver -b="https://amqp.example.com/" -p=s3kr1t
#
RUN chmod +x /solver/start-solver.sh
ENTRYPOINT ["/solver/start-solver.sh"]

7
charts/nebulous-optimiser-solver/templates/activemq-secret.yaml Normal file
View File

@ -0,0 +1,7 @@
apiVersion: v1
kind: Secret
metadata:
name: {{ .Values.activemqSecret.secretName }}
type: Opaque
data:
{{ .Values.activemqSecret.secretKey }}: {{ .Values.activemq.ACTIVEMQ_PASSWORD }}

7
charts/nebulous-optimiser-solver/templates/ampl-license-secret.yaml Normal file
View File

@ -0,0 +1,7 @@
apiVersion: v1
kind: Secret
metadata:
name: {{ .Values.amplLicense.secretName }}
type: Opaque
data:
{{ .Values.amplLicense.secretKey }}: {{ .Values.amplLicense.keyValue }}

27
charts/nebulous-optimiser-solver/templates/deployment.yaml
View File

@ -37,16 +37,27 @@ spec:
- name: http
containerPort: 8080
protocol: TCP
livenessProbe:
httpGet:
path: /
port: http
readinessProbe:
httpGet:
path: /
port: http
resources:
{{- toYaml .Values.resources | nindent 12 }}
env:
- name: APPLICATION_ID
value: {{ .Values.application.id }}
- name: ACTIVEMQ_HOST
value: {{ .Values.activemq.ACTIVEMQ_HOST }}
- name: ACTIVEMQ_PORT
value: {{ .Values.activemq.ACTIVEMQ_PORT | quote }}
- name: ACTIVEMQ_USER
value: {{ .Values.activemq.ACTIVEMQ_USER }}
- name: ACTIVEMQ_PASSWORD
valueFrom:
secretKeyRef:
name: {{ .Values.activemqSecret.secretName }}
key: {{ .Values.activemqSecret.secretKey }}
- name: AMPL_LICENSE
valueFrom:
secretKeyRef:
name: {{ .Values.amplLicense.secretName }}
key: {{ .Values.amplLicense.secretKey }}
{{- with .Values.nodeSelector }}
nodeSelector:
{{- toYaml . | nindent 8 }}

21
charts/nebulous-optimiser-solver/values.yaml
View File

@ -5,7 +5,7 @@
replicaCount: 1
image:
repository: "quay.io/nebulous/optimiser-solver-java-spring-boot-demo"
repository: "quay.io/nebulous/optimiser-solver"
pullPolicy: IfNotPresent
# Overrides the image tag whose default is the chart appVersion.
tag: ""
@ -80,3 +80,22 @@ nodeSelector: {}
tolerations: []
affinity: {}
activemq:
ACTIVEMQ_HOST: nebulous-activemq
ACTIVEMQ_PORT: 5672
ACTIVEMQ_USER: admin
activemqSecret:
secretName: "activemq-password"
secretKey: "ACTIVEMQ_PASSWORD"
keyValue: "YWN0aXZlbXFwYXNzd29yZA=="
amplLicense:
secretName: "ampl-license"
secretKey: "AMPL_LICENSE"
keyValue: ""
application:
id: "dummy-app"

33
java-spring-boot-demo/.gitignore vendored
View File

@ -1,33 +0,0 @@
HELP.md
target/
!.mvn/wrapper/maven-wrapper.jar
!**/src/main/**/target/
!**/src/test/**/target/
### STS ###
.apt_generated
.classpath
.factorypath
.project
.settings
.springBeans
.sts4-cache
### IntelliJ IDEA ###
.idea
*.iws
*.iml
*.ipr
### NetBeans ###
/nbproject/private/
/nbbuild/
/dist/
/nbdist/
/.nb-gradle/
build/
!**/src/main/**/build/
!**/src/test/**/build/
### VS Code ###
.vscode/

15
java-spring-boot-demo/Dockerfile
View File

@ -1,15 +0,0 @@
#
# Build stage
#
FROM docker.io/library/maven:3.9.2-eclipse-temurin-17 AS build
COPY src /home/app/src
COPY pom.xml /home/app
RUN mvn -f /home/app/pom.xml clean package
#
# Package stage
#
FROM docker.io/library/eclipse-temurin:17-jre
COPY --from=build /home/app/target/demo-0.0.1-SNAPSHOT.jar /usr/local/lib/demo.jar
EXPOSE 8080
ENTRYPOINT ["java","-jar","/usr/local/lib/demo.jar"]

42
java-spring-boot-demo/pom.xml
View File

@ -1,42 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>3.1.0</version>
<relativePath/> <!-- lookup parent from repository -->
</parent>
<groupId>com.example</groupId>
<artifactId>demo</artifactId>
<version>0.0.1-SNAPSHOT</version>
<name>demo</name>
<description>Demo project for Spring Boot</description>
<properties>
<java.version>17</java.version>
</properties>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
</plugin>
</plugins>
</build>
</project>

13
java-spring-boot-demo/src/main/java/com/example/demo/DemoApplication.java
View File

@ -1,13 +0,0 @@
package com.example.demo;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
@SpringBootApplication
public class DemoApplication {
public static void main(String[] args) {
SpringApplication.run(DemoApplication.class, args);
}
}

14
java-spring-boot-demo/src/main/java/com/example/demo/DemoController.java
View File

@ -1,14 +0,0 @@
package com.example.demo;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
@RestController
public class DemoController {
@RequestMapping("/")
public Object root() {
return null;
}
}

0
java-spring-boot-demo/src/main/resources/application.properties
View File

13
java-spring-boot-demo/src/test/java/com/example/demo/DemoApplicationTests.java
View File

@ -1,13 +0,0 @@
package com.example.demo;
import org.junit.jupiter.api.Test;
import org.springframework.boot.test.context.SpringBootTest;
@SpringBootTest
class DemoApplicationTests {
@Test
void contextLoads() {
}
}

64
makefile
View File

@ -3,21 +3,21 @@
# Solver component
#
# The NebulOuS solver consists of several interacting actors using the AMQ
# interface of the Theron++ framework.
# interface of the Theron++ framework.
#
# The following packages should be available on Fedora prior to compiling
# The following packages should be available on Fedora prior to compiling
# the file
#
# ccache # for effcient C++ compilations
# qpid-proton-cpp* # Qpid Proton Active Message Queue protocol API
# json-devel # Niels Lohmann's JSON library
#
# ccache # for effcient C++ compilations
# qpid-proton-cpp* # Qpid Proton Active Message Queue protocol API
# json-devel # Niels Lohmann's JSON library
# coin-or-Couenne # The solver to be used by AMPL
#
# In addtition the problem is formuated using A Mathematical Programming
# In addition the problem is formulated using A Mathematical Programming
# Language (AMPL) and so it should be installed from
# https://portal.ampl.com/user/ampl/request/amplce/trial/new
#
# There are source code dependencies that should be cloned to local disk
# There are source code dependencies that should be cloned to local disk
#
# Theron++ # https://github.com/GeirHo/TheronPlusPlus.git
# cxxopts # https://github.com/jarro2783/cxxopts.git
@ -39,7 +39,7 @@ RM = rm -f
# Paths
#------------------------------------------------------------------------------
#
# The default values of the paths are given here to be overridden by build
# The default values of the paths are given here to be overridden by build
# definitions on the command line for creating the component container.
#
# Location of the Theron++ framework relative to this make file and the code
@ -54,8 +54,8 @@ AMPL_INCLUDE ?= /opt/AMPL/amplapi/include
AMPL_LIB ?= /opt/AMPL/amplapi/lib
# The solver component uses the CxxOpts class for parsing the command line
# options since it is header only and lighter than the Options library of
# The solver component uses the CxxOpts class for parsing the command line
# options since it is header only and lighter than the Options library of
# boost, which seems to have lost the most recent C++ features. The CxxOpts
# library can be cloned from https://github.com/jarro2783/cxxopts
@ -65,22 +65,22 @@ CxxOpts_DIR ?= /home/GHo/Documents/Code/CxxOpts/include
# Options for the compiler and linker
#------------------------------------------------------------------------------
#
# Optimisation -O3 is the highest level of optimisation and should be used
# with production code. -Og is the code optimising and offering debugging
# Optimisation -O3 is the highest level of optimisation and should be used
# with production code. -Og is the code optimising and offering debugging
# transparency and should be use while the code is under development
OPTIMISATION_FLAG = -Og
# It is useful to let the compiler generate the dependencies for the various
# files, and the following will produce .d files that can be included at the
# end. The -MMD flag is equivalent with -MD, but the latter will include system
# headers in the output (which we do not need here). The -MP includes an
# empty rule to create the dependencies so that make would not create any errors
# It is useful to let the compiler generate the dependencies for the various
# files, and the following will produce .d files that can be included at the
# end. The -MMD flag is equivalent with -MD, but the latter will include system
# headers in the output (which we do not need here). The -MP includes an
# empty rule to create the dependencies so that make would not create any errors
# if the file name changes.
DEPENDENCY_FLAGS = -MMD -MP
# Options
# Options
GENERAL_OPTIONS = -Wall -std=c++23 -ggdb -D_DEBUG
INCLUDE_DIRECTORIES = -I. -I/usr/include -I$(THERON) -I$(AMPL_INCLUDE) \
@ -89,12 +89,12 @@ INCLUDE_DIRECTORIES = -I. -I/usr/include -I$(THERON) -I$(AMPL_INCLUDE) \
CXXFLAGS = $(GENERAL_OPTIONS) $(INCLUDE_DIRECTORIES) $(DEPENDENCY_FLAGS) \
$(OPTIMISATION_FLAG)
# Putting it together as the actual options given to the compiler and the
# linker. Note that pthread is needed on Linux systems since it seems to
# be the underlying implementation of std::thread. Note that it is
# necessary to use the "gold" linker as the standard linker requires
# the object files in the right order, which is hard to ensure with
# an archive, and the "gold" linker manages this just fine, but it
# Putting it together as the actual options given to the compiler and the
# linker. Note that pthread is needed on Linux systems since it seems to
# be the underlying implementation of std::thread. Note that it is
# necessary to use the "gold" linker as the standard linker requires
# the object files in the right order, which is hard to ensure with
# an archive, and the "gold" linker manages this just fine, but it
# requires the full static path to the custom Theron library.
CFLAGS = $(DEPENDENCY_FLAGS) $(OPTIMISATION_FLAG) $(GENERAL_OPTIONS)
@ -105,13 +105,13 @@ LDFLAGS = -fuse-ld=gold -ggdb -D_DEBUG -pthread $(THERON)/Theron++.a \
# Theron library
#------------------------------------------------------------------------------
#
# The Theron++ library must be built first and the following two targets
# ensures that Make will check if the libray is up-to-date or build it if
# The Theron++ library must be built first and the following two targets
# ensures that Make will check if the libray is up-to-date or build it if
# it is not.
.PHONY: $(THERON)/Theron++.a
.PHONY: $(THERON)/Theron++.a
$(THERON)/Theron++.a:
$(THERON)/Theron++.a:
make -C $(THERON) Library
#------------------------------------------------------------------------------
@ -139,13 +139,13 @@ $(OBJECTS_DIR)/%.o : %.cpp
#------------------------------------------------------------------------------
#
# The only real target is to build the solver component whenever some of
# The only real target is to build the solver component whenever some of
# the object files or the solver actors.
SolverComponent: $(SOLVER_OBJECTS) $(THERON)/Theron++.a
$(CC) -o SolverComponent $(CXXFLAGS) $(SOLVER_OBJECTS) $(LDFLAGS)
# There is also a standard target to clean the automatically generated build
# There is also a standard target to clean the automatically generated build
# files
clean:
@ -156,4 +156,4 @@ clean:
#------------------------------------------------------------------------------
#
-include $(SOLVER_OBJECTS:.o=.d)
-include $(SOLVER_OBJECTS:.o=.d)

38
start-solver.sh Normal file
View File

@ -0,0 +1,38 @@
#!/usr/bin/bash
# call bash directly since we know we're on Fedora
set -euf -o pipefail
amqpbroker=${ACTIVEMQ_HOST:-localhost}
amqpport=${ACTIVEMQ_PORT:-5672}
amqpuser=${ACTIVEMQ_USER:-admin}
amqppassword=${ACTIVEMQ_PASSWORD:-admin}
appid=${APPLICATION_ID:-}
license=${AMPL_LICENSE:-}
while getopts b:P:u:p:e:l: name
do
case $name in
b) amqpbroker="$OPTARG";;
p) amqppassword="$OPTARG";;
u) amqpuser="$OPTARG";;
P) amqpport="$OPTARG";;
e) appid="$OPTARG";;
l) license="$OPTARG";;
?) printf "Usage: %s [-b amqbroker] [-P amqpport] [-u user] [-p amqppassword] [-e appid] [-l license]\n -b amqpbroker overrides ACTIVEMQ_HOST (default localhost)\n -P amqpport overrides ACTIVEMQ_PORT (default 5672)\n -u user overrides ACTIVEMQ_USER (default admin)\n -p amqppassword overrides ACTIVEMQ_PASSWORD\n -e appid overrides APPLICATION_ID\n -l license overrides AMPL_LICENSE\n" "$0"
exit 2;;
esac
done
if [ -z "${appid}" ]; then
printf "Missing Application ID, unable to start the solver"
exit 1
fi
if [ -n "${license}" ] ; then
echo "creating license file /solver/AMPL/ampl.lic"
echo "$license" > /solver/AMPL/ampl.lic
fi
echo "Starting SolverComponent: app id='${appid}' broker='${amqpbroker}:${amqpport}'" && sync
exec ./SolverComponent --AMPLDir=/solver/AMPL --ModelDir=/tmp --Broker="$amqpbroker" --Port="$amqpport" --User="$amqpuser" --Pw="$amqppassword" --Endpoint="$appid"

10
zuul.d/jobs.yaml
View File

@ -8,15 +8,15 @@
- nebulous-optimiser-solver-container-images
description: Build the container images.
files: &image_files
- ^java-spring-boot-demo/
- ^/
vars: &image_vars
promote_container_image_job: nebulous-optimiser-solver-upload-container-images
container_images:
- context: java-spring-boot-demo
- context: .
registry: quay.io
repository: quay.io/nebulous/optimiser-solver-java-spring-boot-demo
repository: quay.io/nebulous/optimiser-solver
namespace: nebulous
repo_shortname: optimiser-solver-java-spring-boot-demo
repo_shortname: optimiser-solver
repo_description: ""
- job:
@ -44,7 +44,7 @@
description: Run Hadolint on Dockerfile(s).
vars:
dockerfiles:
- java-spring-boot-demo/Dockerfile
- Dockerfile
- job:
name: nebulous-optimiser-solver-helm-lint