26 review and improve docker compose scripts

.env.template

@@ -0,0 +1,40 @@
+# Variables set in here are picked up by docker-compose automatically
+
+# Service ports on host
+#TOMCAT_PORT=8081
+#NODEJS_PORT=3000
+#JAVADEBUG_PORT=5005
+#KEYCLOAK_PORT=8080
+
+# Linux developers should uncomment this line:
+#DOCKER_GATEWAY_HOST=172.17.0.1
+
+# SERVICE_PROTOCOL is the protocol used to access the service from the client:
+# "http" or "https"
+#SERVICE_PROTOCOL=http
+
+# SERVICE_PORT is the port used by the client to access the service.  If the
+# client is accessing the embedded reverse proxy directly then this should be
+# the same as the PROXY_EXTERNAL_PORT. If the client is going via another
+# reverse proxy then this should be the port that proxy listens on (e.g. 443).
+#SERVICE_PORT=8089
+
+# PROXY_EXTERNAL_PORT is the port that the reverse proxy is going to listen on.
+# If there are multiple deployments on the same machine then they all need
+# different PROXY_EXTERNAL_PORT settings.
+#PROXY_EXTERNAL_PORT=8089
+
+# Location of the Spyderisk System Modeller documentation to be used in the web application:
+#DOCUMENTATION_URL=https://spyderisk.org/documentation/modeller/latest/
+
+# Secret shared for communication between Spyderisk and Keycloak services
+# The docker-compose.yaml file demonstrates how to insert the value into the Keycloak configuration
+#KEYCLOAK_CREDENTIALS_SECRET=DfkQBcVpjbO6gTMXMBUBfHe45UmFhGxk
+
+# Keycloak admin account credentials
+#KEYCLOAK_ADMIN_USERNAME=admin
+#KEYCLOAK_ADMIN_PASSWORD=password
+
+# Spring boot application property reset.on.start, default value is true
+#RESET_ON_START=false
+

.github/workflows/test.yml

@@ -26,7 +26,7 @@ jobs:
     - name: Build software
       run: |
         docker-compose -f docker-compose.yml -f docker-compose.test.yml pull  # Force download of any newer dependency images
-        docker-compose -f docker-compose.yml -f docker-compose.test.yml up -d --build
+        docker-compose -f docker-compose.yml -f docker-compose.test.yml --env-file .env.template up -d --build
         docker-compose -f docker-compose.yml -f docker-compose.test.yml images  # Useful for debugging
         docker-compose -f docker-compose.yml -f docker-compose.test.yml images -q | xargs docker inspect | grep -C3 RepoTags  # Useful for debugging
 
@@ -48,4 +48,4 @@ jobs:
 
     - name: Return test success
       # If the tests failed then make the whole job fail
-      run: test ${FAIL} -eq 0
+      run: test ${FAIL} -eq 0

docker-compose.override.yml

@@ -17,7 +17,7 @@ services:
 
       # Use the DOCKER_GATEWAY_HOST value (in `.env` file) if it is set (used in Linux), fall back on "host.docker.internal"
       # See https://sthasantosh.com.np/2020/05/23/docker-tip-how-to-use-the-hosts-ip-address-inside-a-docker-container-on-macos-windows-and-linux/
-      KEYCLOAK_AUTH_SERVER_URL: http://${DOCKER_GATEWAY_HOST:-host.docker.internal}:${KEYCLOAK_PORT}/auth/
+      KEYCLOAK_AUTH_SERVER_URL: http://${DOCKER_GATEWAY_HOST:-host.docker.internal}:${KEYCLOAK_PORT:-8080}/auth/
     volumes:
       # Volumes of type "bind" mount a folder from the host machine.
 
@@ -62,15 +62,15 @@ services:
         target: /tmp
     ports:
       # port 3000 is used by the NodeJS server when doing frontend development
-      - ${NODEJS_PORT}:3000
+      - ${NODEJS_PORT:-3000}:3000
       # port 8081 is used by Spring Boot for the backend
-      - ${TOMCAT_PORT}:8081
+      - ${TOMCAT_PORT:-8081}:8081
       # port 5005 is used by the Java debugger
-      - ${JAVADEBUG_PORT}:5005
+      - ${JAVADEBUG_PORT:-5005}:5005
 
   keycloak:
     ports:
-      - ${KEYCLOAK_PORT}:8080
+      - ${KEYCLOAK_PORT:-8080}:8080
 
 volumes:
   gradle:

docker-compose.test.yml

@@ -1,5 +1,5 @@
 # This file is an overlay for the base `docker-compose.yml` file.
-# To use it, do `docker-compose -f docker-compose.yml -f docker-compose.test.yml up`.
+# To use it, do `docker-compose -f docker-compose.yml -f docker-compose.test.yml --env-file .env.template up`.
 # This docker-compose file is used by the CI pipeline to execute the test.
 # The tests can be executed using e.g. `docker-compose exec -T ssm sh -c 'cd /system-modeller && gradle test'`
 
@@ -15,3 +15,5 @@ services:
       # Port 8080 is the one exposed by keycloak by default and not related to any port-mapping.
       # Environment variables are not available at build time (only at runtime).
       KEYCLOAK_AUTH_SERVER_URL: http://keycloak:8080/auth/
+    env_file:
+      - .env.template

docker-compose.yml

@@ -4,6 +4,32 @@
 version: '3.7'
 
 services:
+  proxy:
+    image: nginx:stable-alpine3.17
+    # If there are multiple deployments of the SSM on the same host then they each need to expose a different PROXY_EXTERNAL_PORT
+    container_name: ssm-proxy
+    ports:
+      - ${PROXY_EXTERNAL_PORT:-8089}:80
+    expose:
+      - 80
+    environment:
+      scheme: ${SERVICE_PROTOCOL:-http}
+      server_port: ${SERVICE_PORT:-8089}
+      kc_proxy_pass: '${EXTERNAL_KEYCLOAK_AUTH_SERVER_URL:-http://keycloak:8080/auth}'
+      documentation_url: '${DOCUMENTATION_URL:-https://spyderisk.org/documentation/modeller/latest/}'
+      tomcat_port: ${TOMCAT_PORT:-8081}
+    # When nginx starts it does a health check on the "upstream" servers and if none of them in a group are present it will fail.
+    # Hence, nginx must start after the ssm and keycloak.
+    # See https://docs.nginx.com/nginx/admin-guide/load-balancer/tcp-health-check/
+    restart: on-failure
+    depends_on:
+      - ssm
+      - keycloak
+    entrypoint: /tmp/import/entrypoint.sh
+    volumes:
+      - type: bind
+        source: ./provisioning/nginx
+        target: /tmp/import
 
   ssm:
     build:
@@ -15,7 +41,8 @@ services:
       # These env variables override the values in application.properties
       # https://docs.spring.io/spring-boot/docs/current/reference/html/spring-boot-features.html#boot-features-external-config
       SPRING_DATA_MONGODB_HOST: mongo
-      KEYCLOAK_CREDENTIALS_SECRET: ${KEYCLOAK_CREDENTIALS_SECRET}
+      KEYCLOAK_CREDENTIALS_SECRET: ${KEYCLOAK_CREDENTIALS_SECRET:-DfkQBcVpjbO6gTMXMBUBfHe45UmFhGxk}
+      RESET_ON_START: ${RESET_ON_START:-true}
     volumes:
       # Persistent named volume for the jena-tdb storage
       - type: volume
@@ -30,7 +57,7 @@ services:
       - keycloak
 
   mongo:
-    image: mongo:4.2.5-bionic
+    image: mongo:5.0.16-focal
     volumes:
       - type: volume
         source: mongo-db
@@ -45,9 +72,9 @@ services:
     # Override the normal entrypoint of `/opt/keycloak/bin/kc.sh`.
     entrypoint: /tmp/import/entrypoint.sh
     environment:
-      KEYCLOAK_ADMIN: ${KEYCLOAK_ADMIN_USERNAME}
-      KEYCLOAK_ADMIN_PASSWORD: ${KEYCLOAK_ADMIN_PASSWORD}
-      KEYCLOAK_CREDENTIALS_SECRET: ${KEYCLOAK_CREDENTIALS_SECRET}
+      KEYCLOAK_ADMIN: ${KEYCLOAK_ADMIN_USERNAME:-admin}
+      KEYCLOAK_ADMIN_PASSWORD: ${KEYCLOAK_ADMIN_PASSWORD:-password}
+      KEYCLOAK_CREDENTIALS_SECRET: ${KEYCLOAK_CREDENTIALS_SECRET:-DfkQBcVpjbO6gTMXMBUBfHe45UmFhGxk}
       PROXY_ADDRESS_FORWARDING: '${KEYCLOAK_PROXY_ADDRESS_FORWARDING:-false}'
     volumes:
       - type: bind

provisioning/keycloak/healthcheck.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+#
+# This software is distributed WITHOUT ANY WARRANTY, without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, except where
+# stated in the Licence Agreement supplied with the software.
+#
+# Received by Panos Melas, University of Southampton IT Innovation Centre, in
+# good faith on 28/04/2023 from
+# https://stackoverflow.com/questions/53301299/add-healthcheck-in-keycloak-docker-swarm-service
+
+# This is a workaround script for the missing cURL command not included in
+# newer keycloak docker images. The script checks the health status of the
+# keycloak service at http://localhost:8080/auth/health/ready
+# {
+#    "status": "UP",
+#    "checks": [
+#    ]
+# }
+#
+
+exec 3<>/dev/tcp/localhost/8080
+
+echo -e "GET /auth/health/ready HTTP/1.1\nhost: localhost:8080\n" >&3
+
+timeout --preserve-status 1 cat <&3 | grep -m 1 '"status":\s\+"UP"'
+ERROR=$?
+
+exec 3<&-
+exec 3>&-
+
+exit $ERROR

provisioning/nginx/.gitignore

@@ -0,0 +1 @@
+nginx.conf

provisioning/nginx/entrypoint.sh

@@ -0,0 +1,16 @@
+#!/bin/sh
+
+# This script is a replacement for the standard entrypoint script used in the
+# nginx image. It expects that there is a directory from the host mounted at
+# `/tmp/import` containing the `nginx.conf.template` file. It expects that the
+# `SERVICE_PROTOCOL` and `SERVICE_PORT` environment variables are set in the
+# `.env` file.
+
+# The `endsubst` replaces `scheme` and `server_port` variables and copies
+# `nginx.conf.template` to `nginx.conf` Then the `nginx` command is run the
+# nginx service.
+
+envsubst '$${scheme} $${server_port} $${kc_proxy_pass} $${documentation_url} $${tomcat_port}' < \
+    /tmp/import/nginx.conf.template > /etc/nginx/nginx.conf
+
+nginx -g 'daemon off;'

provisioning/nginx/nginx.conf.template

@@ -0,0 +1,78 @@
+
+user  nginx;
+worker_processes  auto;
+
+error_log  /var/log/nginx/error.log warn;
+pid        /var/run/nginx.pid;
+
+
+events {
+    worker_connections  1024;
+}
+
+
+http {
+    include       /etc/nginx/mime.types;
+    default_type  application/octet-stream;
+
+    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                      '$status $body_bytes_sent "$http_referer" '
+                      '"$http_user_agent" "$http_host"';
+
+    access_log  /var/log/nginx/access.log  main;
+
+    sendfile        on;
+
+    # This first line sets the HTTP Host header in the request to the one that was received by nginx (preserving it).
+    # It includes the request port and is essential for the SSM to redirect the browser to the correct Keycloak address.
+    proxy_set_header Host              $http_host;
+    # These two are used by keycloak if it is set to proxy address forwarding mode:
+    proxy_set_header X-Forwarded-For   $proxy_add_x_forwarded_for;
+    proxy_set_header X-Forwarded-Proto $scheme;  # If this proxy is behind another reverse proxy that terminates SSL then $scheme must be explicitly set to "https"
+    # These headers do not seem to have any effect but can't hurt and may be used in logging.
+    proxy_set_header X-Real-IP         $remote_addr;
+    proxy_set_header X-Forwarded-Host  $server_name;
+    proxy_set_header X-Forwarded-Port  $server_port;
+
+    server {
+        listen 80;
+        absolute_redirect off;
+
+        # For each endpoint, we are explicit about how to treat URLs without a trailing slash. Not doing this causes some hidden problems.
+        # $scheme and $http_host come from the request URL and are e.g. "https" and "some.domain:port". They are needed to preserve this part of the URL.
+
+        location / {
+            return 301 $scheme://$http_host/system-modeller/;
+        }
+        location = /system-modeller {
+            rewrite ^(.*)$ $scheme://$http_host$1/ redirect;
+        }
+        location /system-modeller/ {
+            client_max_body_size 100M;
+            #proxy_pass http://ssm:8080/system-modeller/;
+            proxy_pass http://ssm:$tomcat_port/system-modeller/;
+        }
+
+        location = /auth {
+            rewrite ^(.*)$ $scheme://$http_host$1/ redirect;
+        }
+        location /auth/ {
+            proxy_pass $kc_proxy_pass/;
+        }
+
+        location = /documentation {
+            rewrite ^(.*)$ $scheme://$http_host$1/ redirect;
+        }
+        location /documentation/ {
+            rewrite ^/documentation/(.*) $documentation_url$1 redirect;
+        }
+
+        location /documentation/redirect/ {
+            # Here we do a browser redirect of the URL for things starting /documentation/redirect
+            # These are special addresses embedded in the SSM where we want to decouple the documentation identifier in the SSM 
+            # from the actual place in the documentation that should be accessed.
+            # Right now, all locations defined in the SSM need to redirect to a heading in the Reference Guide so this is easy.
+            rewrite ^/documentation/redirect/(.*) $documentation_url/Reference%20Guide/#$1 redirect;
+        }
+    }
+}

provisioning/ssm/entrypoint.sh

@@ -0,0 +1,13 @@
+#!/bin/sh
+
+# This script is a replacement for the standard entrypoint script used in the
+# catalina image. It expects that there is a directory from the host mounted at
+# `/tmp/startup` containing the `entrypoint.sh` file.
+#
+
+#https://github-registry-files.githubusercontent.com/619830179/a8341180-decd-11ed-8428-25c0158a373b?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAIWNJYAX4CSVEH53A%2F20230530%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20230530T140817Z&X-Amz-Expires=300&X-Amz-Signature=66ef5deb95184f3eda6b0c13f1043a870403d62964b39f10363512bc6201d640&X-Amz-SignedHeaders=host&actor_id=0&key_id=0&repo_id=619830179&response-content-disposition=filename%3Ddomain-network-6a3-1-1.zip&response-content-type=application%2Foctet-stream
+
+wget https://github.com/SPYDERISK/domain-network/packages/1826148 -O /tmp/knowledgebases/domain-network.zip
+
+echo "entrypoint: starting up catalina"
+/var/lib/tomcat/bin/catalina.sh run

src/main/resources/application.properties

@@ -10,7 +10,7 @@ spring.application.version=@VERSION@
 server.port=8081
 ######### WARNING: This property should match (API_)END_POINT in /src/main/webapp/config/config.js #########
 server.servlet.contextPath=/system-modeller
-reset.on.start=true
+reset.on.start=${RESET_ON_START:true}
 
 # Knowledgebase source location (domain model zip bundles to install)
 knowledgebases.source.folder=/code/knowledgebases