Merge pull request #1476 from NASA-AMMOS/bug/sequencing-server-memory-leak

Sequencing Server Memory Leak

Author: goetzrrGit

deployment/docker-compose.yml

@@ -126,6 +126,7 @@ services:
       SEQUENCING_DB_PASSWORD: "${SEQUENCING_PASSWORD}"
       SEQUENCING_LOCAL_STORE: /usr/src/app/sequencing_file_store
       SEQUENCING_WORKER_NUM: 8
+      SEQUENCING_MAX_WORKER_NUM : 8
       SEQUENCING_MAX_WORKER_HEAP_MB: 1000
       TRANSPILER_ENABLED: "true"
     image: "${REPOSITORY_DOCKER_URL}/aerie-sequencing:${DOCKER_TAG}"

docker-compose.yml

@@ -90,6 +90,7 @@ services:
       SEQUENCING_DB_PASSWORD: "${SEQUENCING_PASSWORD}"
       SEQUENCING_LOCAL_STORE: /usr/src/app/sequencing_file_store
       SEQUENCING_WORKER_NUM: 8
+      SEQUENCING_MAX_WORKER_NUM : 8
       SEQUENCING_MAX_WORKER_HEAP_MB: 1000
       TRANSPILER_ENABLED: "true"
     image: aerie_sequencing

sequencing-server/src/app.ts

@@ -51,6 +51,7 @@ export let graphqlClient = new GraphQLClient(getEnv().MERLIN_GRAPHQL_URL, {
 export const piscina = new Piscina({
   filename: new URL('worker.js', import.meta.url).pathname,
   minThreads: parseInt(getEnv().SEQUENCING_WORKER_NUM),
+  maxThreads: parseInt(getEnv().SEQUENCING_MAX_WORKER_NUM),
   resourceLimits: { maxOldGenerationSizeMb: parseInt(getEnv().SEQUENCING_MAX_WORKER_HEAP_MB) },
 });
 export const promiseThrottler = new PromiseThrottler(parseInt(getEnv().SEQUENCING_WORKER_NUM) - 2);
@@ -301,6 +302,7 @@ app.listen(PORT, () => {
   logger.info(`connected to port ${PORT}`);
   logger.info(`Worker pool initialized:
               Total workers started: ${piscina.threads.length},
+              Max Workers Allowed: ${getEnv().SEQUENCING_MAX_WORKER_NUM},
               Heap Size per Worker: ${getEnv().SEQUENCING_MAX_WORKER_HEAP_MB} MB`);
 
   if (getEnv().TRANSPILER_ENABLED === 'true') {

sequencing-server/src/env.ts

@@ -10,6 +10,7 @@ export type Env = {
   SEQUENCING_DB_PASSWORD: string;
   STORAGE: string;
   SEQUENCING_WORKER_NUM: string;
+  SEQUENCING_MAX_WORKER_NUM : string
   SEQUENCING_MAX_WORKER_HEAP_MB: string;
   TRANSPILER_ENABLED: string;
 };
@@ -27,6 +28,7 @@ export const defaultEnv: Env = {
   STORAGE: 'sequencing_file_store',
   SEQUENCING_WORKER_NUM: '8',
   SEQUENCING_MAX_WORKER_HEAP_MB: '1000',
+  SEQUENCING_MAX_WORKER_NUM : '8',
   TRANSPILER_ENABLED: 'true',
 };
 
@@ -44,6 +46,7 @@ export function getEnv(): Env {
   const SEQUENCING_DB_PASSWORD = env['SEQUENCING_DB_PASSWORD'] ?? defaultEnv.SEQUENCING_DB_PASSWORD;
   const STORAGE = env['SEQUENCING_LOCAL_STORE'] ?? defaultEnv.STORAGE;
   const SEQUENCING_WORKER_NUM = env['SEQUENCING_WORKER_NUM'] ?? defaultEnv.SEQUENCING_WORKER_NUM;
+  const SEQUENCING_MAX_WORKER_NUM = env['SEQUENCING_MAX_WORKER_NUM'] ?? defaultEnv.SEQUENCING_MAX_WORKER_NUM;
   const SEQUENCING_MAX_WORKER_HEAP_MB =
     env['SEQUENCING_MAX_WORKER_HEAP_MB'] ?? defaultEnv.SEQUENCING_MAX_WORKER_HEAP_MB;
   const TRANSPILER_ENABLED = env['TRANSPILER_ENABLED'] ?? defaultEnv.TRANSPILER_ENABLED;
@@ -59,6 +62,7 @@ export function getEnv(): Env {
     SEQUENCING_DB_PASSWORD,
     STORAGE,
     SEQUENCING_WORKER_NUM,
+    SEQUENCING_MAX_WORKER_NUM,
     SEQUENCING_MAX_WORKER_HEAP_MB,
     TRANSPILER_ENABLED,
   };