Remove outdated Crypto logic and improve ETL configuration

common/pom.xml

@@ -21,6 +21,11 @@
 			<groupId>com.google.guava</groupId>
 			<artifactId>guava</artifactId>
 		</dependency>
+		<dependency>
+			<groupId>jakarta.annotation</groupId>
+			<artifactId>jakarta.annotation-api</artifactId>
+			<version>2.1.1</version>
+		</dependency>
 		<dependency>
 			<groupId>com.fasterxml.jackson.core</groupId>
 			<artifactId>jackson-databind</artifactId>

common/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/crypto/Crypto.java

@@ -1,74 +1,147 @@
 package edu.harvard.hms.dbmi.avillach.hpds.crypto;
 
+import org.apache.commons.io.IOUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
 
+import jakarta.annotation.PostConstruct;
+import javax.crypto.*;
+import javax.crypto.spec.GCMParameterSpec;
+import javax.crypto.spec.SecretKeySpec;
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.nio.charset.Charset;
-import java.security.InvalidAlgorithmParameterException;
-import java.security.InvalidKeyException;
-import java.security.NoSuchAlgorithmException;
-import java.security.SecureRandom;
+import java.security.*;
 import java.util.HashMap;
 
-import javax.crypto.BadPaddingException;
-import javax.crypto.Cipher;
-import javax.crypto.IllegalBlockSizeException;
-import javax.crypto.NoSuchPaddingException;
-import javax.crypto.SecretKey;
-import javax.crypto.ShortBufferException;
-import javax.crypto.spec.GCMParameterSpec;
-import javax.crypto.spec.SecretKeySpec;
-
-import org.apache.commons.io.IOUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
+/**
+ * Provides encryption and decryption functionality using AES-GCM.
+ * <p>
+ * This class manages encryption keys and offers methods to encrypt and decrypt data.
+ * It supports configurable encryption via the {@code encryption.enabled} property.
+ * <p>
+ * Features:
+ * <ul>
+ *     <li>Loads encryption keys from a specified file path.</li>
+ *     <li>Encrypts and decrypts data using AES/GCM/NoPadding.</li>
+ *     <li>Supports optional encryption (data remains unmodified if encryption is disabled).</li>
+ * </ul>
+ * <p>
+ * Encryption behavior is controlled by the {@code encryption.enabled} property, which
+ * is set via {@link org.springframework.beans.factory.annotation.Value} from
+ * {@code application.properties}. When encryption is enabled, the class loads a default
+ * encryption key at initialization.
+ *
+ */
+@Component
 public class Crypto {
 
 	public static final String DEFAULT_KEY_NAME = "DEFAULT";
-
-	// This needs to be set in a static initializer block to be overridable in tests.
-	private static final String DEFAULT_ENCRYPTION_KEY_PATH;
-	static{
-		DEFAULT_ENCRYPTION_KEY_PATH = "/opt/local/hpds/encryption_key";
-	}
+	private static final String DEFAULT_ENCRYPTION_KEY_PATH = "/opt/local/hpds/encryption_key";
 
 	private static final Logger LOGGER = LoggerFactory.getLogger(Crypto.class);
+	private static final HashMap<String, byte[]> keys = new HashMap<>();
+
+	@Value("${encryption.enabled:true}")
+	private boolean encryptionEnabled;
 
-	private static final HashMap<String, byte[]> keys = new HashMap<String, byte[]>();
+	public static boolean ENCRYPTION_ENABLED = true;
 
+	@PostConstruct
+	public void init() {
+		ENCRYPTION_ENABLED = encryptionEnabled;
+		LOGGER.info("ENCRYPTION_ENABLED set to: {}", ENCRYPTION_ENABLED);
+		loadDefaultKey();
+	}
+
+	/**
+	 * Loads the default encryption key from the predefined file path.
+	 * <p>
+	 * This method checks if encryption is enabled before attempting to load the key.
+	 * If encryption is disabled, no action is taken.
+	 * <p>
+	 * The key is loaded using {@link #loadKey(String, String)} with the default key name
+	 * and default encryption key file path.
+	 */
 	public static void loadDefaultKey() {
-		loadKey(DEFAULT_KEY_NAME, DEFAULT_ENCRYPTION_KEY_PATH);
+		if (ENCRYPTION_ENABLED) {
+			loadKey(DEFAULT_KEY_NAME, DEFAULT_ENCRYPTION_KEY_PATH);
+		}
 	}
 
+	/**
+	 * Loads an encryption key from the specified file path and stores it in memory.
+	 * <p>
+	 * The key is read as a string from the file, trimmed of any extra spaces, and
+	 * converted into a byte array before being stored in the key map.
+	 * <p>
+	 * If the key file is not found or an error occurs while reading, an error is logged.
+	 *
+	 * @param keyName  The name under which the key will be stored.
+	 * @param filePath The file path from which the encryption key is loaded.
+	 */
 	public static void loadKey(String keyName, String filePath) {
 		try {
 			setKey(keyName, IOUtils.toString(new FileInputStream(filePath), Charset.forName("UTF-8")).trim().getBytes());
-			LOGGER.info("****LOADED CRYPTO KEY****");	
+			LOGGER.info("****LOADED CRYPTO KEY****");
 		} catch (IOException e) {
 			LOGGER.error("****CRYPTO KEY NOT FOUND****", e);
 		}
 	}
 
-	public static byte[] encryptData(byte[] plaintextBytes) {
-		return encryptData(DEFAULT_KEY_NAME, plaintextBytes);
+	/**
+	 * Encrypts the given plaintext using the default encryption key.
+	 * <p>
+	 * If encryption is disabled, the plaintext is returned as-is.
+	 * This method delegates encryption to {@link #encryptData(String, byte[])}
+	 * using the default key.
+	 *
+	 * @param plaintext The byte array to be encrypted.
+	 * @return The encrypted byte array, or the original plaintext if encryption is disabled.
+	 */
+	public static byte[] encryptData(byte[] plaintext) {
+		return encryptData(DEFAULT_KEY_NAME, plaintext);
 	}
-
-	public static byte[] encryptData(String keyName, byte[] plaintextBytes) {
+
+	/**
+	 * Encrypts the given plaintext using the specified encryption key.
+	 * <p>
+	 * This method uses AES/GCM/NoPadding encryption with a randomly generated IV.
+	 * The IV is included in the output for decryption purposes.
+	 * <p>
+	 * The method returns a byte array structured as follows:
+	 * - First 4 bytes: The length of the IV.
+	 * - Next IV-length bytes: The IV itself.
+	 * - Remaining bytes: The encrypted ciphertext.
+	 * <p>
+	 * If encryption is disabled, the plaintext is returned unmodified.
+	 *
+	 * @param keyName   The name of the encryption key to use.
+	 * @param plaintext The byte array containing the data to encrypt.
+	 * @return The encrypted byte array, or the original plaintext if encryption is disabled.
+	 * @throws RuntimeException If an error occurs during encryption.
+	 */
+	public static byte[] encryptData(String keyName, byte[] plaintext) {
+		if (!ENCRYPTION_ENABLED) {
+			return plaintext;
+		}
+
 		byte[] key = keys.get(keyName);
 		SecureRandom secureRandom = new SecureRandom();
 		SecretKey secretKey = new SecretKeySpec(key, "AES");
-		byte[] iv = new byte[12]; //NEVER REUSE THIS IV WITH SAME KEY
+		byte[] iv = new byte[12]; // NEVER REUSE THIS IV WITH SAME KEY
 		secureRandom.nextBytes(iv);
 		byte[] cipherText;
 		Cipher cipher;
 		try {
 			cipher = Cipher.getInstance("AES/GCM/NoPadding");
-			GCMParameterSpec parameterSpec = new GCMParameterSpec(128, iv); //128 bit auth tag length
+			GCMParameterSpec parameterSpec = new GCMParameterSpec(128, iv); // 128-bit auth tag length
 			cipher.init(Cipher.ENCRYPT_MODE, secretKey, parameterSpec);
-			cipherText = new byte[cipher.getOutputSize(plaintextBytes.length)];
-			cipher.doFinal(plaintextBytes, 0, plaintextBytes.length, cipherText, 0);
+			cipherText = new byte[cipher.getOutputSize(plaintext.length)];
+			cipher.doFinal(plaintext, 0, plaintext.length, cipherText, 0);
 			LOGGER.debug("Length of cipherText : " + cipherText.length);
 			ByteBuffer byteBuffer = ByteBuffer.allocate(4 + iv.length + cipherText.length);
 			byteBuffer.putInt(iv.length);
@@ -81,13 +154,40 @@ public static byte[] encryptData(String keyName, byte[] plaintextBytes) {
 		}
 	}
 
-	public static byte[] decryptData(byte[] encrypted) {
-		return decryptData(DEFAULT_KEY_NAME, encrypted);
+	/**
+	 * Decrypts the provided data using the default encryption key.
+	 * <p>
+	 * If encryption is disabled, the method returns the input data as-is.
+	 *
+	 * @param data The byte array to be decrypted.
+	 * @return The decrypted byte array, or the original data if encryption is disabled.
+	 */
+	public static byte[] decryptData(byte[] data) {
+		return decryptData(DEFAULT_KEY_NAME, data);
 	}
 
-	public static byte[] decryptData(String keyName, byte[] encrypted) {
+	/**
+	 * Decrypts the provided data using the specified encryption key.
+	 * <p>
+	 * If encryption is disabled, the method returns the input data as-is.
+	 * <p>
+	 * The method assumes the input data is structured as follows:
+	 * - First 4 bytes: The length of the IV (Initialization Vector).
+	 * - Next IV-length bytes: The actual IV.
+	 * - Remaining bytes: The ciphertext.
+	 *
+	 * @param keyName The name of the encryption key to use for decryption.
+	 * @param data The byte array containing the encrypted data.
+	 * @return The decrypted byte array, or the original data if encryption is disabled.
+	 * @throws RuntimeException If an error occurs during decryption.
+	 */
+	public static byte[] decryptData(String keyName, byte[] data) {
+		if (!ENCRYPTION_ENABLED) {
+			return data;
+		}
+
 		byte[] key = keys.get(keyName);
-		ByteBuffer byteBuffer = ByteBuffer.wrap(encrypted);
+		ByteBuffer byteBuffer = ByteBuffer.wrap(data);
 		int ivLength = byteBuffer.getInt();
 		byte[] iv = new byte[ivLength];
 		byteBuffer.get(iv);
@@ -98,7 +198,8 @@ public static byte[] decryptData(String keyName, byte[] encrypted) {
 			cipher = Cipher.getInstance("AES/GCM/NoPadding");
 			cipher.init(Cipher.DECRYPT_MODE, new SecretKeySpec(key, "AES"), new GCMParameterSpec(128, iv));
 			return cipher.doFinal(cipherText);
-		} catch (NoSuchAlgorithmException | NoSuchPaddingException | InvalidKeyException | InvalidAlgorithmParameterException | IllegalBlockSizeException | BadPaddingException e) {
+		} catch (NoSuchAlgorithmException | NoSuchPaddingException | InvalidKeyException | InvalidAlgorithmParameterException |
+				 IllegalBlockSizeException | BadPaddingException e) {
 			throw new RuntimeException("Exception caught trying to decrypt data : " + e, e);
 		}
 	}
@@ -110,5 +211,4 @@ private static void setKey(String keyName, byte[] key) {
 	public static boolean hasKey(String keyName) {
 		return keys.containsKey(keyName);
 	}
-
-}
+}

docker/pic-sure-hpds-etl/Dockerfile

@@ -1,34 +1,19 @@
-FROM maven:3.9.4-amazoncorretto-21 AS build
-
-RUN yum update -y && yum install -y git && yum clean all
-
-WORKDIR /app
-
-COPY .m2 /root/.m2
-
-COPY . .
-
-RUN mvn clean install -DskipTests
-
-FROM eclipse-temurin:21-alpine
-
-RUN apk add --no-cache --purge -uU bash curl wget unzip gnupg openssl && \
-    rm -rf /var/cache/apk/* /tmp/*
-
-WORKDIR /app
-COPY --from=build /app/docker/pic-sure-hpds-etl/SQLLoader-jar-with-dependencies.jar .
-COPY --from=build /app/docker/pic-sure-hpds-etl/CSVLoader-jar-with-dependencies.jar .
-COPY --from=build /app/docker/pic-sure-hpds-etl/CSVLoaderNewSearch-jar-with-dependencies.jar .
-COPY --from=build /app/docker/pic-sure-hpds-etl/CSVDumper-jar-with-dependencies.jar .
-COPY --from=build /app/docker/pic-sure-hpds-etl/VCFLocalLoader-jar-with-dependencies.jar .
-COPY --from=build /app/docker/pic-sure-hpds-etl/VariantMetadataLoader-jar-with-dependencies.jar .
-COPY --from=build /app/docker/pic-sure-hpds-etl/UnifiedVCFLocalLoader-jar-with-dependencies.jar .
-COPY --from=build /app/docker/pic-sure-hpds-etl/MultialleleCounter-jar-with-dependencies.jar .
-COPY --from=build /app/docker/pic-sure-hpds-etl/RekeyDataset-jar-with-dependencies.jar .
-COPY --from=build /app/docker/pic-sure-hpds-etl/RemoveConceptFromMetadata-jar-with-dependencies.jar .
-COPY --from=build /app/docker/pic-sure-hpds-etl/HideAnnotationCategoryValue-jar-with-dependencies.jar .
-COPY --from=build /app/docker/pic-sure-hpds-etl/SequentialLoader-jar-with-dependencies.jar .
-COPY --from=build /app/docker/pic-sure-hpds-etl/LowRAMMultiCSVLoader-jar-with-dependencies.jar .
-COPY --from=build /app/docker/pic-sure-hpds-etl/create_key.sh .
-
-ENTRYPOINT ["sh", "-c", "java $JAVA_OPTS -Xmx${HEAPSIZE:-2048}m -jar ${LOADER_NAME:-CSVLoader}-jar-with-dependencies.jar"]
+FROM openjdk:21-jdk-slim AS build
+
+RUN apt-get update -y && apt-get install -y gnupg openssl && rm -rf /var/lib/apt/lists/*
+
+ADD create_key.sh .
+ADD SQLLoader-jar-with-dependencies.jar .
+ADD CSVLoader-jar-with-dependencies.jar .
+ADD CSVLoaderNewSearch-jar-with-dependencies.jar .
+ADD CSVDumper-jar-with-dependencies.jar .
+ADD VCFLocalLoader-jar-with-dependencies.jar .
+ADD VariantMetadataLoader-jar-with-dependencies.jar .
+ADD UnifiedVCFLocalLoader-jar-with-dependencies.jar .
+ADD MultialleleCounter-jar-with-dependencies.jar .
+ADD RekeyDataset-jar-with-dependencies.jar .
+ADD RemoveConceptFromMetadata-jar-with-dependencies.jar .
+ADD HideAnnotationCategoryValue-jar-with-dependencies.jar .
+ADD SequentialLoader-jar-with-dependencies.jar .
+
+ENTRYPOINT java $JAVA_OPTS -Xmx${HEAPSIZE:-2048}m -jar ${LOADER_NAME:-CSVLoader}-jar-with-dependencies.jar

etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/etl/LoadingStore.java

@@ -29,14 +29,14 @@ public class LoadingStore {
 	TreeMap<String, ColumnMeta> metadataMap = new TreeMap<>();
 
 	private static Logger log = LoggerFactory.getLogger(LoadingStore.class);
-	
+
 	public LoadingCache<String, PhenoCube> store = CacheBuilder.newBuilder()
-			.maximumSize(16)
+			.maximumSize(2048)
 			.removalListener(new RemovalListener<String, PhenoCube>() {
 
 				@Override
 				public void onRemoval(RemovalNotification<String, PhenoCube> arg0) {
-					log.info("removing " + arg0.getKey());
+					//log.debug("Cache removal and writing to disk: " + arg0.getKey());
 					if(arg0.getValue().getLoadingMap()!=null) {
 						complete(arg0.getValue());
 					}