ENT-6947 Intern common types to reduce heap footprint (corda#7239)

ENT-6947: Implement interning for SecureHash, CordaX500Name, PublicKey, AsbtractParty and SignatureAttachmentConstraint, including automatic detection of internable types off companion objects in AMQP & Kyro deserialization.  In some cases, add new factory methods to companion objects, and make main code base use them.

Performance tested in performance cluster with no negative impact visible (so default concurrency setting seems okay).

Testing suggests 5-6x memory saving for tokens in TokensSDK in memory selector.  Should see approx. 1 million tokens per GB or better (1.5 million for the tokens we tested with).

Author: rick-r3

.ci/api-current.txt

@@ -118,7 +118,7 @@ class StandaloneCordaRPClientTest {
 
         val hash = HashingInputStream(Hashing.sha256(), rpcProxy.openAttachment(id)).use { it ->
             it.copyTo(NULL_OUTPUT_STREAM)
-            SecureHash.SHA256(it.hash().asBytes())
+            SecureHash.createSHA256(it.hash().asBytes())
         }
         assertEquals(attachment.sha256, hash)
     }
@@ -133,7 +133,7 @@ class StandaloneCordaRPClientTest {
 
         val hash = HashingInputStream(Hashing.sha256(), rpcProxy.openAttachment(id)).use { it ->
             it.copyTo(NULL_OUTPUT_STREAM)
-            SecureHash.SHA256(it.hash().asBytes())
+            SecureHash.createSHA256(it.hash().asBytes())
         }
         assertEquals(attachment.sha256, hash)
     }

.idea/codeStyles/Project.xml

@@ -9,4 +9,4 @@ package net.corda.common.logging
  * (originally added to source control for ease of use)
  */
 
-internal const val CURRENT_MAJOR_RELEASE = "4.10-SNAPSHOT"
+internal const val CURRENT_MAJOR_RELEASE = "4.10-SNAPSHOT"

.idea/codeStyles/codeStyleConfig.xml

@@ -1,6 +1,7 @@
 import net.corda.gradle.jarfilter.JarFilterTask
 import net.corda.gradle.jarfilter.MetaFixerTask
 import proguard.gradle.ProGuardTask
+
 import static org.gradle.api.JavaVersion.VERSION_1_8
 
 plugins {
@@ -77,6 +78,7 @@ def patchCore = tasks.register('patchCore', Zip) {
         exclude 'net/corda/core/serialization/internal/AttachmentsHolderImpl.class'
         exclude 'net/corda/core/serialization/internal/CheckpointSerializationFactory*.class'
         exclude 'net/corda/core/internal/rules/*.class'
+        exclude 'net/corda/core/internal/utilities/PrivateInterner*.class'
     }
 
     reproducibleFileOrder = true

client/rpc/src/smoke-test/kotlin/net/corda/kotlin/rpc/StandaloneCordaRPClientTest.kt

@@ -0,0 +1,16 @@
+package net.corda.core.internal.utilities
+
+import net.corda.core.KeepForDJVM
+
+@KeepForDJVM
+class PrivateInterner<T>(val verifier: IternabilityVerifier<T> = AlwaysInternableVerifier()) {
+    // DJVM implementation does not intern and does not use Guava
+    fun <S : T> intern(sample: S): S = sample
+
+    @KeepForDJVM
+    companion object {
+        @Suppress("UNUSED_PARAMETER")
+        fun findFor(clazz: Class<*>?): PrivateInterner<Any>? = null
+    }
+}
+

common/logging/src/main/kotlin/net/corda/common/logging/Constants.kt

@@ -58,7 +58,7 @@ dependencies {
     testCompile "org.assertj:assertj-core:${assertj_version}"
 
     // Guava: Google utilities library.
-    testCompile "com.google.guava:guava:$guava_version"
+    compile "com.google.guava:guava:$guava_version"
 
     // For caches rather than guava
     compile "com.github.ben-manes.caffeine:caffeine:$caffeine_version"

core-deterministic/build.gradle

@@ -1,12 +1,15 @@
 package net.corda.core.contracts
 
+import net.corda.core.CordaInternal
 import net.corda.core.DoNotImplement
 import net.corda.core.KeepForDJVM
 import net.corda.core.contracts.AlwaysAcceptAttachmentConstraint.isSatisfiedBy
 import net.corda.core.crypto.SecureHash
 import net.corda.core.crypto.isFulfilledBy
 import net.corda.core.internal.AttachmentWithContext
 import net.corda.core.internal.isUploaderTrusted
+import net.corda.core.internal.utilities.Internable
+import net.corda.core.internal.utilities.PrivateInterner
 import net.corda.core.serialization.CordaSerializable
 import net.corda.core.transactions.TransactionBuilder
 import net.corda.core.utilities.loggerFor
@@ -119,7 +122,16 @@ data class SignatureAttachmentConstraint(val key: PublicKey) : AttachmentConstra
         return if (!key.isFulfilledBy(attachment.signerKeys.map { it })) {
             log.warn("Untrusted signing key: expected $key. but contract attachment contains ${attachment.signerKeys}")
             false
-        }
-        else true
+        } else true
+    }
+
+    companion object : Internable<SignatureAttachmentConstraint> {
+        @CordaInternal
+        override val interner = PrivateInterner<SignatureAttachmentConstraint>()
+
+        /**
+         * Factory method to be used in preference to the constructor.
+         */
+        fun create(key: PublicKey) = interner.intern(SignatureAttachmentConstraint(key))
     }
 }

core-deterministic/src/main/kotlin/net/corda/core/internal/utilities/PrivateInterner.kt

@@ -3,7 +3,13 @@ package net.corda.core.crypto
 import net.corda.core.KeepForDJVM
 import net.corda.core.serialization.deserialize
 import java.io.ByteArrayOutputStream
-import java.security.*
+import java.security.InvalidAlgorithmParameterException
+import java.security.InvalidKeyException
+import java.security.PrivateKey
+import java.security.Provider
+import java.security.PublicKey
+import java.security.Signature
+import java.security.SignatureException
 import java.security.spec.AlgorithmParameterSpec
 
 /**
@@ -80,7 +86,7 @@ class CompositeSignature : Signature(SIGNATURE_ALGORITHM) {
         fun engineVerify(sigBytes: ByteArray): Boolean {
             val sig = sigBytes.deserialize<CompositeSignaturesWithKeys>()
             return if (verifyKey.isFulfilledBy(sig.sigs.map { it.by })) {
-                val clearData = SecureHash.SHA256(buffer.toByteArray())
+                val clearData = SecureHash.createSHA256(buffer.toByteArray())
                 sig.sigs.all { it.isValid(clearData) }
             } else {
                 false

core/build.gradle

@@ -5,11 +5,12 @@ import net.corda.core.DeleteForDJVM
 import net.corda.core.KeepForDJVM
 import net.corda.core.crypto.internal.AliasPrivateKey
 import net.corda.core.crypto.internal.Instances.withSignature
-import net.corda.core.crypto.internal.`id-Curve25519ph`
 import net.corda.core.crypto.internal.bouncyCastlePQCProvider
 import net.corda.core.crypto.internal.cordaBouncyCastleProvider
 import net.corda.core.crypto.internal.cordaSecurityProvider
+import net.corda.core.crypto.internal.`id-Curve25519ph`
 import net.corda.core.crypto.internal.providerMap
+import net.corda.core.internal.utilities.PrivateInterner
 import net.corda.core.serialization.serialize
 import net.i2p.crypto.eddsa.EdDSAEngine
 import net.i2p.crypto.eddsa.EdDSAPrivateKey
@@ -710,7 +711,8 @@ object Crypto {
             keyPairGenerator.initialize(signatureScheme.algSpec, newSecureRandom())
         else
             keyPairGenerator.initialize(signatureScheme.keySize!!, newSecureRandom())
-        return keyPairGenerator.generateKeyPair()
+        val newKeyPair = keyPairGenerator.generateKeyPair()
+        return KeyPair(internPublicKey(newKeyPair.public), newKeyPair.private)
     }
 
     /**
@@ -840,7 +842,7 @@ object Crypto {
         val publicKeySpec = ECPublicKeySpec(pointQ, parameterSpec)
         val publicKeyD = BCECPublicKey(privateKey.algorithm, publicKeySpec, BouncyCastleProvider.CONFIGURATION)
 
-        return KeyPair(publicKeyD, privateKeyD)
+        return KeyPair(internPublicKey(publicKeyD), privateKeyD)
     }
 
     // Deterministically generate an EdDSA key.
@@ -853,7 +855,7 @@ object Crypto {
         val bytes = macBytes.copyOf(params.curve.field.getb() / 8) // Need to pad the entropy to the valid seed length.
         val privateKeyD = EdDSAPrivateKeySpec(bytes, params)
         val publicKeyD = EdDSAPublicKeySpec(privateKeyD.a, params)
-        return KeyPair(EdDSAPublicKey(publicKeyD), EdDSAPrivateKey(privateKeyD))
+        return KeyPair(internPublicKey(EdDSAPublicKey(publicKeyD)), EdDSAPrivateKey(privateKeyD))
     }
 
     /**
@@ -892,7 +894,7 @@ object Crypto {
         val bytes = entropy.toByteArray().copyOf(params.curve.field.getb() / 8) // Need to pad the entropy to the valid seed length.
         val priv = EdDSAPrivateKeySpec(bytes, params)
         val pub = EdDSAPublicKeySpec(priv.a, params)
-        return KeyPair(EdDSAPublicKey(pub), EdDSAPrivateKey(priv))
+        return KeyPair(internPublicKey(EdDSAPublicKey(pub)), EdDSAPrivateKey(priv))
     }
 
     // Custom key pair generator from an entropy required for various tests. It is similar to deriveKeyPairECDSA,
@@ -918,7 +920,7 @@ object Crypto {
         val publicKeySpec = ECPublicKeySpec(pointQ, parameterSpec)
         val pub = BCECPublicKey("EC", publicKeySpec, BouncyCastleProvider.CONFIGURATION)
 
-        return KeyPair(pub, priv)
+        return KeyPair(internPublicKey(pub), priv)
     }
 
     // Compute the HMAC-SHA512 using a privateKey as the MAC_key and a seed ByteArray.
@@ -990,11 +992,14 @@ object Crypto {
         }
     }
 
+    private val interner = PrivateInterner<PublicKey>()
+    private fun internPublicKey(key: PublicKey): PublicKey = interner.intern(key)
+
     private fun convertIfBCEdDSAPublicKey(key: PublicKey): PublicKey {
-        return when (key) {
+        return internPublicKey(when (key) {
             is BCEdDSAPublicKey -> EdDSAPublicKey(X509EncodedKeySpec(key.encoded))
             else -> key
-        }
+        })
     }
 
     private fun convertIfBCEdDSAPrivateKey(key: PrivateKey): PrivateKey {
@@ -1026,11 +1031,11 @@ object Crypto {
     @JvmStatic
     fun toSupportedPublicKey(key: PublicKey): PublicKey {
         return when (key) {
-            is BCECPublicKey -> key
-            is BCRSAPublicKey -> key
-            is BCSphincs256PublicKey -> key
-            is EdDSAPublicKey -> key
-            is CompositeKey -> key
+            is BCECPublicKey -> internPublicKey(key)
+            is BCRSAPublicKey -> internPublicKey(key)
+            is BCSphincs256PublicKey -> internPublicKey(key)
+            is EdDSAPublicKey -> internPublicKey(key)
+            is CompositeKey -> internPublicKey(key)
             is BCEdDSAPublicKey -> convertIfBCEdDSAPublicKey(key)
             else -> decodePublicKey(key.encoded)
         }

core/src/main/kotlin/net/corda/core/contracts/AttachmentConstraint.kt

@@ -3,9 +3,12 @@
 package net.corda.core.crypto
 
 import io.netty.util.concurrent.FastThreadLocal
+import net.corda.core.CordaInternal
 import net.corda.core.DeleteForDJVM
 import net.corda.core.KeepForDJVM
 import net.corda.core.crypto.internal.DigestAlgorithmFactory
+import net.corda.core.internal.utilities.Internable
+import net.corda.core.internal.utilities.PrivateInterner
 import net.corda.core.serialization.CordaSerializable
 import net.corda.core.utilities.OpaqueBytes
 import net.corda.core.utilities.parseAsHex
@@ -66,7 +69,7 @@ sealed class SecureHash(bytes: ByteArray) : OpaqueBytes(bytes) {
         }
 
         override fun generate(data: ByteArray): SecureHash {
-            return HASH(algorithm, digestAs(algorithm, data))
+            return interner.intern(HASH(algorithm, digestAs(algorithm, data)))
         }
     }
 
@@ -110,7 +113,7 @@ sealed class SecureHash(bytes: ByteArray) : OpaqueBytes(bytes) {
         return if(concatAlgorithm == SHA2_256) {
             concatBytes.sha256()
         } else {
-            HASH(concatAlgorithm, digestAs(concatAlgorithm, concatBytes))
+            interner.intern(HASH(concatAlgorithm, digestAs(concatAlgorithm, concatBytes)))
         }
     }
 
@@ -121,12 +124,15 @@ sealed class SecureHash(bytes: ByteArray) : OpaqueBytes(bytes) {
     fun reHash() : SecureHash = hashAs(algorithm, bytes)
 
     // Like static methods in Java, except the 'companion' is a singleton that can have state.
-    companion object {
+    companion object : Internable<SecureHash> {
         const val SHA2_256 = "SHA-256"
         const val SHA2_384 = "SHA-384"
         const val SHA2_512 = "SHA-512"
         const val DELIMITER = ':'
 
+        @CordaInternal
+        override val interner = PrivateInterner<SecureHash>()
+
         /**
          * Converts a SecureHash hash value represented as a {algorithm:}hexadecimal [String] into a [SecureHash].
          * @param str An optional algorithm id followed by a delimiter and the sequence of hexadecimal digits that represents a hash value.
@@ -157,7 +163,7 @@ sealed class SecureHash(bytes: ByteArray) : OpaqueBytes(bytes) {
             val digestLength = digestFor(algorithm).digestLength
             val data = value.parseAsHex()
             return when (data.size) {
-                digestLength -> HASH(algorithm, data)
+                digestLength -> interner.intern(HASH(algorithm, data))
                 else -> throw IllegalArgumentException("Provided string is ${data.size} bytes not $digestLength bytes in hex: $value")
             }
         }
@@ -171,12 +177,18 @@ sealed class SecureHash(bytes: ByteArray) : OpaqueBytes(bytes) {
         fun parse(str: String?): SHA256 {
             return str?.toUpperCase()?.parseAsHex()?.let {
                 when (it.size) {
-                    32 -> SHA256(it)
+                    32 -> interner.intern(SHA256(it))
                     else -> throw IllegalArgumentException("Provided string is ${it.size} bytes not 32 bytes in hex: $str")
                 }
             } ?: throw IllegalArgumentException("Provided string is null")
         }
 
+        /**
+         * Factory method for SHA256 to be used in preference to the constructor.
+         */
+        @JvmStatic
+        fun createSHA256(bytes: ByteArray): SHA256 = interner.intern(SHA256(bytes))
+
         private val messageDigests: ConcurrentMap<String, DigestSupplier> = ConcurrentHashMap()
 
         private fun digestFor(algorithm: String): DigestSupplier {
@@ -202,9 +214,9 @@ sealed class SecureHash(bytes: ByteArray) : OpaqueBytes(bytes) {
         fun hashAs(algorithm: String, bytes: ByteArray): SecureHash {
             val hashBytes = digestAs(algorithm, bytes)
             return if (algorithm == SHA2_256) {
-                SHA256(hashBytes)
+                interner.intern(SHA256(hashBytes))
             } else {
-                HASH(algorithm, hashBytes)
+                interner.intern(HASH(algorithm, hashBytes))
             }
         }
 
@@ -222,7 +234,7 @@ sealed class SecureHash(bytes: ByteArray) : OpaqueBytes(bytes) {
             } else {
                 val digest = digestFor(algorithm).get()
                 val hash = digest.componentDigest(bytes)
-                HASH(algorithm, hash)
+                interner.intern(HASH(algorithm, hash))
             }
         }
 
@@ -240,7 +252,7 @@ sealed class SecureHash(bytes: ByteArray) : OpaqueBytes(bytes) {
             } else {
                 val digest = digestFor(algorithm).get()
                 val hash = digest.nonceDigest(bytes)
-                HASH(algorithm, hash)
+                interner.intern(HASH(algorithm, hash))
             }
         }
 
@@ -249,7 +261,7 @@ sealed class SecureHash(bytes: ByteArray) : OpaqueBytes(bytes) {
          * @param bytes The [ByteArray] to hash.
          */
         @JvmStatic
-        fun sha256(bytes: ByteArray) = SHA256(digestAs(SHA2_256, bytes))
+        fun sha256(bytes: ByteArray) = interner.intern(SHA256(digestAs(SHA2_256, bytes)))
 
         /**
          * Computes the SHA-256 hash of the [ByteArray], and then computes the SHA-256 hash of the hash.
@@ -282,7 +294,7 @@ sealed class SecureHash(bytes: ByteArray) : OpaqueBytes(bytes) {
                 randomSHA256()
             } else {
                 val digest = digestFor(algorithm)
-                HASH(algorithm, digest.get().digest(secureRandomBytes(digest.digestLength)))
+                interner.intern(HASH(algorithm, digest.get().digest(secureRandomBytes(digest.digestLength))))
             }
         }
 
@@ -291,7 +303,7 @@ sealed class SecureHash(bytes: ByteArray) : OpaqueBytes(bytes) {
          * This field provides more intuitive access from Java.
          */
         @JvmField
-        val zeroHash: SHA256 = SHA256(ByteArray(32) { 0.toByte() })
+        val zeroHash: SHA256 = interner.intern(SHA256(ByteArray(32) { 0.toByte() }))
 
         /**
          * A SHA-256 hash value consisting of 32 0x00 bytes.
@@ -305,7 +317,7 @@ sealed class SecureHash(bytes: ByteArray) : OpaqueBytes(bytes) {
          * This field provides more intuitive access from Java.
          */
         @JvmField
-        val allOnesHash: SHA256 = SHA256(ByteArray(32) { 255.toByte() })
+        val allOnesHash: SHA256 = interner.intern(SHA256(ByteArray(32) { 255.toByte() }))
 
         /**
          * A SHA-256 hash value consisting of 32 0xFF bytes.
@@ -323,8 +335,8 @@ sealed class SecureHash(bytes: ByteArray) : OpaqueBytes(bytes) {
             return hashConstants.getOrPut(algorithm) {
                 val digestLength = digestFor(algorithm).digestLength
                 HashConstants(
-                        zero = HASH(algorithm, ByteArray(digestLength)),
-                        allOnes = HASH(algorithm, ByteArray(digestLength) { 255.toByte() })
+                        zero = interner.intern(HASH(algorithm, ByteArray(digestLength))),
+                        allOnes = interner.intern(HASH(algorithm, ByteArray(digestLength) { 255.toByte() }))
                 )
             }
         }