ENT-11255: Scan attachments to determine if they are Kotlin 1.2 or later

The node now sends a transaction to the verifier if any of its attachments were compiled with Kotlin 1.2 (the net.corda.node.verification.external system property has been removed). It uses kotlinx-metadata to read the Kotlin metadata in the attachment to determine this. For now this scanning is done each time the attachment is loaded from the database.

The existing external verification integration tests were converted into smoke tests so that 4.11 nodes could be involved. This required various improvements to NodeProcess.Factory. A new JAVA_8_HOME environment variable, pointing to JDK 8, is required to run these tests.

There is still some follow-up work that needs to be done:

Sending transactions from a 4.11 node to a 4.12 node works, but not the other way round. A new WireTransaction component group needs to be introduced for storing 4.12 attachments so that they can be safely ignored by 4.11 nodes, and the 4.12 node needs to be able to load both 4.11 and 4.12 versions of the same contracts CorDapp so that they can be both attached to the transaction.
Even though attachments are cached when retrieved from the database, the Kotlin metadata version should be stored in the attachments db table, rather than being scanned each time.
Finally, VerificationService was refactored into NodeVerificationSupport and can be passed into SignedTransaction.verifyInternal, instead of needing the much heavier VerifyingServiceHub. This makes it easier for internal tools to verify transactions and spawn the verifier if necessary.

Author: shamsasari

.ci/dev/nightly-regression/Jenkinsfile

@@ -46,6 +46,7 @@ pipeline {
         CORDA_ARTIFACTORY_USERNAME = "${env.ARTIFACTORY_CREDENTIALS_USR}"
         CORDA_USE_CACHE = "corda-remotes"
         JAVA_HOME = "/usr/lib/jvm/java-17-amazon-corretto"
+        JAVA_8_HOME = "/usr/lib/jvm/java-1.8.0-amazon-corretto"
     }
 
     stages {

.ci/dev/regression/Jenkinsfile

@@ -71,6 +71,7 @@ pipeline {
         SNYK_TOKEN = credentials('c4-os-snyk-api-token-secret') //Jenkins credential type: Secret text
         C4_OS_SNYK_ORG_ID = credentials('corda4-os-snyk-org-id')
         JAVA_HOME = "/usr/lib/jvm/java-17-amazon-corretto"
+        JAVA_8_HOME = "/usr/lib/jvm/java-1.8.0-amazon-corretto"
     }
 
     stages {

Jenkinsfile

@@ -54,6 +54,7 @@ pipeline {
         CORDA_GRADLE_SCAN_KEY = credentials('gradle-build-scans-key')
         CORDA_USE_CACHE = "corda-remotes"
         JAVA_HOME="/usr/lib/jvm/java-17-amazon-corretto"
+        JAVA_8_HOME = "/usr/lib/jvm/java-1.8.0-amazon-corretto"
     }
 
     stages {

client/rpc/src/smoke-test/java/net/corda/java/rpc/StandaloneCordaRPCJavaClientTest.java

@@ -1,6 +1,5 @@
 package net.corda.java.rpc;
 
-import net.corda.client.rpc.CordaRPCConnection;
 import net.corda.core.contracts.Amount;
 import net.corda.core.identity.CordaX500Name;
 import net.corda.core.identity.Party;
@@ -10,88 +9,48 @@
 import net.corda.finance.flows.AbstractCashFlow;
 import net.corda.finance.flows.CashIssueFlow;
 import net.corda.nodeapi.internal.config.User;
-import net.corda.smoketesting.NodeConfig;
+import net.corda.smoketesting.NodeParams;
 import net.corda.smoketesting.NodeProcess;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.*;
+import java.util.Currency;
+import java.util.HashSet;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.atomic.AtomicInteger;
-import java.util.stream.Stream;
 
 import static java.util.Collections.singletonList;
 import static kotlin.test.AssertionsKt.assertEquals;
-import static kotlin.test.AssertionsKt.fail;
 import static net.corda.finance.workflows.GetBalances.getCashBalance;
+import static net.corda.kotlin.rpc.StandaloneCordaRPClientTest.gatherCordapps;
 
 public class StandaloneCordaRPCJavaClientTest {
+    private final User superUser = new User("superUser", "test", new HashSet<>(singletonList("ALL")));
 
-    public static void copyCordapps(NodeProcess.Factory factory, NodeConfig notaryConfig) {
-        Path cordappsDir = (factory.baseDirectory(notaryConfig).resolve(NodeProcess.CORDAPPS_DIR_NAME));
-        try {
-            Files.createDirectories(cordappsDir);
-        } catch (IOException ex) {
-            fail("Failed to create directories");
-        }
-        try (Stream<Path> paths = Files.walk(Paths.get("build", "resources", "smokeTest"))) {
-            paths.filter(path -> path.toFile().getName().startsWith("cordapp")).forEach(file -> {
-                try {
-                    Files.copy(file, cordappsDir.resolve(file.getFileName()));
-                } catch (IOException ex) {
-                    fail("Failed to copy cordapp jar");
-                }
-            });
-        } catch (IOException e) {
-            fail("Failed to walk files");
-        }
-    }
-
-    private List<String> perms = singletonList("ALL");
-    private Set<String> permSet = new HashSet<>(perms);
-    private User superUser = new User("superUser", "test", permSet);
+    private final AtomicInteger port = new AtomicInteger(15000);
+    private final NodeProcess.Factory factory = new NodeProcess.Factory();
 
-    private AtomicInteger port = new AtomicInteger(15000);
-
-    private NodeProcess notary;
     private CordaRPCOps rpcProxy;
-    private CordaRPCConnection connection;
     private Party notaryNodeIdentity;
 
-    private NodeConfig notaryConfig = new NodeConfig(
-            new CordaX500Name("Notary Service", "Zurich", "CH"),
-            port.getAndIncrement(),
-            port.getAndIncrement(),
-            port.getAndIncrement(),
-            true,
-            singletonList(superUser),
-            true
-    );
-
     @Before
     public void setUp() {
-        NodeProcess.Factory factory = new NodeProcess.Factory();
-        copyCordapps(factory, notaryConfig);
-        notary = factory.create(notaryConfig);
-        connection = notary.connect(superUser);
-        rpcProxy = connection.getProxy();
+        NodeProcess notary = factory.createNotaries(new NodeParams(
+                new CordaX500Name("Notary Service", "Zurich", "CH"),
+                port.getAndIncrement(),
+                port.getAndIncrement(),
+                port.getAndIncrement(),
+                singletonList(superUser),
+                gatherCordapps()
+        )).get(0);
+        rpcProxy = notary.connect(superUser).getProxy();
         notaryNodeIdentity = rpcProxy.nodeInfo().getLegalIdentities().get(0);
     }
 
     @After
     public void done() {
-        try {
-            connection.close();
-        } finally {
-            if (notary != null) {
-                notary.close();
-            }
-        }
+        factory.close();
     }
 
     @Test

client/rpc/src/smoke-test/kotlin/net/corda/kotlin/rpc/StandaloneCordaRPClientTest.kt

@@ -27,6 +27,7 @@ import net.corda.core.utilities.getOrThrow
 import net.corda.core.utilities.minutes
 import net.corda.core.utilities.seconds
 import net.corda.finance.DOLLARS
+import net.corda.finance.GBP
 import net.corda.finance.POUNDS
 import net.corda.finance.SWISS_FRANCS
 import net.corda.finance.USD
@@ -35,32 +36,35 @@ import net.corda.finance.flows.CashIssueFlow
 import net.corda.finance.flows.CashPaymentFlow
 import net.corda.finance.workflows.getCashBalance
 import net.corda.finance.workflows.getCashBalances
-import net.corda.java.rpc.StandaloneCordaRPCJavaClientTest
 import net.corda.nodeapi.internal.config.User
 import net.corda.sleeping.SleepingFlow
-import net.corda.smoketesting.NodeConfig
+import net.corda.smoketesting.NodeParams
 import net.corda.smoketesting.NodeProcess
 import org.hamcrest.text.MatchesPattern
 import org.junit.After
+import org.junit.AfterClass
 import org.junit.Before
+import org.junit.BeforeClass
 import org.junit.Ignore
 import org.junit.Rule
 import org.junit.Test
 import org.junit.rules.ExpectedException
 import java.io.FilterInputStream
 import java.io.InputStream
 import java.io.OutputStream.nullOutputStream
-import java.util.Currency
+import java.nio.file.Path
 import java.util.concurrent.CountDownLatch
 import java.util.concurrent.atomic.AtomicInteger
 import java.util.regex.Pattern
+import kotlin.io.path.Path
+import kotlin.io.path.listDirectoryEntries
 import kotlin.test.assertEquals
 import kotlin.test.assertFalse
 import kotlin.test.assertNotEquals
 import kotlin.test.assertTrue
 
 class StandaloneCordaRPClientTest {
-    private companion object {
+    companion object {
         private val log = contextLogger()
         val superUser = User("superUser", "test", permissions = setOf("ALL"))
         val nonUser = User("nonUser", "test", permissions = emptySet())
@@ -69,46 +73,57 @@ class StandaloneCordaRPClientTest {
         val port = AtomicInteger(15200)
         const val ATTACHMENT_SIZE = 2116
         val timeout = 60.seconds
+
+        private val factory = NodeProcess.Factory()
+
+        private lateinit var notary: NodeProcess
+
+        private val notaryConfig = NodeParams(
+                legalName = CordaX500Name(organisation = "Notary Service", locality = "Zurich", country = "CH"),
+                p2pPort = port.andIncrement,
+                rpcPort = port.andIncrement,
+                rpcAdminPort = port.andIncrement,
+                users = listOf(superUser, nonUser, rpcUser, flowUser),
+                cordappJars = gatherCordapps()
+        )
+
+        @BeforeClass
+        @JvmStatic
+        fun startNotary() {
+            notary = factory.createNotaries(notaryConfig)[0]
+        }
+
+        @AfterClass
+        @JvmStatic
+        fun close() {
+            factory.close()
+        }
+
+        @JvmStatic
+        fun gatherCordapps(): List<Path> {
+            return Path("build", "resources", "smokeTest").listDirectoryEntries("cordapp*.jar")
+        }
     }
 
-    private lateinit var factory: NodeProcess.Factory
-    private lateinit var notary: NodeProcess
-    private lateinit var rpcProxy: CordaRPCOps
     private lateinit var connection: CordaRPCConnection
-    private lateinit var notaryNode: NodeInfo
+    private lateinit var rpcProxy: CordaRPCOps
     private lateinit var notaryNodeIdentity: Party
 
-    private val notaryConfig = NodeConfig(
-            legalName = CordaX500Name(organisation = "Notary Service", locality = "Zurich", country = "CH"),
-            p2pPort = port.andIncrement,
-            rpcPort = port.andIncrement,
-            rpcAdminPort = port.andIncrement,
-            isNotary = true,
-            users = listOf(superUser, nonUser, rpcUser, flowUser)
-    )
-
     @get:Rule
     val exception: ExpectedException = ExpectedException.none()
 
     @Before
     fun setUp() {
-        factory = NodeProcess.Factory()
-        StandaloneCordaRPCJavaClientTest.copyCordapps(factory, notaryConfig)
-        notary = factory.create(notaryConfig)
         connection = notary.connect(superUser)
         rpcProxy = connection.proxy
-        notaryNode = fetchNotaryIdentity()
         notaryNodeIdentity = rpcProxy.nodeInfo().legalIdentitiesAndCerts.first().party
     }
 
     @After
-    fun done() {
-        connection.use {
-            notary.close()
-        }
+    fun closeConnection() {
+        connection.close()
     }
 
-
     @Test(timeout=300_000)
 	fun `test attachments`() {
         val attachment = InputStreamAndHash.createInMemoryTestZip(ATTACHMENT_SIZE, 1)
@@ -168,8 +183,7 @@ class StandaloneCordaRPClientTest {
 
     @Test(timeout=300_000)
 	fun `test state machines`() {
-        val (stateMachines, updates) = rpcProxy.stateMachinesFeed()
-        assertEquals(0, stateMachines.size)
+        val (_, updates) = rpcProxy.stateMachinesFeed()
 
         val updateLatch = CountDownLatch(1)
         val updateCount = AtomicInteger(0)
@@ -190,8 +204,9 @@ class StandaloneCordaRPClientTest {
 
     @Test(timeout=300_000)
 	fun `test vault track by`() {
-        val (vault, vaultUpdates) = rpcProxy.vaultTrackBy<Cash.State>(paging = PageSpecification(DEFAULT_PAGE_NUM))
-        assertEquals(0, vault.totalStatesAvailable)
+        val initialGbpBalance = rpcProxy.getCashBalance(GBP)
+
+        val (_, vaultUpdates) = rpcProxy.vaultTrackBy<Cash.State>(paging = PageSpecification(DEFAULT_PAGE_NUM))
 
         val updateLatch = CountDownLatch(1)
         vaultUpdates.subscribe { update ->
@@ -207,34 +222,35 @@ class StandaloneCordaRPClientTest {
         // Check that this cash exists in the vault
         val cashBalance = rpcProxy.getCashBalances()
         log.info("Cash Balances: $cashBalance")
-        assertEquals(1, cashBalance.size)
-        assertEquals(629.POUNDS, cashBalance[Currency.getInstance("GBP")])
+        assertEquals(629.POUNDS, cashBalance[GBP]!! - initialGbpBalance)
     }
 
     @Test(timeout=300_000)
 	fun `test vault query by`() {
-        // Now issue some cash
-        rpcProxy.startFlow(::CashIssueFlow, 629.POUNDS, OpaqueBytes.of(0), notaryNodeIdentity)
-                .returnValue.getOrThrow(timeout)
-
         val criteria = QueryCriteria.VaultQueryCriteria(status = Vault.StateStatus.ALL)
         val paging = PageSpecification(DEFAULT_PAGE_NUM, 10)
         val sorting = Sort(setOf(Sort.SortColumn(SortAttribute.Standard(Sort.VaultStateAttribute.RECORDED_TIME), Sort.Direction.DESC)))
 
+        val initialStateCount = rpcProxy.vaultQueryBy<Cash.State>(criteria, paging, sorting).totalStatesAvailable
+        val initialGbpBalance = rpcProxy.getCashBalance(GBP)
+
+        // Now issue some cash
+        rpcProxy.startFlow(::CashIssueFlow, 629.POUNDS, OpaqueBytes.of(0), notaryNodeIdentity)
+                .returnValue.getOrThrow(timeout)
+
         val queryResults = rpcProxy.vaultQueryBy<Cash.State>(criteria, paging, sorting)
-        assertEquals(1, queryResults.totalStatesAvailable)
+        assertEquals(1, queryResults.totalStatesAvailable - initialStateCount)
         assertEquals(queryResults.states.first().state.data.amount.quantity, 629.POUNDS.quantity)
 
         rpcProxy.startFlow(::CashPaymentFlow, 100.POUNDS, notaryNodeIdentity, true, notaryNodeIdentity).returnValue.getOrThrow()
 
         val moreResults = rpcProxy.vaultQueryBy<Cash.State>(criteria, paging, sorting)
-        assertEquals(3, moreResults.totalStatesAvailable)   // 629 - 100 + 100
+        assertEquals(3, moreResults.totalStatesAvailable - initialStateCount)   // 629 - 100 + 100
 
         // Check that this cash exists in the vault
         val cashBalances = rpcProxy.getCashBalances()
         log.info("Cash Balances: $cashBalances")
-        assertEquals(1, cashBalances.size)
-        assertEquals(629.POUNDS, cashBalances[Currency.getInstance("GBP")])
+        assertEquals(629.POUNDS, cashBalances[GBP]!! - initialGbpBalance)
     }
 
     @Test(timeout=300_000)