feat: recursive verifier for decider and last folding proof (#9626)

Construct the _hiding_ circuit, which recursively verifies the last
folding proof and the decider proof in Client IVC and amend the e2e
prover accordingly. The ClientIVC proof becomes a mega proof for the
hiding circuit and a goblin proof which simplifies the work required to
transform this in a zero knowledge proof.

Author: maramihali

barretenberg/cpp/src/barretenberg/bb/main.cpp

@@ -380,8 +380,7 @@ void client_ivc_prove_output_all_msgpack(const std::string& bytecodePath,
 
     // Write the proof and verification keys into the working directory in  'binary' format (in practice it seems this
     // directory is passed by bb.js)
-    std::string vkPath = outputDir + "/final_decider_vk"; // the vk of the last circuit in the stack
-    std::string accPath = outputDir + "/pg_acc";
+    std::string vkPath = outputDir + "/mega_vk"; // the vk of the last circuit in the stack
     std::string proofPath = outputDir + "/client_ivc_proof";
     std::string translatorVkPath = outputDir + "/translator_vk";
     std::string eccVkPath = outputDir + "/ecc_vk";
@@ -391,12 +390,11 @@ void client_ivc_prove_output_all_msgpack(const std::string& bytecodePath,
     auto translator_vk = std::make_shared<TranslatorVK>(ivc.goblin.get_translator_proving_key());
 
     auto last_vk = std::make_shared<DeciderVerificationKey>(ivc.honk_vk);
-    vinfo("ensure valid proof: ", ivc.verify(proof, { ivc.verifier_accumulator, last_vk }));
+    vinfo("ensure valid proof: ", ivc.verify(proof));
 
     vinfo("write proof and vk data to files..");
     write_file(proofPath, to_buffer(proof));
     write_file(vkPath, to_buffer(ivc.honk_vk));
-    write_file(accPath, to_buffer(ivc.verifier_accumulator));
     write_file(translatorVkPath, to_buffer(translator_vk));
     write_file(eccVkPath, to_buffer(eccvm_vk));
 }
@@ -419,26 +417,23 @@ template <typename T> std::shared_ptr<T> read_to_shared_ptr(const std::filesyste
  * @return true (resp., false) if the proof is valid (resp., invalid).
  */
 bool verify_client_ivc(const std::filesystem::path& proof_path,
-                       const std::filesystem::path& accumulator_path,
-                       const std::filesystem::path& final_vk_path,
+                       const std::filesystem::path& mega_vk,
                        const std::filesystem::path& eccvm_vk_path,
                        const std::filesystem::path& translator_vk_path)
 {
     init_bn254_crs(1);
     init_grumpkin_crs(1 << 15);
 
     const auto proof = from_buffer<ClientIVC::Proof>(read_file(proof_path));
-    const auto accumulator = read_to_shared_ptr<ClientIVC::DeciderVerificationKey>(accumulator_path);
-    accumulator->verification_key->pcs_verification_key = std::make_shared<VerifierCommitmentKey<curve::BN254>>();
-    const auto final_vk = read_to_shared_ptr<ClientIVC::VerificationKey>(final_vk_path);
+    const auto final_vk = read_to_shared_ptr<ClientIVC::VerificationKey>(mega_vk);
+    final_vk->pcs_verification_key = std::make_shared<VerifierCommitmentKey<curve::BN254>>();
+
     const auto eccvm_vk = read_to_shared_ptr<ECCVMFlavor::VerificationKey>(eccvm_vk_path);
     eccvm_vk->pcs_verification_key =
         std::make_shared<VerifierCommitmentKey<curve::Grumpkin>>(eccvm_vk->circuit_size + 1);
     const auto translator_vk = read_to_shared_ptr<TranslatorFlavor::VerificationKey>(translator_vk_path);
     translator_vk->pcs_verification_key = std::make_shared<VerifierCommitmentKey<curve::BN254>>();
-
-    const bool verified = ClientIVC::verify(
-        proof, accumulator, std::make_shared<ClientIVC::DeciderVerificationKey>(final_vk), eccvm_vk, translator_vk);
+    const bool verified = ClientIVC::verify(proof, final_vk, eccvm_vk, translator_vk);
     vinfo("verified: ", verified);
     return verified;
 }
@@ -499,7 +494,6 @@ void client_ivc_prove_output_all(const std::string& bytecodePath,
     using Builder = Flavor::CircuitBuilder;
     using ECCVMVK = ECCVMFlavor::VerificationKey;
     using TranslatorVK = TranslatorFlavor::VerificationKey;
-    using DeciderVK = ClientIVC::DeciderVerificationKey;
 
     init_bn254_crs(1 << 22);
     init_grumpkin_crs(1 << 16);
@@ -531,23 +525,19 @@ void client_ivc_prove_output_all(const std::string& bytecodePath,
 
     // Write the proof and verification keys into the working directory in  'binary' format (in practice it seems this
     // directory is passed by bb.js)
-    std::string vkPath = outputPath + "/final_decider_vk"; // the vk of the last circuit in the stack
-    std::string accPath = outputPath + "/pg_acc";
+    std::string vkPath = outputPath + "/mega_vk"; // the vk of the last circuit in the stack
     std::string proofPath = outputPath + "/client_ivc_proof";
     std::string translatorVkPath = outputPath + "/translator_vk";
     std::string eccVkPath = outputPath + "/ecc_vk";
 
     auto proof = ivc.prove();
     auto eccvm_vk = std::make_shared<ECCVMVK>(ivc.goblin.get_eccvm_proving_key());
     auto translator_vk = std::make_shared<TranslatorVK>(ivc.goblin.get_translator_proving_key());
-
-    auto last_vk = std::make_shared<DeciderVK>(ivc.honk_vk);
-    vinfo("ensure valid proof: ", ivc.verify(proof, { ivc.verifier_accumulator, last_vk }));
+    vinfo("ensure valid proof: ", ivc.verify(proof));
 
     vinfo("write proof and vk data to files..");
     write_file(proofPath, to_buffer(proof));
     write_file(vkPath, to_buffer(ivc.honk_vk)); // maybe dereference
-    write_file(accPath, to_buffer(ivc.verifier_accumulator));
     write_file(translatorVkPath, to_buffer(translator_vk));
     write_file(eccVkPath, to_buffer(eccvm_vk));
 }
@@ -561,18 +551,15 @@ void client_ivc_prove_output_all(const std::string& bytecodePath,
 void prove_tube(const std::string& output_path)
 {
     using ClientIVC = stdlib::recursion::honk::ClientIVCRecursiveVerifier;
-    using StackDeciderVK = ClientIVC::FoldVerifierInput::DeciderVK;
     using StackHonkVK = typename MegaFlavor::VerificationKey;
     using ECCVMVk = ECCVMFlavor::VerificationKey;
     using TranslatorVk = TranslatorFlavor::VerificationKey;
-    using FoldVerifierInput = ClientIVC::FoldVerifierInput;
     using GoblinVerifierInput = ClientIVC::GoblinVerifierInput;
     using VerifierInput = ClientIVC::VerifierInput;
     using Builder = UltraCircuitBuilder;
     using GrumpkinVk = bb::VerifierCommitmentKey<curve::Grumpkin>;
 
-    std::string vkPath = output_path + "/final_decider_vk"; // the vk of the last circuit in the stack
-    std::string accPath = output_path + "/pg_acc";
+    std::string vkPath = output_path + "/mega_vk"; // the vk of the last circuit in the stack
     std::string proofPath = output_path + "/client_ivc_proof";
     std::string translatorVkPath = output_path + "/translator_vk";
     std::string eccVkPath = output_path + "/ecc_vk";
@@ -583,10 +570,7 @@ void prove_tube(const std::string& output_path)
 
     // Read the proof  and verification data from given files
     auto proof = from_buffer<ClientIVC::Proof>(read_file(proofPath));
-    std::shared_ptr<StackHonkVK> final_stack_vk =
-        std::make_shared<StackHonkVK>(from_buffer<StackHonkVK>(read_file(vkPath)));
-    std::shared_ptr<StackDeciderVK> verifier_accumulator =
-        std::make_shared<StackDeciderVK>(from_buffer<StackDeciderVK>(read_file(accPath)));
+    std::shared_ptr<StackHonkVK> mega_vk = std::make_shared<StackHonkVK>(from_buffer<StackHonkVK>(read_file(vkPath)));
     std::shared_ptr<TranslatorVk> translator_vk =
         std::make_shared<TranslatorVk>(from_buffer<TranslatorVk>(read_file(translatorVkPath)));
     std::shared_ptr<ECCVMVk> eccvm_vk = std::make_shared<ECCVMVk>(from_buffer<ECCVMVk>(read_file(eccVkPath)));
@@ -595,30 +579,30 @@ void prove_tube(const std::string& output_path)
     // TODO(https://github.com/AztecProtocol/barretenberg/issues/1025)
     eccvm_vk->pcs_verification_key = std::make_shared<GrumpkinVk>(eccvm_vk->circuit_size + 1);
 
-    FoldVerifierInput fold_verifier_input{ verifier_accumulator, { final_stack_vk } };
     GoblinVerifierInput goblin_verifier_input{ eccvm_vk, translator_vk };
-    VerifierInput input{ fold_verifier_input, goblin_verifier_input };
+    VerifierInput input{ mega_vk, goblin_verifier_input };
     auto builder = std::make_shared<Builder>();
-    // Padding needed for sending the right number of public inputs
+
+    // Preserve the public inputs that should be passed to the base rollup by making them public inputs to the tube
+    // circuit
     // TODO(https://github.com/AztecProtocol/barretenberg/issues/1048): INSECURE - make this tube proof actually use
-    // these public inputs by turning proof into witnesses and call
-    // set_public on each witness
-    auto num_public_inputs = static_cast<uint32_t>(static_cast<uint256_t>(proof.folding_proof[1]));
-    num_public_inputs -= bb::AGGREGATION_OBJECT_SIZE;             // don't add the agg object
-    num_public_inputs -= bb::PROPAGATED_DATABUS_COMMITMENTS_SIZE; // exclude propagated databus commitments
+    // these public inputs by turning proof into witnesses and calling set_public on each witness
+    auto num_public_inputs = static_cast<uint32_t>(static_cast<uint256_t>(proof.mega_proof[1]));
+    num_public_inputs -= bb::AGGREGATION_OBJECT_SIZE; // don't add the agg object
+
     for (size_t i = 0; i < num_public_inputs; i++) {
-        auto offset = acir_format::HONK_RECURSION_PUBLIC_INPUT_OFFSET;
-        builder->add_public_variable(proof.folding_proof[i + offset]);
+        auto offset = bb::HONK_PROOF_PUBLIC_INPUT_OFFSET;
+        builder->add_public_variable(proof.mega_proof[i + offset]);
     }
     ClientIVC verifier{ builder, input };
 
     verifier.verify(proof);
 
-    // TODO(https://github.com/AztecProtocol/barretenberg/issues/1069): Add aggregation to goblin recursive verifiers.
-    // This is currently just setting the aggregation object to the default one.
     AggregationObjectIndices current_aggregation_object =
         stdlib::recursion::init_default_agg_obj_indices<Builder>(*builder);
 
+    // TODO(https://github.com/AztecProtocol/barretenberg/issues/1069): Add aggregation to goblin recursive verifiers.
+    // This is currently just setting the aggregation object to the default one.
     builder->add_recursive_proof(current_aggregation_object);
 
     using Prover = UltraProver_<UltraFlavor>;
@@ -1477,15 +1461,11 @@ int main(int argc, char* argv[])
         if (command == "verify_client_ivc") {
             std::filesystem::path output_dir = get_option(args, "-o", "./target");
             std::filesystem::path client_ivc_proof_path = output_dir / "client_ivc_proof";
-            std::filesystem::path accumulator_path = output_dir / "pg_acc";
-            std::filesystem::path final_vk_path = output_dir / "final_decider_vk";
+            std::filesystem::path mega_vk_path = output_dir / "mega_vk";
             std::filesystem::path eccvm_vk_path = output_dir / "ecc_vk";
             std::filesystem::path translator_vk_path = output_dir / "translator_vk";
 
-            return verify_client_ivc(
-                       client_ivc_proof_path, accumulator_path, final_vk_path, eccvm_vk_path, translator_vk_path)
-                       ? 0
-                       : 1;
+            return verify_client_ivc(client_ivc_proof_path, mega_vk_path, eccvm_vk_path, translator_vk_path) ? 0 : 1;
         }
         if (command == "fold_and_verify_program") {
             return foldAndVerifyProgram(bytecode_path, witness_path) ? 0 : 1;

barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.cpp

@@ -213,41 +213,102 @@ void ClientIVC::accumulate(ClientCircuit& circuit, const std::shared_ptr<Verific
     }
 }
 
+/**
+ * @brief Construct the hiding circuit, which recursively verifies the last folding proof and decider proof, and
+ * then produce a proof of the circuit's correctness with MegaHonk.
+ *
+ * @details The aim of this intermediate stage is to reduce the cost of producing a zero-knowledge ClientIVCProof.
+ * @return HonkProof - a Mega proof
+ */
+HonkProof ClientIVC::construct_and_prove_hiding_circuit()
+{
+    trace_usage_tracker.print(); // print minimum structured sizes for each block
+    ASSERT(verification_queue.size() == 1);
+    ASSERT(merge_verification_queue.size() == 1); // ensure only a single merge proof remains in the queue
+
+    FoldProof& fold_proof = verification_queue[0].proof;
+    HonkProof decider_proof = decider_prove();
+
+    fold_output.accumulator = nullptr;
+
+    ClientCircuit builder{ goblin.op_queue };
+    // The last circuit being folded is a kernel circuit whose public inputs need to be passed to the base rollup
+    // circuit. So, these have to be preserved as public inputs to the hiding circuit (and, subsequently, as public
+    // inputs to the tube circuit) which are intermediate stages.
+    // TODO(https://github.com/AztecProtocol/barretenberg/issues/1048): link these properly, likely insecure
+    auto num_public_inputs = static_cast<uint32_t>(static_cast<uint256_t>(fold_proof[PUBLIC_INPUTS_SIZE_INDEX]));
+    vinfo("num_public_inputs of the last folding proof BEFORE SUBTRACTION", num_public_inputs);
+    num_public_inputs -= bb::AGGREGATION_OBJECT_SIZE;             // exclude aggregation object
+    num_public_inputs -= bb::PROPAGATED_DATABUS_COMMITMENTS_SIZE; // exclude propagated databus commitments
+    vinfo("num_public_inputs of the last folding proof ", num_public_inputs);
+    for (size_t i = 0; i < num_public_inputs; i++) {
+        size_t offset = HONK_PROOF_PUBLIC_INPUT_OFFSET;
+        builder.add_public_variable(fold_proof[i + offset]);
+    }
+
+    process_recursive_merge_verification_queue(builder);
+
+    // Construct stdlib accumulator, decider vkey and folding proof
+    auto stdlib_verifier_accumulator =
+        std::make_shared<RecursiveDeciderVerificationKey>(&builder, verifier_accumulator);
+
+    auto stdlib_decider_vk =
+        std::make_shared<RecursiveVerificationKey>(&builder, verification_queue[0].honk_verification_key);
+
+    auto stdlib_proof = bb::convert_proof_to_witness(&builder, fold_proof);
+
+    // Perform recursive folding verification of the last folding proof
+    FoldingRecursiveVerifier folding_verifier{ &builder, stdlib_verifier_accumulator, { stdlib_decider_vk } };
+    auto recursive_verifier_accumulator = folding_verifier.verify_folding_proof(stdlib_proof);
+    verification_queue.clear();
+
+    // Perform recursive decider verification
+    DeciderRecursiveVerifier decider{ &builder, recursive_verifier_accumulator };
+    decider.verify_proof(decider_proof);
+
+    builder.add_recursive_proof(stdlib::recursion::init_default_agg_obj_indices<ClientCircuit>(builder));
+
+    // Construct the last merge proof for the present circuit and add to merge verification queue
+    MergeProof merge_proof = goblin.prove_merge(builder);
+    merge_verification_queue.emplace_back(merge_proof);
+
+    auto decider_pk = std::make_shared<DeciderProvingKey>(builder);
+    honk_vk = std::make_shared<VerificationKey>(decider_pk->proving_key);
+    MegaProver prover(decider_pk);
+
+    HonkProof proof = prover.construct_proof();
+
+    return proof;
+}
+
 /**
  * @brief Construct a proof for the IVC, which, if verified, fully establishes its correctness
  *
  * @return Proof
  */
 ClientIVC::Proof ClientIVC::prove()
 {
-    trace_usage_tracker.print();                  // print minimum structured sizes for each block
-    ASSERT(verification_queue.size() == 1);       // ensure only a single fold proof remains in the queue
+    HonkProof mega_proof = construct_and_prove_hiding_circuit();
     ASSERT(merge_verification_queue.size() == 1); // ensure only a single merge proof remains in the queue
-    FoldProof& fold_proof = verification_queue[0].proof;
     MergeProof& merge_proof = merge_verification_queue[0];
-    HonkProof decider_proof = decider_prove();
-    // Free the accumulator to save memory
-    fold_output.accumulator = nullptr;
-    return { fold_proof, std::move(decider_proof), goblin.prove(merge_proof) };
+    return { mega_proof, goblin.prove(merge_proof) };
 };
 
 bool ClientIVC::verify(const Proof& proof,
-                       const std::shared_ptr<DeciderVerificationKey>& accumulator,
-                       const std::shared_ptr<DeciderVerificationKey>& final_stack_vk,
+                       const std::shared_ptr<VerificationKey>& ultra_vk,
                        const std::shared_ptr<ClientIVC::ECCVMVerificationKey>& eccvm_vk,
                        const std::shared_ptr<ClientIVC::TranslatorVerificationKey>& translator_vk)
 {
-    // Goblin verification (merge, eccvm, translator)
+
+    // Verify the hiding circuit proof
+    MegaVerifier verifer{ ultra_vk };
+    bool ultra_verified = verifer.verify_proof(proof.mega_proof);
+    vinfo("Mega verified: ", ultra_verified);
+    // Goblin verification (final merge, eccvm, translator)
     GoblinVerifier goblin_verifier{ eccvm_vk, translator_vk };
     bool goblin_verified = goblin_verifier.verify(proof.goblin_proof);
-
-    // Decider verification
-    ClientIVC::FoldingVerifier folding_verifier({ accumulator, final_stack_vk });
-    auto verifier_accumulator = folding_verifier.verify_folding_proof(proof.folding_proof);
-
-    ClientIVC::DeciderVerifier decider_verifier(verifier_accumulator);
-    bool decision = decider_verifier.verify_proof(proof.decider_proof);
-    return goblin_verified && decision;
+    vinfo("Goblin verified: ", goblin_verified);
+    return goblin_verified && ultra_verified;
 }
 
 /**
@@ -256,11 +317,11 @@ bool ClientIVC::verify(const Proof& proof,
  * @param proof
  * @return bool
  */
-bool ClientIVC::verify(const Proof& proof, const std::vector<std::shared_ptr<DeciderVerificationKey>>& vk_stack)
+bool ClientIVC::verify(const Proof& proof)
 {
     auto eccvm_vk = std::make_shared<ECCVMVerificationKey>(goblin.get_eccvm_proving_key());
     auto translator_vk = std::make_shared<TranslatorVerificationKey>(goblin.get_translator_proving_key());
-    return verify(proof, vk_stack[0], vk_stack[1], eccvm_vk, translator_vk);
+    return verify(proof, honk_vk, eccvm_vk, translator_vk);
 }
 
 /**
@@ -283,9 +344,7 @@ HonkProof ClientIVC::decider_prove() const
 bool ClientIVC::prove_and_verify()
 {
     auto proof = prove();
-
-    auto verifier_inst = std::make_shared<DeciderVerificationKey>(this->verification_queue[0].honk_verification_key);
-    return verify(proof, { this->verifier_accumulator, verifier_inst });
+    return verify(proof);
 }
 
 /**