feat: remove auto verify mode from ClientIVC (#10599)

Removes auto verify mode from `ClientIVC`. Auto verify mode was a
temporary feature designed to append kernel logic (recursive
verifications etc.) to kernel circuits generated from noir kernel
programs before the `verify_proof()` calls were integrated into those
programs. Now that these calls are present in the kernels (mock and
genuine), the kernel logic is automatically constructed via
ivc_recursion_constraints via the normal `create_circuit()` pathway. If
in the future there is need for a backend which automatically appends
recursive verification logic, it should be implemented in an entirely
separate class.

Note: This change means we can no longer generate an IVC proof from the
noir `fold_*` programs since they do not contain explicit recursive
verifiers. All such tests/flows have been removed.

Closes #1116 (remove
manual setting of is_kernel)
Closes #1101 (remove
auto-verify)

Author: ledwards2225

Earthfile

@@ -26,12 +26,8 @@ barretenberg-acir-tests-bb:
     ENV TEST_SRC /usr/src/acir_artifacts
     ENV VERBOSE=1
 
-    # Fold and verify an ACIR program stack using ClientIvc
-    RUN INPUT_TYPE=compiletime_stack FLOW=prove_and_verify_client_ivc ./run_acir_tests.sh fold_basic
-    # Fold and verify an ACIR program stack using ClientIvc, then natively verify the ClientIVC proof.
-    RUN INPUT_TYPE=compiletime_stack FLOW=prove_then_verify_client_ivc ./run_acir_tests.sh fold_basic
     # Fold and verify an ACIR program stack using ClientIvc, recursively verify as part of the Tube circuit and produce and verify a Honk proof
-    RUN FLOW=prove_then_verify_tube ./run_acir_tests.sh fold_basic
+    RUN FLOW=prove_then_verify_tube ./run_acir_tests.sh 6_array
     # Run 1_mul through native bb build, all_cmds flow, to test all cli args.
     RUN FLOW=all_cmds ./run_acir_tests.sh 1_mul
 
@@ -95,7 +91,7 @@ barretenberg-acir-tests-bb-client-ivc:
     # Construct and verify a ClientIVC proof for a single arbitrary program
     RUN FLOW=prove_and_verify_client_ivc ./run_acir_tests.sh 6_array
     # Construct and separately verify a ClientIVC proof for all acir programs
-    RUN FLOW=prove_then_verify_client_ivc CLIENT_IVC_SKIPS=true ./run_acir_tests.sh
+    RUN FLOW=prove_then_verify_client_ivc ./run_acir_tests.sh 6_array databus databus_two_calldata
 
 barretenberg-acir-tests-sol:
     FROM ../build-images/+from-registry
@@ -174,7 +170,5 @@ barretenberg-acir-tests-bb.js:
     RUN BIN=../ts/dest/node/main.js FLOW=prove_then_verify_ultra_honk ./run_acir_tests.sh 6_array assert_statement
     # Run a single arbitrary test not involving recursion through bb.js for MegaHonk
     RUN BIN=../ts/dest/node/main.js FLOW=prove_and_verify_mega_honk ./run_acir_tests.sh 6_array
-    # Run fold_basic test through bb.js which runs ClientIVC on fold basic
-    RUN BIN=../ts/dest/node/main.js FLOW=fold_and_verify_program ./run_acir_tests.sh fold_basic
     # Run 1_mul through bb.js build, all_cmds flow, to test all cli args.
     RUN BIN=../ts/dest/node/main.js FLOW=all_cmds ./run_acir_tests.sh 1_mul

acir_tests/flows/fold_and_verify_program.sh

@@ -5,7 +5,7 @@ mkdir -p ./proofs
 
 VFLAG=${VERBOSE:+-v}
 
-$BIN prove --scheme client_ivc --input_type compiletime_stack $VFLAG -c $CRS_PATH -b ./target/program.json
+$BIN write_arbitrary_valid_proof_and_vk_to_file --scheme client_ivc $VFLAG -c $CRS_PATH
 $BIN prove_tube -k vk -p proof -c $CRS_PATH $VFLAG
 $BIN verify_tube -k vk -p proof -c $CRS_PATH $VFLAG
 

acir_tests/flows/prove_then_verify_tube.sh

@@ -8,7 +8,6 @@ class API {
     struct Flags {
         std::optional<std::string> output_type; // bytes, fields, bytes_and_fields, fields_msgpack
         std::optional<std::string> input_type;  // compiletime_stack, runtime_stack
-        bool no_auto_verify; // TODO(https://github.com/AztecProtocol/barretenberg/issues/1101): remove
     };
 
     virtual void prove(const Flags& flags,
@@ -36,5 +35,8 @@ class API {
                            const std::filesystem::path& proof_path,
                            const std::filesystem::path& vk_path,
                            const std::filesystem::path& output_path) = 0;
+
+    virtual void write_arbitrary_valid_proof_and_vk_to_file(const API::Flags& flags,
+                                                            const std::filesystem::path& output_dir) = 0;
 };
 } // namespace bb

cpp/src/barretenberg/bb/api.hpp

@@ -3,6 +3,7 @@
 #include "barretenberg/bb/acir_format_getters.hpp"
 #include "barretenberg/bb/api.hpp"
 #include "barretenberg/bb/init_srs.hpp"
+#include "barretenberg/client_ivc/mock_circuit_producer.hpp"
 #include "barretenberg/common/throw_or_abort.hpp"
 #include "libdeflate.h"
 
@@ -125,35 +126,22 @@ class ClientIVCAPI : public API {
         return folding_stack;
     };
 
-    static std::shared_ptr<ClientIVC> _accumulate(std::vector<acir_format::AcirProgram>& folding_stack,
-                                                  bool auto_verify = false)
+    static std::shared_ptr<ClientIVC> _accumulate(std::vector<acir_format::AcirProgram>& folding_stack)
     {
         using Builder = MegaCircuitBuilder;
         using Program = acir_format::AcirProgram;
         using namespace acir_format;
 
-        vinfo("performing accumulation with auto-verify = ", auto_verify);
-
         TraceSettings trace_settings{ E2E_FULL_TEST_STRUCTURE };
-        auto ivc = std::make_shared<ClientIVC>(trace_settings, auto_verify);
+        auto ivc = std::make_shared<ClientIVC>(trace_settings);
 
         const ProgramMetadata metadata{ ivc };
 
         // Accumulate the entire program stack into the IVC
-        // TODO(https://github.com/AztecProtocol/barretenberg/issues/1116): remove manual setting of is_kernel
-        bool is_kernel = false;
         for (Program& program : folding_stack) {
             // Construct a bberg circuit from the acir representation then accumulate it into the IVC
             Builder circuit = acir_format::create_circuit<Builder>(program, metadata);
 
-            // Set the internal is_kernel flag based on the local mechanism only if it has not already been set to true
-            if (ivc->auto_verify_mode) {
-                if (!circuit.databus_propagation_data.is_kernel) {
-                    circuit.databus_propagation_data.is_kernel = is_kernel;
-                }
-                is_kernel = !is_kernel;
-            }
-
             // Do one step of ivc accumulator or, if there is only one circuit in the stack, prove that circuit. In this
             // case, no work is added to the Goblin opqueue, but VM proofs for trivials inputs are produced.
             ivc->accumulate(circuit, /*one_circuit=*/folding_stack.size() == 1);
@@ -183,8 +171,7 @@ class ClientIVCAPI : public API {
         std::vector<acir_format::AcirProgram> folding_stack =
             _build_folding_stack(*flags.input_type, bytecode_path, witness_path);
 
-        bool auto_verify = !flags.no_auto_verify;
-        std::shared_ptr<ClientIVC> ivc = _accumulate(folding_stack, auto_verify);
+        std::shared_ptr<ClientIVC> ivc = _accumulate(folding_stack);
         ClientIVC::Proof proof = ivc->prove();
 
         // Write the proof and verification keys into the working directory in  'binary' format (in practice it seems
@@ -246,11 +233,56 @@ class ClientIVCAPI : public API {
 
         std::vector<acir_format::AcirProgram> folding_stack =
             _build_folding_stack(*flags.input_type, bytecode_path, witness_path);
-        std::shared_ptr<ClientIVC> ivc = _accumulate(folding_stack, /*auto_verify=*/true);
+        std::shared_ptr<ClientIVC> ivc = _accumulate(folding_stack);
         const bool verified = ivc->prove_and_verify();
         return verified;
     };
 
+    /**
+     * @brief Write an arbitrary but valid ClientIVC proof and VK to files
+     * @details used to test the prove_tube flow
+     *
+     * @param flags
+     * @param output_dir
+     */
+    void write_arbitrary_valid_proof_and_vk_to_file(const API::Flags& flags,
+                                                    const std::filesystem::path& output_dir) override
+    {
+        if (!flags.output_type || *flags.output_type != "fields_msgpack") {
+            throw_or_abort("No output_type or output_type not supported");
+        }
+
+        if (!flags.input_type || !(*flags.input_type == "compiletime_stack" || *flags.input_type == "runtime_stack")) {
+            throw_or_abort("No input_type or input_type not supported");
+        }
+
+        // TODO(https://github.com/AztecProtocol/barretenberg/issues/1163) set these dynamically
+        init_bn254_crs(1 << 20);
+        init_grumpkin_crs(1 << 15);
+
+        ClientIVC ivc{ { CLIENT_IVC_BENCH_STRUCTURE } };
+
+        // Construct and accumulate a series of mocked private function execution circuits
+        PrivateFunctionExecutionMockCircuitProducer circuit_producer;
+        size_t NUM_CIRCUITS = 2;
+        for (size_t idx = 0; idx < NUM_CIRCUITS; ++idx) {
+            auto circuit = circuit_producer.create_next_circuit(ivc);
+            ivc.accumulate(circuit);
+        }
+
+        ClientIVC::Proof proof = ivc.prove();
+
+        // Write the proof and verification keys into the working directory in 'binary' format
+        vinfo("writing ClientIVC proof and vk...");
+        write_file(output_dir / "client_ivc_proof", to_buffer(proof));
+
+        auto eccvm_vk = std::make_shared<ECCVMFlavor::VerificationKey>(ivc.goblin.get_eccvm_proving_key());
+        auto translator_vk =
+            std::make_shared<TranslatorFlavor::VerificationKey>(ivc.goblin.get_translator_proving_key());
+        write_file(output_dir / "client_ivc_vk",
+                   to_buffer(ClientIVC::VerificationKey{ ivc.honk_vk, eccvm_vk, translator_vk }));
+    };
+
     void gates([[maybe_unused]] const API::Flags& flags,
                [[maybe_unused]] const std::filesystem::path& bytecode_path,
                [[maybe_unused]] const std::filesystem::path& witness_path) override

cpp/src/barretenberg/bb/api_client_ivc.hpp

@@ -190,8 +190,6 @@ void prove_tube(const std::string& output_path)
 {
     using namespace stdlib::recursion::honk;
 
-    using GoblinVerifierInput = ClientIVCRecursiveVerifier::GoblinVerifierInput;
-    using VerifierInput = ClientIVCRecursiveVerifier::VerifierInput;
     using Builder = UltraCircuitBuilder;
     using GrumpkinVk = bb::VerifierCommitmentKey<curve::Grumpkin>;
 
@@ -211,8 +209,6 @@ void prove_tube(const std::string& output_path)
     // TODO(https://github.com/AztecProtocol/barretenberg/issues/1025)
     vk.eccvm->pcs_verification_key = std::make_shared<GrumpkinVk>(vk.eccvm->circuit_size + 1);
 
-    GoblinVerifierInput goblin_verifier_input{ vk.eccvm, vk.translator };
-    VerifierInput input{ vk.mega, goblin_verifier_input };
     auto builder = std::make_shared<Builder>();
 
     // Preserve the public inputs that should be passed to the base rollup by making them public inputs to the tube
@@ -226,7 +222,7 @@ void prove_tube(const std::string& output_path)
         auto offset = bb::HONK_PROOF_PUBLIC_INPUT_OFFSET;
         builder->add_public_variable(proof.mega_proof[i + offset]);
     }
-    ClientIVCRecursiveVerifier verifier{ builder, input };
+    ClientIVCRecursiveVerifier verifier{ builder, vk };
 
     ClientIVCRecursiveVerifier::Output client_ivc_rec_verifier_output = verifier.verify(proof);
 
@@ -1119,8 +1115,7 @@ int main(int argc, char* argv[])
 
         const API::Flags flags = [&args]() {
             return API::Flags{ .output_type = get_option(args, "--output_type", "fields_msgpack"),
-                               .input_type = get_option(args, "--input_type", "compiletime_stack"),
-                               .no_auto_verify = flag_present(args, "--no_auto_verify") };
+                               .input_type = get_option(args, "--input_type", "compiletime_stack") };
         }();
 
         const std::string command = args[0];
@@ -1158,6 +1153,12 @@ int main(int argc, char* argv[])
                 return api.prove_and_verify(flags, bytecode_path, witness_path) ? 0 : 1;
             }
 
+            if (command == "write_arbitrary_valid_proof_and_vk_to_file") {
+                const std::filesystem::path output_dir = get_option(args, "-o", "./target");
+                api.write_arbitrary_valid_proof_and_vk_to_file(flags, output_dir);
+                return 1;
+            }
+
             throw_or_abort("Invalid command passed to execute_command in bb");
             return 1;
         };

cpp/src/barretenberg/bb/main.cpp

@@ -161,10 +161,6 @@ void ClientIVC::accumulate(ClientCircuit& circuit,
                            const std::shared_ptr<MegaVerificationKey>& precomputed_vk,
                            const bool mock_vk)
 {
-    if (auto_verify_mode && circuit.databus_propagation_data.is_kernel) {
-        complete_kernel_circuit_logic(circuit);
-    }
-
     // Construct merge proof for the present circuit and add to merge verification queue
     MergeProof merge_proof = goblin.prove_merge(circuit);
     merge_verification_queue.emplace_back(merge_proof);
@@ -407,10 +403,8 @@ std::vector<std::shared_ptr<MegaFlavor::VerificationKey>> ClientIVC::precompute_
 
     // Reset the scheme so it can be reused for actual accumulation, maintaining the trace structure setting as is
     TraceSettings settings = trace_settings;
-    bool auto_verify = auto_verify_mode;
     *this = ClientIVC();
     this->trace_settings = settings;
-    this->auto_verify_mode = auto_verify;
 
     return vkeys;
 }

cpp/src/barretenberg/client_ivc/client_ivc.cpp

@@ -126,10 +126,6 @@ class ClientIVC {
     // Settings related to the use of fixed block sizes for each gate in the execution trace
     TraceSettings trace_settings;
 
-    // TODO(https://github.com/AztecProtocol/barretenberg/issues/1101): eventually do away with this.
-    // Setting auto_verify_mode = true will cause kernel completion logic to be added to kernels automatically
-    bool auto_verify_mode;
-
     std::shared_ptr<typename MegaFlavor::CommitmentKey> bn254_commitment_key;
 
     GoblinProver goblin;
@@ -140,10 +136,9 @@ class ClientIVC {
 
     bool initialized = false; // Is the IVC accumulator initialized
 
-    ClientIVC(TraceSettings trace_settings = {}, bool auto_verify_mode = false)
+    ClientIVC(TraceSettings trace_settings = {})
         : trace_usage_tracker(trace_settings)
         , trace_settings(trace_settings)
-        , auto_verify_mode(auto_verify_mode)
         , bn254_commitment_key(trace_settings.structure.has_value()
                                    ? std::make_shared<CommitmentKey<curve::BN254>>(trace_settings.dyadic_size())
                                    : nullptr)
@@ -191,5 +186,12 @@ class ClientIVC {
 
     std::vector<std::shared_ptr<MegaVerificationKey>> precompute_folding_verification_keys(
         std::vector<ClientCircuit> circuits);
+
+    VerificationKey get_vk() const
+    {
+        return { honk_vk,
+                 std::make_shared<ECCVMVerificationKey>(goblin.get_eccvm_proving_key()),
+                 std::make_shared<TranslatorVerificationKey>(goblin.get_translator_proving_key()) };
+    }
 };
 } // namespace bb