feat: unified create circuit from acir (#10440)

The main goal of this PR is to allow for all circuits (including
kernels) to be created using a single `create_circuit()` method. (Prior
to this PR things had diverged for kernels/ivc_recursion_constraints and
a separate `create_kernel_circuit()` method was required). To facilitate
this and as general cleanup, this PR introduces struct `ProgramMetadata`
so that the create_circuit interface is reduced to
`create_circuit(AcirProgram&, ProgramMetadata&)`.

Note: `ProgramMetadata` simply contains all the stuff that used to be
individual defaulted inputs to the create_circuit methods (plus a
pointer to a ClientIVC instance). This is a better pattern but I haven't
yet made an effort to address whether some of the parameters can be
removed altogether. (It may be that they cannot).

Author: ledwards2225

barretenberg/cpp/src/barretenberg/bb/api_client_ivc.hpp

@@ -125,79 +125,38 @@ class ClientIVCAPI : public API {
         return folding_stack;
     };
 
-    static ClientIVC _accumulate(std::vector<acir_format::AcirProgram>& folding_stack)
+    static std::shared_ptr<ClientIVC> _accumulate(std::vector<acir_format::AcirProgram>& folding_stack,
+                                                  bool auto_verify = false)
     {
         using Builder = MegaCircuitBuilder;
         using Program = acir_format::AcirProgram;
-
         using namespace acir_format;
 
-        // TODO(https://github.com/AztecProtocol/barretenberg/issues/1163) set these dynamically
-        init_bn254_crs(1 << 20);
-        init_grumpkin_crs(1 << 15);
+        vinfo("performing accumulation with auto-verify = ", auto_verify);
+
+        TraceSettings trace_settings{ E2E_FULL_TEST_STRUCTURE };
+        auto ivc = std::make_shared<ClientIVC>(trace_settings, auto_verify);
 
-        // TODO(#7371) dedupe this with the rest of the similar code
-        // TODO(https://github.com/AztecProtocol/barretenberg/issues/1101): remove use of auto_verify_mode
-        ClientIVC ivc{ { E2E_FULL_TEST_STRUCTURE }, /*auto_verify_mode=*/true };
+        const ProgramMetadata metadata{ ivc };
 
         // Accumulate the entire program stack into the IVC
         // TODO(https://github.com/AztecProtocol/barretenberg/issues/1116): remove manual setting of is_kernel
         bool is_kernel = false;
         for (Program& program : folding_stack) {
             // Construct a bberg circuit from the acir representation then accumulate it into the IVC
-            Builder circuit = acir_format::create_circuit<Builder>(
-                program.constraints, true, 0, program.witness, false, ivc.goblin.op_queue);
+            Builder circuit = acir_format::create_circuit<Builder>(program, metadata);
 
             // Set the internal is_kernel flag based on the local mechanism only if it has not already been set to true
-            if (!circuit.databus_propagation_data.is_kernel) {
-                circuit.databus_propagation_data.is_kernel = is_kernel;
+            if (ivc->auto_verify_mode) {
+                if (!circuit.databus_propagation_data.is_kernel) {
+                    circuit.databus_propagation_data.is_kernel = is_kernel;
+                }
+                is_kernel = !is_kernel;
             }
-            is_kernel = !is_kernel;
 
             // Do one step of ivc accumulator or, if there is only one circuit in the stack, prove that circuit. In this
             // case, no work is added to the Goblin opqueue, but VM proofs for trivials inputs are produced.
-            ivc.accumulate(circuit, /*one_circuit=*/folding_stack.size() == 1);
-        }
-
-        return ivc;
-    };
-
-    static ClientIVC _accumulate_without_auto_verify(std::vector<acir_format::AcirProgram>& folding_stack)
-    {
-        using Builder = MegaCircuitBuilder;
-        using Program = acir_format::AcirProgram;
-
-        using namespace acir_format;
-
-        // TODO(https://github.com/AztecProtocol/barretenberg/issues/1163) set these dynamically
-        init_bn254_crs(1 << 20);
-        init_grumpkin_crs(1 << 15);
-
-        // TODO(#7371) dedupe this with the rest of the similar code
-        // TODO(https://github.com/AztecProtocol/barretenberg/issues/1101): remove use of auto_verify_mode
-        ClientIVC ivc{ { E2E_FULL_TEST_STRUCTURE }, /*auto_verify_mode=*/false };
-
-        // Accumulate the entire program stack into the IVC
-        // TODO(https://github.com/AztecProtocol/barretenberg/issues/1116): remove manual setting of is_kernel once
-        // databus has been integrated into noir kernel programs
-        bool is_kernel = false;
-        for (Program& program : folding_stack) {
-
-            Builder circuit;
-
-            is_kernel = !program.constraints.ivc_recursion_constraints.empty();
-            if (is_kernel) {
-                vinfo("Accumulating KERNEL.");
-                circuit = create_kernel_circuit(program.constraints, ivc, program.witness);
-            } else {
-                vinfo("Accumulating APP.");
-                circuit = create_circuit<Builder>(
-                    program.constraints, /*recursive=*/false, 0, program.witness, false, ivc.goblin.op_queue);
-            }
-
-            // Do one step of ivc accumulator or, if there is only one circuit in the stack, prove that circuit. In this
-            // case, no work is added to the Goblin opqueue, but VM proofs for trivial inputs are produced.
-            ivc.accumulate(circuit, /*one_circuit=*/folding_stack.size() == 1);
+            ivc->accumulate(circuit, /*one_circuit=*/folding_stack.size() == 1);
         }
 
         return ivc;
@@ -217,33 +176,27 @@ class ClientIVCAPI : public API {
             throw_or_abort("No input_type or input_type not supported");
         }
 
-        std::vector<acir_format::AcirProgram> folding_stack =
-            _build_folding_stack(*flags.input_type, bytecode_path, witness_path);
-
         // TODO(https://github.com/AztecProtocol/barretenberg/issues/1163) set these dynamically
         init_bn254_crs(1 << 20);
         init_grumpkin_crs(1 << 15);
 
-        ClientIVC ivc;
-        if (flags.no_auto_verify) {
-            vinfo("performing accumulation WITHOUT auto-verify");
-            ivc = _accumulate_without_auto_verify(folding_stack);
-        } else {
-            vinfo("performing accumulation with auto-verify");
-            ivc = _accumulate(folding_stack);
-        }
-        ClientIVC::Proof proof = ivc.prove();
+        std::vector<acir_format::AcirProgram> folding_stack =
+            _build_folding_stack(*flags.input_type, bytecode_path, witness_path);
+
+        bool auto_verify = !flags.no_auto_verify;
+        std::shared_ptr<ClientIVC> ivc = _accumulate(folding_stack, auto_verify);
+        ClientIVC::Proof proof = ivc->prove();
 
         // Write the proof and verification keys into the working directory in  'binary' format (in practice it seems
         // this directory is passed by bb.js)
         vinfo("writing ClientIVC proof and vk...");
         write_file(output_dir / "client_ivc_proof", to_buffer(proof));
 
-        auto eccvm_vk = std::make_shared<ECCVMFlavor::VerificationKey>(ivc.goblin.get_eccvm_proving_key());
+        auto eccvm_vk = std::make_shared<ECCVMFlavor::VerificationKey>(ivc->goblin.get_eccvm_proving_key());
         auto translator_vk =
-            std::make_shared<TranslatorFlavor::VerificationKey>(ivc.goblin.get_translator_proving_key());
+            std::make_shared<TranslatorFlavor::VerificationKey>(ivc->goblin.get_translator_proving_key());
         write_file(output_dir / "client_ivc_vk",
-                   to_buffer(ClientIVC::VerificationKey{ ivc.honk_vk, eccvm_vk, translator_vk }));
+                   to_buffer(ClientIVC::VerificationKey{ ivc->honk_vk, eccvm_vk, translator_vk }));
     };
 
     /**
@@ -286,10 +239,15 @@ class ClientIVCAPI : public API {
         if (!flags.input_type || !(*flags.input_type == "compiletime_stack" || *flags.input_type == "runtime_stack")) {
             throw_or_abort("No input_type or input_type not supported");
         }
+
+        // TODO(https://github.com/AztecProtocol/barretenberg/issues/1163) set these dynamically
+        init_bn254_crs(1 << 20);
+        init_grumpkin_crs(1 << 15);
+
         std::vector<acir_format::AcirProgram> folding_stack =
             _build_folding_stack(*flags.input_type, bytecode_path, witness_path);
-        ClientIVC ivc = _accumulate(folding_stack);
-        const bool verified = ivc.prove_and_verify();
+        std::shared_ptr<ClientIVC> ivc = _accumulate(folding_stack, /*auto_verify=*/true);
+        const bool verified = ivc->prove_and_verify();
         return verified;
     };
 

barretenberg/cpp/src/barretenberg/bb/main.cpp

@@ -110,21 +110,15 @@ bool proveAndVerify(const std::string& bytecodePath, const bool recursive, const
 }
 
 template <IsUltraFlavor Flavor>
-bool proveAndVerifyHonkAcirFormat(acir_format::AcirFormat constraint_system,
-                                  const bool recursive,
-                                  acir_format::WitnessVector witness)
+bool proveAndVerifyHonkAcirFormat(acir_format::AcirProgram program, acir_format::ProgramMetadata metadata)
 {
     using Builder = Flavor::CircuitBuilder;
     using Prover = UltraProver_<Flavor>;
     using Verifier = UltraVerifier_<Flavor>;
     using VerificationKey = Flavor::VerificationKey;
 
-    bool honk_recursion = false;
-    if constexpr (IsAnyOf<Flavor, UltraFlavor>) {
-        honk_recursion = true;
-    }
     // Construct a bberg circuit from the acir representation
-    auto builder = acir_format::create_circuit<Builder>(constraint_system, recursive, 0, witness, honk_recursion);
+    auto builder = acir_format::create_circuit<Builder>(program, metadata);
 
     // Construct Honk proof
     Prover prover{ builder };
@@ -149,15 +143,15 @@ bool proveAndVerifyHonkAcirFormat(acir_format::AcirFormat constraint_system,
 template <IsUltraFlavor Flavor>
 bool proveAndVerifyHonk(const std::string& bytecodePath, const bool recursive, const std::string& witnessPath)
 {
-    bool honk_recursion = false;
-    if constexpr (IsAnyOf<Flavor, UltraFlavor>) {
-        honk_recursion = true;
-    }
+    constexpr bool honk_recursion = IsAnyOf<Flavor, UltraFlavor>;
+    const acir_format::ProgramMetadata metadata{ .recursive = recursive, .honk_recursion = honk_recursion };
+
     // Populate the acir constraint system and witness from gzipped data
-    auto constraint_system = get_constraint_system(bytecodePath, honk_recursion);
-    auto witness = get_witness(witnessPath);
+    acir_format::AcirProgram program;
+    program.constraints = get_constraint_system(bytecodePath, metadata.honk_recursion);
+    program.witness = get_witness(witnessPath);
 
-    return proveAndVerifyHonkAcirFormat<Flavor>(constraint_system, recursive, witness);
+    return proveAndVerifyHonkAcirFormat<Flavor>(program, metadata);
 }
 
 /**
@@ -171,14 +165,14 @@ bool proveAndVerifyHonk(const std::string& bytecodePath, const bool recursive, c
 template <IsUltraFlavor Flavor>
 bool proveAndVerifyHonkProgram(const std::string& bytecodePath, const bool recursive, const std::string& witnessPath)
 {
-    bool honk_recursion = false;
-    if constexpr (IsAnyOf<Flavor, UltraFlavor>) {
-        honk_recursion = true;
-    }
-    auto program_stack = acir_format::get_acir_program_stack(bytecodePath, witnessPath, honk_recursion);
+    constexpr bool honk_recursion = IsAnyOf<Flavor, UltraFlavor>;
+    const acir_format::ProgramMetadata metadata{ .recursive = recursive, .honk_recursion = honk_recursion };
+
+    auto program_stack = acir_format::get_acir_program_stack(bytecodePath, witnessPath, metadata.honk_recursion);
+
     while (!program_stack.empty()) {
-        auto stack_item = program_stack.back();
-        if (!proveAndVerifyHonkAcirFormat<Flavor>(stack_item.constraints, recursive, stack_item.witness)) {
+        auto program = program_stack.back();
+        if (!proveAndVerifyHonkAcirFormat<Flavor>(program, metadata)) {
             return false;
         }
         program_stack.pop_back();
@@ -329,25 +323,29 @@ void gateCount(const std::string& bytecodePath, bool recursive, bool honk_recurs
     // All circuit reports will be built into the string below
     std::string functions_string = "{\"functions\": [\n  ";
     auto constraint_systems = get_constraint_systems(bytecodePath, honk_recursion);
+
+    const acir_format::ProgramMetadata metadata{ .recursive = recursive,
+                                                 .honk_recursion = honk_recursion,
+                                                 .collect_gates_per_opcode = true };
     size_t i = 0;
-    for (auto constraint_system : constraint_systems) {
-        auto builder = acir_format::create_circuit<Builder>(
-            constraint_system, recursive, 0, {}, honk_recursion, std::make_shared<bb::ECCOpQueue>(), true);
+    for (const auto& constraint_system : constraint_systems) {
+        acir_format::AcirProgram program{ constraint_system };
+        auto builder = acir_format::create_circuit<Builder>(program, metadata);
         builder.finalize_circuit(/*ensure_nonzero=*/true);
         size_t circuit_size = builder.num_gates;
         vinfo("Calculated circuit size in gateCount: ", circuit_size);
 
         // Build individual circuit report
         std::string gates_per_opcode_str;
-        for (size_t j = 0; j < constraint_system.gates_per_opcode.size(); j++) {
-            gates_per_opcode_str += std::to_string(constraint_system.gates_per_opcode[j]);
-            if (j != constraint_system.gates_per_opcode.size() - 1) {
+        for (size_t j = 0; j < program.constraints.gates_per_opcode.size(); j++) {
+            gates_per_opcode_str += std::to_string(program.constraints.gates_per_opcode[j]);
+            if (j != program.constraints.gates_per_opcode.size() - 1) {
                 gates_per_opcode_str += ",";
             }
         }
 
         auto result_string = format("{\n        \"acir_opcodes\": ",
-                                    constraint_system.num_acir_opcodes,
+                                    program.constraints.num_acir_opcodes,
                                     ",\n        \"circuit_size\": ",
                                     circuit_size,
                                     ",\n        \"gates_per_opcode\": [",
@@ -713,17 +711,15 @@ UltraProver_<Flavor> compute_valid_prover(const std::string& bytecodePath,
     using Builder = Flavor::CircuitBuilder;
     using Prover = UltraProver_<Flavor>;
 
-    bool honk_recursion = false;
-    if constexpr (IsAnyOf<Flavor, UltraFlavor, UltraKeccakFlavor, UltraRollupFlavor>) {
-        honk_recursion = true;
-    }
-    auto constraint_system = get_constraint_system(bytecodePath, honk_recursion);
-    acir_format::WitnessVector witness = {};
+    constexpr bool honk_recursion = IsAnyOf<Flavor, UltraFlavor, UltraKeccakFlavor, UltraRollupFlavor>;
+    const acir_format::ProgramMetadata metadata{ .recursive = recursive, .honk_recursion = honk_recursion };
+
+    acir_format::AcirProgram program{ get_constraint_system(bytecodePath, metadata.honk_recursion) };
     if (!witnessPath.empty()) {
-        witness = get_witness(witnessPath);
+        program.witness = get_witness(witnessPath);
     }
 
-    auto builder = acir_format::create_circuit<Builder>(constraint_system, recursive, 0, witness, honk_recursion);
+    auto builder = acir_format::create_circuit<Builder>(program, metadata);
     auto prover = Prover{ builder };
     init_bn254_crs(prover.proving_key->proving_key.circuit_size);
     return std::move(prover);
@@ -846,28 +842,23 @@ void write_vk_for_ivc(const std::string& bytecodePath, const std::string& output
     using Prover = ClientIVC::MegaProver;
     using DeciderProvingKey = ClientIVC::DeciderProvingKey;
     using VerificationKey = ClientIVC::MegaVerificationKey;
+    using Program = acir_format::AcirProgram;
+    using ProgramMetadata = acir_format::ProgramMetadata;
 
     // TODO(https://github.com/AztecProtocol/barretenberg/issues/1163) set these dynamically
     init_bn254_crs(1 << 20);
     init_grumpkin_crs(1 << 15);
 
-    auto constraints = get_constraint_system(bytecodePath, /*honk_recursion=*/false);
-    acir_format::WitnessVector witness = {};
+    Program program{ get_constraint_system(bytecodePath, /*honk_recursion=*/false), /*witness=*/{} };
+    auto& ivc_constraints = program.constraints.ivc_recursion_constraints;
 
     TraceSettings trace_settings{ E2E_FULL_TEST_STRUCTURE };
 
-    // The presence of ivc recursion constraints determines whether or not the program is a kernel
-    bool is_kernel = !constraints.ivc_recursion_constraints.empty();
+    const ProgramMetadata metadata{ .ivc = ivc_constraints.empty()
+                                               ? nullptr
+                                               : create_mock_ivc_from_constraints(ivc_constraints, trace_settings) };
+    Builder builder = acir_format::create_circuit<Builder>(program, metadata);
 
-    Builder builder;
-    if (is_kernel) {
-        // Create a mock IVC instance based on the IVC recursion constraints in the kernel program
-        ClientIVC mock_ivc = create_mock_ivc_from_constraints(constraints.ivc_recursion_constraints, trace_settings);
-        builder = acir_format::create_kernel_circuit(constraints, mock_ivc, witness);
-    } else {
-        builder = acir_format::create_circuit<Builder>(
-            constraints, /*recursive=*/false, 0, witness, /*honk_recursion=*/false);
-    }
     // Add public inputs corresponding to pairing point accumulator
     builder.add_pairing_point_accumulator(stdlib::recursion::init_default_agg_obj_indices<Builder>(builder));
 
@@ -907,10 +898,12 @@ void write_recursion_inputs_honk(const std::string& bytecodePath,
     using VerificationKey = Flavor::VerificationKey;
     using FF = Flavor::FF;
 
-    bool honk_recursion = true;
-    auto constraints = get_constraint_system(bytecodePath, honk_recursion);
-    auto witness = get_witness(witnessPath);
-    auto builder = acir_format::create_circuit<Builder>(constraints, recursive, 0, witness, honk_recursion);
+    const acir_format::ProgramMetadata metadata{ .recursive = recursive, .honk_recursion = true };
+
+    acir_format::AcirProgram program;
+    program.constraints = get_constraint_system(bytecodePath, metadata.honk_recursion);
+    program.witness = get_witness(witnessPath);
+    auto builder = acir_format::create_circuit<Builder>(program, metadata);
 
     // Construct Honk proof and verification key
     Prover prover{ builder };
@@ -1058,15 +1051,13 @@ void prove_honk_output_all(const std::string& bytecodePath,
     using Prover = UltraProver_<Flavor>;
     using VerificationKey = Flavor::VerificationKey;
 
-    bool honk_recursion = false;
-    if constexpr (IsAnyOf<Flavor, UltraFlavor, UltraKeccakFlavor>) {
-        honk_recursion = true;
-    }
+    constexpr bool honk_recursion = IsAnyOf<Flavor, UltraFlavor, UltraKeccakFlavor>;
+    const acir_format::ProgramMetadata metadata{ .recursive = recursive, .honk_recursion = honk_recursion };
 
-    auto constraint_system = get_constraint_system(bytecodePath, honk_recursion);
-    auto witness = get_witness(witnessPath);
+    acir_format::AcirProgram program{ get_constraint_system(bytecodePath, metadata.honk_recursion),
+                                      get_witness(witnessPath) };
 
-    auto builder = acir_format::create_circuit<Builder>(constraint_system, recursive, 0, witness, honk_recursion);
+    auto builder = acir_format::create_circuit<Builder>(program, metadata);
 
     // Construct Honk proof
     Prover prover{ builder };