feat: Verification key stuff (#8431)

The previous
[PR](AztecProtocol/aztec-packages#8230) in this
series set up the basic infrastructure for completing kernel circuits
from acir RecursionConstraints, including connecting the dummy proof in
the constraint to the genuine witnesses known internally in AztecIvc.
This PR completes this line work by using the (genuine) verification key
witnesses in the constraint to construct the stdlib verification keys
from which the recursive verifiers are instantiated.

Doing this properly involved correcting/implementing/testing various
serialization/deserialization/construction methods for native and stdlib
verification keys - this accounts for most of the changes in this PR.
The corrections mostly applied to Mega and were related to not
accounting for `databus_propagation_data`. I also updated some of the
corresponding Ultra methods in an attempt to make them a bit more
readable and more easily maintainable.

closes #1090

Author: ledwards2225

cpp/src/barretenberg/aztec_ivc/aztec_ivc.cpp

@@ -4,16 +4,32 @@
 namespace bb {
 
 /**
- * @brief Instantiate a stdlib verification queue corresponding to the native counterpart
+ * @brief Instantiate a stdlib verification queue for use in the kernel completion logic
+ * @details Construct a stdlib proof/verification_key for each entry in the native verification queue. By default, both
+ * are constructed from their counterpart in the native queue. Alternatively, Stdlib verification keys can be provided
+ * directly as input to this method. (The later option is used, for example, when constructing recursive verifiers based
+ * on the verification key witnesses from an acir recursion constraint. This option is not provided for proofs since
+ * valid proof witnesses are in general not known at the time of acir constraint generation).
  *
  * @param circuit
  */
-void AztecIVC::instantiate_stdlib_verification_queue(ClientCircuit& circuit)
+void AztecIVC::instantiate_stdlib_verification_queue(
+    ClientCircuit& circuit, const std::vector<std::shared_ptr<RecursiveVerificationKey>>& input_keys)
 {
+    bool vkeys_provided = !input_keys.empty();
+    if (vkeys_provided && verification_queue.size() != input_keys.size()) {
+        info("Warning: Incorrect number of verification keys provided in stdlib verification queue instantiation.");
+        ASSERT(false);
+    }
+
+    size_t key_idx = 0;
     for (auto& [proof, vkey, type] : verification_queue) {
-        // Construct stdlib verification key and proof
+        // Construct stdlib proof directly from the internal native queue data
         auto stdlib_proof = bb::convert_proof_to_witness(&circuit, proof);
-        auto stdlib_vkey = std::make_shared<RecursiveVerificationKey>(&circuit, vkey);
+
+        // Use the provided stdlib vkey if present, otherwise construct one from the internal native queue
+        auto stdlib_vkey =
+            vkeys_provided ? input_keys[key_idx++] : std::make_shared<RecursiveVerificationKey>(&circuit, vkey);
 
         stdlib_verification_queue.push_back({ stdlib_proof, stdlib_vkey, type });
     }

cpp/src/barretenberg/aztec_ivc/aztec_ivc.hpp

@@ -111,7 +111,8 @@ class AztecIVC {
 
     bool initialized = false; // Is the IVC accumulator initialized
 
-    void instantiate_stdlib_verification_queue(ClientCircuit& circuit);
+    void instantiate_stdlib_verification_queue(
+        ClientCircuit& circuit, const std::vector<std::shared_ptr<RecursiveVerificationKey>>& input_keys = {});
 
     void perform_recursive_verification_and_databus_consistency_checks(
         ClientCircuit& circuit,

cpp/src/barretenberg/commitment_schemes/verification_key.hpp

@@ -34,6 +34,7 @@ template <> class VerifierCommitmentKey<curve::BN254> {
     using Commitment = typename Curve::AffineElement;
 
     VerifierCommitmentKey() { srs = srs::get_crs_factory<Curve>()->get_verifier_crs(); };
+    bool operator==(const VerifierCommitmentKey&) const = default;
 
     Commitment get_g1_identity() { return srs->get_g1_identity(); }
 

cpp/src/barretenberg/dsl/acir_format/acir_format.cpp

@@ -479,6 +479,12 @@ MegaCircuitBuilder create_circuit(AcirFormat& constraint_system,
 
 /**
  * @brief Create a kernel circuit from a constraint system and an IVC instance
+ * @details This method processes ivc_recursion_constraints using the kernel completion logic contained in AztecIvc.
+ * Since verification keys are known at the time of acir generation, the verification key witnesses contained in the
+ * constraints are used directly to instantiate the recursive verifiers. On the other hand, the proof witnesses
+ * contained in the constraints are generally 'dummy' values since proofs are not known during acir generation (with the
+ * exception of public inputs). This is remedied by connecting the dummy proof witnesses to the genuine proof witnesses,
+ * known internally to the IVC class, via copy constraints.
  *
  * @param constraint_system AcirFormat constraint system possibly containing IVC recursion constraints
  * @param ivc An IVC instance containing internal data about proofs to be verified
@@ -491,6 +497,8 @@ MegaCircuitBuilder create_kernel_circuit(AcirFormat& constraint_system,
                                          const WitnessVector& witness,
                                          const size_t size_hint)
 {
+    using StdlibVerificationKey = AztecIVC::RecursiveVerificationKey;
+
     // Construct the main kernel circuit logic excluding recursive verifiers
     auto circuit = create_circuit<MegaCircuitBuilder>(constraint_system,
                                                       size_hint,
@@ -499,18 +507,26 @@ MegaCircuitBuilder create_kernel_circuit(AcirFormat& constraint_system,
                                                       ivc.goblin.op_queue,
                                                       /*collect_gates_per_opcode=*/false);
 
-    // We expect the length of the internal verification queue to matche the number of ivc recursion constraints
+    // We expect the length of the internal verification queue to match the number of ivc recursion constraints
     if (constraint_system.ivc_recursion_constraints.size() != ivc.verification_queue.size()) {
         info("WARNING: Mismatch in number of recursive verifications during kernel creation!");
         ASSERT(false);
     }
 
-    // Create stdlib representations of each {proof, vkey} pair in the queue based on their native counterparts
-    ivc.instantiate_stdlib_verification_queue(circuit);
+    // Construct a stdlib verification key for each constraint based on the verification key witness indices therein
+    std::vector<std::shared_ptr<StdlibVerificationKey>> stdlib_verification_keys;
+    stdlib_verification_keys.reserve(constraint_system.ivc_recursion_constraints.size());
+    for (const auto& constraint : constraint_system.ivc_recursion_constraints) {
+        stdlib_verification_keys.push_back(std::make_shared<StdlibVerificationKey>(
+            StdlibVerificationKey::from_witness_indices(circuit, constraint.key)));
+    }
+
+    // Create stdlib representations of each {proof, vkey} pair to be recursively verified
+    ivc.instantiate_stdlib_verification_queue(circuit, stdlib_verification_keys);
 
-    // Connect each {proof, vkey} pair from the constraint to the corresponding entry in the internal verification
-    // queue. This ensures that the witnesses utlized in constraints generated based on acir are properly connected to
-    // the constraints generated herein via the ivc scheme (e.g. recursive verifications).
+    // Connect the proof/public_input witness indices from each constraint to the corresponding proof witnesses in the
+    // internal verification queue. This ensures that the witnesses utlized in constraints generated based on acir are
+    // properly connected to the constraints generated herein via the ivc scheme (e.g. recursive verifications).
     for (auto [constraint, queue_entry] :
          zip_view(constraint_system.ivc_recursion_constraints, ivc.stdlib_verification_queue)) {
 
@@ -520,11 +536,9 @@ MegaCircuitBuilder create_kernel_circuit(AcirFormat& constraint_system,
         ASSERT(complete_proof_indices.size() == queue_entry.proof.size());
 
         // Assert equality between the proof indices from the constraint data and those of the internal proof
-        for (auto [proof_idx, proof_value] : zip_view(complete_proof_indices, queue_entry.proof)) {
+        for (auto [proof_value, proof_idx] : zip_view(queue_entry.proof, complete_proof_indices)) {
             circuit.assert_equal(proof_value.get_witness_index(), proof_idx);
         }
-        // TODO(https://github.com/AztecProtocol/barretenberg/issues/1090): assert equality between the internal vkey
-        // and the constaint vkey, or simply use the constraint vkey directly to construct the stdlib vkey used in IVC.
     }
 
     // Complete the kernel circuit with all required recursive verifications, databus consistency checks etc.

cpp/src/barretenberg/ecc/fields/field_conversion.hpp

@@ -93,7 +93,7 @@ std::vector<bb::fr> convert_grumpkin_fr_to_bn254_frs(const grumpkin::fr& val);
  */
 template <typename T> std::vector<bb::fr> convert_to_bn254_frs(const T& val)
 {
-    if constexpr (IsAnyOf<T, bool, uint32_t, uint64_t, bb::fr>) {
+    if constexpr (IsAnyOf<T, bool, uint32_t, uint64_t, size_t, bb::fr>) {
         std::vector<bb::fr> fr_vec{ val };
         return fr_vec;
     } else if constexpr (IsAnyOf<T, grumpkin::fr>) {

cpp/src/barretenberg/flavor/flavor.hpp

@@ -93,10 +93,10 @@ namespace bb {
  */
 class PrecomputedEntitiesBase {
   public:
+    bool operator==(const PrecomputedEntitiesBase& other) const = default;
     uint64_t circuit_size;
     uint64_t log_circuit_size;
     uint64_t num_public_inputs;
-    CircuitType circuit_type; // TODO(#392)
 };
 
 /**
@@ -158,6 +158,7 @@ class VerificationKey_ : public PrecomputedCommitments {
     AggregationObjectPubInputIndices recursive_proof_public_input_indices = {};
     uint64_t pub_inputs_offset = 0;
 
+    bool operator==(const VerificationKey_&) const = default;
     VerificationKey_() = default;
     VerificationKey_(const size_t circuit_size, const size_t num_public_inputs)
     {
@@ -173,32 +174,25 @@ class VerificationKey_ : public PrecomputedCommitments {
      */
     std::vector<FF> to_field_elements()
     {
+        using namespace bb::field_conversion;
+
+        auto serialize_to_field_buffer = [](const auto& input, std::vector<FF>& buffer) {
+            std::vector<FF> input_fields = convert_to_bn254_frs(input);
+            buffer.insert(buffer.end(), input_fields.begin(), input_fields.end());
+        };
+
         std::vector<FF> elements;
-        std::vector<FF> circuit_size_elements = bb::field_conversion::convert_to_bn254_frs(this->circuit_size);
-        elements.insert(elements.end(), circuit_size_elements.begin(), circuit_size_elements.end());
-        // do the same for the rest of the fields
-        std::vector<FF> num_public_inputs_elements =
-            bb::field_conversion::convert_to_bn254_frs(this->num_public_inputs);
-        elements.insert(elements.end(), num_public_inputs_elements.begin(), num_public_inputs_elements.end());
-        std::vector<FF> pub_inputs_offset_elements =
-            bb::field_conversion::convert_to_bn254_frs(this->pub_inputs_offset);
-        elements.insert(elements.end(), pub_inputs_offset_elements.begin(), pub_inputs_offset_elements.end());
-
-        std::vector<FF> contains_recursive_proof_elements =
-            bb::field_conversion::convert_to_bn254_frs(this->contains_recursive_proof);
-        elements.insert(
-            elements.end(), contains_recursive_proof_elements.begin(), contains_recursive_proof_elements.end());
-
-        std::vector<FF> recursive_proof_public_input_indices_elements =
-            bb::field_conversion::convert_to_bn254_frs(this->recursive_proof_public_input_indices);
-        elements.insert(elements.end(),
-                        recursive_proof_public_input_indices_elements.begin(),
-                        recursive_proof_public_input_indices_elements.end());
-
-        for (Commitment& comm : this->get_all()) {
-            std::vector<FF> comm_elements = bb::field_conversion::convert_to_bn254_frs(comm);
-            elements.insert(elements.end(), comm_elements.begin(), comm_elements.end());
+
+        serialize_to_field_buffer(this->circuit_size, elements);
+        serialize_to_field_buffer(this->num_public_inputs, elements);
+        serialize_to_field_buffer(this->pub_inputs_offset, elements);
+        serialize_to_field_buffer(this->contains_recursive_proof, elements);
+        serialize_to_field_buffer(this->recursive_proof_public_input_indices, elements);
+
+        for (Commitment& commitment : this->get_all()) {
+            serialize_to_field_buffer(commitment, elements);
         }
+
         return elements;
     }
 

cpp/src/barretenberg/stdlib/primitives/field/field_conversion.hpp

@@ -161,4 +161,22 @@ template <typename Builder, typename T> std::vector<fr<Builder>> convert_to_bn25
     }
 }
 
+/**
+ * @brief Deserialize an object of specified type from a buffer of field elements; update provided read count in place
+ *
+ * @tparam TargetType Type to reconstruct from buffer of field elements
+ * @param builder
+ * @param elements Buffer of field elements
+ * @param num_frs_read Index at which to read into buffer
+ */
+template <typename TargetType, typename Builder>
+TargetType deserialize_from_frs(Builder& builder, std::span<fr<Builder>> elements, size_t& num_frs_read)
+{
+    size_t num_frs = calc_num_bn254_frs<Builder, TargetType>();
+    ASSERT(elements.size() >= num_frs_read + num_frs);
+    TargetType result = convert_from_bn254_frs<Builder, TargetType>(builder, elements.subspan(num_frs_read, num_frs));
+    num_frs_read += num_frs;
+    return result;
+}
+
 } // namespace bb::stdlib::field_conversion

cpp/src/barretenberg/stdlib/protogalaxy_verifier/recursive_decider_verification_key.hpp

@@ -22,6 +22,7 @@ template <IsRecursiveFlavor Flavor> class RecursiveDeciderVerificationKey_ {
     using Builder = typename Flavor::CircuitBuilder;
     using NativeFlavor = typename Flavor::NativeFlavor;
     using DeciderVerificationKey = bb::DeciderVerificationKey_<NativeFlavor>;
+    using VerifierCommitmentKey = typename NativeFlavor::VerifierCommitmentKey;
 
     Builder* builder;
 
@@ -101,7 +102,9 @@ template <IsRecursiveFlavor Flavor> class RecursiveDeciderVerificationKey_ {
     {
         auto native_honk_vk = std::make_shared<NativeVerificationKey>(verification_key->circuit_size,
                                                                       verification_key->num_public_inputs);
-        native_honk_vk->pcs_verification_key = verification_key->pcs_verification_key;
+        native_honk_vk->pcs_verification_key = verification_key->pcs_verification_key == nullptr
+                                                   ? std::make_shared<VerifierCommitmentKey>()
+                                                   : verification_key->pcs_verification_key;
         native_honk_vk->pub_inputs_offset = verification_key->pub_inputs_offset;
         native_honk_vk->contains_recursive_proof = verification_key->contains_recursive_proof;
         native_honk_vk->recursive_proof_public_input_indices = verification_key->recursive_proof_public_input_indices;

cpp/src/barretenberg/stdlib_circuit_builders/databus.hpp

@@ -68,6 +68,8 @@ enum class BusId { CALLDATA, SECONDARY_CALLDATA, RETURNDATA };
  *
  */
 struct DatabusPropagationData {
+    bool operator==(const DatabusPropagationData&) const = default;
+
     // Flags indicating whether the public inputs contain commitment(s) to app/kernel return data
     bool contains_app_return_data_commitment = false;
     bool contains_kernel_return_data_commitment = false;
@@ -80,6 +82,12 @@ struct DatabusPropagationData {
 
     // Is this a kernel circuit (used to determine when databus consistency checks can be appended to a circuit in IVC)
     bool is_kernel = false;
+
+    MSGPACK_FIELDS(contains_app_return_data_commitment,
+                   contains_kernel_return_data_commitment,
+                   app_return_data_public_input_idx,
+                   kernel_return_data_public_input_idx,
+                   is_kernel);
 };
 
 } // namespace bb

cpp/src/barretenberg/stdlib_circuit_builders/flavor_serialization.test.cpp

@@ -0,0 +1,65 @@
+#include "barretenberg/common/serialize.hpp"
+#include "barretenberg/ecc/curves/bn254/fr.hpp"
+#include "barretenberg/numeric/uint256/uint256.hpp"
+#include "barretenberg/plonk_honk_shared/library/grand_product_delta.hpp"
+#include "barretenberg/relations/permutation_relation.hpp"
+#include "barretenberg/relations/relation_parameters.hpp"
+#include "barretenberg/stdlib_circuit_builders/mock_circuits.hpp"
+#include "barretenberg/stdlib_circuit_builders/plookup_tables/fixed_base/fixed_base.hpp"
+#include "barretenberg/stdlib_circuit_builders/plookup_tables/types.hpp"
+#include "barretenberg/stdlib_circuit_builders/ultra_circuit_builder.hpp"
+#include "barretenberg/sumcheck/sumcheck_round.hpp"
+#include "barretenberg/ultra_honk/ultra_prover.hpp"
+#include "barretenberg/ultra_honk/ultra_verifier.hpp"
+
+#include <gtest/gtest.h>
+
+using namespace bb;
+
+template <typename Flavor> class FlavorSerializationTests : public ::testing::Test {
+  public:
+    using Builder = typename Flavor::CircuitBuilder;
+    using DeciderProvingKey = DeciderProvingKey_<Flavor>;
+    using VerificationKey = typename Flavor::VerificationKey;
+
+  protected:
+    static void SetUpTestSuite() { bb::srs::init_crs_factory("../srs_db/ignition"); }
+};
+
+using FlavorTypes = testing::Types<UltraFlavor, UltraKeccakFlavor, MegaFlavor>;
+TYPED_TEST_SUITE(FlavorSerializationTests, FlavorTypes);
+
+// Test msgpack serialization/deserialization of verification keys
+TYPED_TEST(FlavorSerializationTests, VerificationKeySerialization)
+{
+    using Builder = typename TestFixture::Builder;
+    using DeciderProvingKey = typename TestFixture::DeciderProvingKey;
+    using VerificationKey = typename TestFixture::VerificationKey;
+
+    Builder builder;
+
+    // Add some arbitrary arithmetic gates that utilize public inputs
+    MockCircuits::add_arithmetic_gates_with_public_inputs(builder, /*num_gates=*/100);
+
+    auto proving_key = std::make_shared<DeciderProvingKey>(builder);
+    VerificationKey original_vkey{ proving_key->proving_key };
+
+    // Set the pcs ptr to null since this will not be reconstructed correctly from buffer
+    original_vkey.pcs_verification_key = nullptr;
+
+    // Populate some non-zero values in the databus_propagation_data to ensure its being handled
+    if constexpr (IsMegaBuilder<Builder>) {
+        original_vkey.databus_propagation_data.contains_app_return_data_commitment = 1;
+        original_vkey.databus_propagation_data.contains_kernel_return_data_commitment = 1;
+        original_vkey.databus_propagation_data.app_return_data_public_input_idx = 2;
+        original_vkey.databus_propagation_data.kernel_return_data_public_input_idx = 4;
+        original_vkey.databus_propagation_data.is_kernel = 1;
+    }
+
+    // Serialize and deserialize the verification key
+    std::vector<uint8_t> vkey_buffer = to_buffer(original_vkey);
+    VerificationKey deserialized_vkey = from_buffer<VerificationKey>(vkey_buffer);
+
+    // Ensure the original is equal to the reconstructed
+    EXPECT_EQ(original_vkey, deserialized_vkey);
+}