feat: ProverPolynomials owns its memory (AztecProtocol#3560)

Makes ProverPolynomials no longer use a span
- Polynomial class already used shared memory, now allowing shifted
polynomials to be represented by the same class
- Fixes AztecProtocol#2213 by
using ranges defined in entity classes
- For safety, deletes the full-copy assignment and constructor of
ProverPolynomials, so that we correctly std::move it around. This can
possibly be removed, but wanted to catch any now-different-semantics
usage and avoid extra copies
- Makes explicit share() calls wherever semantics were to share before
due to std::span usage. shifted() is similar, shares underlying memory,
shifted
- Remove cruft needed with a separate polynomial storages variable, some
might remain

---------

Co-authored-by: ludamad <adam@aztecprotocol.com>

Author: ludamad

barretenberg/cpp/src/barretenberg/benchmark/honk_bench/ultra_honk_rounds.bench.cpp

@@ -45,7 +45,7 @@ BBERG_PROFILE static void test_round(State& state, size_t index) noexcept
     for (auto _ : state) {
         state.PauseTiming();
         honk::UltraComposer composer;
-        // TODO: https://github.com/AztecProtocol/barretenberg/issues/761 benchmark both sparse and dense circuits
+        // TODO(https://github.com/AztecProtocol/barretenberg/issues/761) benchmark both sparse and dense circuits
         honk::UltraProver prover = bench_utils::get_prover(
             composer, &bench_utils::generate_ecdsa_verification_test_circuit<UltraCircuitBuilder>, 10);
         test_round_inner(state, prover, index);

barretenberg/cpp/src/barretenberg/commitment_schemes/gemini/gemini.cpp

@@ -77,7 +77,7 @@ std::vector<typename barretenberg::Polynomial<typename Curve::ScalarField>> Gemi
     Polynomial& batched_G = gemini_polynomials.emplace_back(std::move(batched_to_be_shifted));
     constexpr size_t offset_to_folded = 2; // Offset because of F an G
     // A₀(X) = F(X) + G↺(X) = F(X) + G(X)/X.
-    Polynomial A_0(batched_F);
+    Polynomial A_0 = batched_F;
     A_0 += batched_G.shifted();
 
     // Allocate everything before parallel computation

barretenberg/cpp/src/barretenberg/commitment_schemes/kzg/kzg.hpp

@@ -33,7 +33,7 @@ template <typename Curve> class KZG {
                                       const Polynomial& polynomial,
                                       const std::shared_ptr<BaseTranscript>& prover_trancript)
     {
-        Polynomial quotient(polynomial);
+        Polynomial quotient = polynomial;
         quotient[0] -= opening_pair.evaluation;
         // Computes the coefficients for the quotient polynomial q(X) = (p(X) - v) / (X - r) through an FFT
         quotient.factor_roots(opening_pair.challenge);

barretenberg/cpp/src/barretenberg/commitment_schemes/shplonk/shplonk.test.cpp

@@ -44,7 +44,7 @@ TYPED_TEST(ShplonkTest, ShplonkSimple)
 
     // Aggregate polynomials and their opening pairs
     std::vector<OpeningPair> opening_pairs = { { r1, eval1 }, { r2, eval2 } };
-    std::vector<Polynomial> polynomials = { poly1, poly2 };
+    std::vector<Polynomial> polynomials = { poly1.share(), poly2.share() };
 
     // Execute the shplonk prover functionality
     const Fr nu_challenge = prover_transcript->get_challenge("Shplonk:nu");

barretenberg/cpp/src/barretenberg/commitment_schemes/zeromorph/zeromorph.hpp

@@ -1,4 +1,5 @@
 #pragma once
+#include "barretenberg/commitment_schemes/commitment_key.hpp"
 #include "barretenberg/common/ref_vector.hpp"
 #include "barretenberg/common/zip_view.hpp"
 #include "barretenberg/polynomials/polynomial.hpp"
@@ -34,8 +35,8 @@ template <typename Curve> class ZeroMorphProver_ {
     using Commitment = typename Curve::AffineElement;
     using Polynomial = barretenberg::Polynomial<FF>;
 
-    // TODO(#742): Set this N_max to be the number of G1 elements in the mocked zeromorph SRS once it's in place. (Then,
-    // eventually, set it based on the real SRS). For now we set it to be large but more or less arbitrary.
+    // TODO(#742): Set this N_max to be the number of G1 elements in the mocked zeromorph SRS once it's in place.
+    // (Then, eventually, set it based on the real SRS). For now we set it to be large but more or less arbitrary.
     static const size_t N_max = 1 << 22;
 
   public:
@@ -59,7 +60,7 @@ template <typename Curve> class ZeroMorphProver_ {
      * @param u_challenge Multivariate challenge u = (u_0, ..., u_{d-1})
      * @return std::vector<Polynomial> The quotients q_k
      */
-    static std::vector<Polynomial> compute_multilinear_quotients(Polynomial polynomial, std::span<FF> u_challenge)
+    static std::vector<Polynomial> compute_multilinear_quotients(Polynomial polynomial, std::span<const FF> u_challenge)
     {
         size_t log_N = numeric::get_msb(polynomial.size());
         // The size of the multilinear challenge must equal the log of the polynomial size
@@ -74,12 +75,12 @@ template <typename Curve> class ZeroMorphProver_ {
 
         // Compute the coefficients of q_{n-1}
         size_t size_q = 1 << (log_N - 1);
-        Polynomial q = Polynomial(size_q);
+        Polynomial q{ size_q };
         for (size_t l = 0; l < size_q; ++l) {
             q[l] = polynomial[size_q + l] - polynomial[l];
         }
 
-        quotients[log_N - 1] = q;
+        quotients[log_N - 1] = q.share();
 
         std::vector<FF> f_k;
         f_k.resize(size_q);
@@ -94,13 +95,13 @@ template <typename Curve> class ZeroMorphProver_ {
             }
 
             size_q = size_q / 2;
-            q = Polynomial(size_q);
+            q = Polynomial{ size_q };
 
             for (size_t l = 0; l < size_q; ++l) {
                 q[l] = f_k[size_q + l] - f_k[l];
             }
 
-            quotients[log_N - k - 1] = q;
+            quotients[log_N - k - 1] = q.share();
             g = f_k;
         }
 
@@ -206,7 +207,7 @@ template <typename Curve> class ZeroMorphProver_ {
         Polynomial& g_batched,
         std::vector<Polynomial>& quotients,
         FF v_evaluation,
-        std::span<FF> u_challenge,
+        std::span<const FF> u_challenge,
         FF x_challenge,
         std::vector<Polynomial> concatenation_groups_batched = {})
     {
@@ -305,8 +306,8 @@ template <typename Curve> class ZeroMorphProver_ {
     }
 
     /**
-     * @brief Prove a set of multilinear evaluation claims for unshifted polynomials f_i and to-be-shifted polynomials
-     * g_i
+     * @brief Prove a set of multilinear evaluation claims for unshifted polynomials f_i and to-be-shifted
+     * polynomials g_i
      *
      * @param f_polynomials Unshifted polynomials
      * @param g_polynomials To-be-shifted polynomials (of which the shifts h_i were evaluated by sumcheck)
@@ -315,23 +316,22 @@ template <typename Curve> class ZeroMorphProver_ {
      * @param commitment_key
      * @param transcript
      */
-    static void prove(const auto& f_polynomials,
-                      const auto& g_polynomials,
-                      auto&& f_evaluations,
-                      auto&& g_shift_evaluations,
-                      auto& multilinear_challenge,
-                      auto& commitment_key,
-                      auto transcript,
-                      const std::vector<std::span<FF>>& concatenated_polynomials = {},
+    static void prove(const std::vector<Polynomial>& f_polynomials,
+                      const std::vector<Polynomial>& g_polynomials,
+                      const std::vector<FF>& f_evaluations,
+                      const std::vector<FF>& g_shift_evaluations,
+                      const std::vector<FF>& multilinear_challenge,
+                      const std::shared_ptr<CommitmentKey<Curve>>& commitment_key,
+                      const std::shared_ptr<BaseTranscript>& transcript,
+                      const std::vector<Polynomial>& concatenated_polynomials = {},
                       const std::vector<FF>& concatenated_evaluations = {},
-                      // TODO(https://github.com/AztecProtocol/barretenberg/issues/743) remove span
-                      const std::vector<RefVector<std::span<FF>>>& concatenation_groups = {})
+                      const std::vector<RefVector<Polynomial>>& concatenation_groups = {})
     {
         // Generate batching challenge \rho and powers 1,...,\rho^{m-1}
         const FF rho = transcript->get_challenge("rho");
 
         // Extract multilinear challenge u and claimed multilinear evaluations from Sumcheck output
-        std::span<FF> u_challenge = multilinear_challenge;
+        std::span<const FF> u_challenge = multilinear_challenge;
         size_t log_N = u_challenge.size();
         size_t N = 1 << log_N;
 
@@ -341,16 +341,16 @@ template <typename Curve> class ZeroMorphProver_ {
         // v = sum_{i=0}^{m-1}\rho^i*f_i(u) + sum_{i=0}^{l-1}\rho^{m+i}*h_i(u).
         // Note: g_batched is formed from the to-be-shifted polynomials, but the batched evaluation incorporates the
         // evaluations produced by sumcheck of h_i = g_i_shifted.
-        auto batched_evaluation = FF(0);
+        FF batched_evaluation{ 0 };
         Polynomial f_batched(N); // batched unshifted polynomials
-        FF batching_scalar = FF(1);
+        FF batching_scalar{ 1 };
         for (auto [f_poly, f_eval] : zip_view(f_polynomials, f_evaluations)) {
             f_batched.add_scaled(f_poly, batching_scalar);
             batched_evaluation += batching_scalar * f_eval;
             batching_scalar *= rho;
         }
 
-        Polynomial g_batched(N); // batched to-be-shifted polynomials
+        Polynomial g_batched{ N }; // batched to-be-shifted polynomials
         for (auto [g_poly, g_shift_eval] : zip_view(g_polynomials, g_shift_evaluations)) {
             g_batched.add_scaled(g_poly, batching_scalar);
             batched_evaluation += batching_scalar * g_shift_eval;
@@ -360,7 +360,6 @@ template <typename Curve> class ZeroMorphProver_ {
         size_t num_groups = concatenation_groups.size();
         size_t num_chunks_per_group = concatenation_groups.empty() ? 0 : concatenation_groups[0].size();
         // Concatenated polynomials
-        // std::vector<Polynomial> concatenated_polynomials;
         Polynomial concatenated_batched(N);
 
         // construct concatention_groups_batched
@@ -381,7 +380,7 @@ template <typename Curve> class ZeroMorphProver_ {
 
         // Compute the full batched polynomial f = f_batched + g_batched.shifted() = f_batched + h_batched. This is the
         // polynomial for which we compute the quotients q_k and prove f(u) = v_batched.
-        auto f_polynomial = f_batched;
+        Polynomial f_polynomial = f_batched;
         f_polynomial += g_batched.shifted();
         f_polynomial += concatenated_batched;
 
@@ -625,8 +624,8 @@ template <typename Curve> class ZeroMorphVerifier_ {
     }
 
     /**
-     * @brief Verify a set of multilinear evaluation claims for unshifted polynomials f_i and to-be-shifted polynomials
-     * g_i
+     * @brief Verify a set of multilinear evaluation claims for unshifted polynomials f_i and to-be-shifted
+     * polynomials g_i
      *
      * @param commitments Commitments to polynomials f_i and g_i (unshifted and to-be-shifted)
      * @param claimed_evaluations Claimed evaluations v_i = f_i(u) and w_i = h_i(u) = g_i_shifted(u)
@@ -710,4 +709,4 @@ template <typename Curve> class ZeroMorphVerifier_ {
     }
 };
 
-} // namespace proof_system::honk::pcs::zeromorph
+} // namespace proof_system::honk::pcs::zeromorph

barretenberg/cpp/src/barretenberg/commitment_schemes/zeromorph/zeromorph.test.cpp

@@ -37,12 +37,10 @@ template <class Curve> class ZeroMorphTest : public CommitmentTest<Curve> {
      */
     bool execute_zeromorph_protocol(size_t NUM_UNSHIFTED, size_t NUM_SHIFTED)
     {
-        bool verified = false;
-
         size_t N = 16;
         size_t log_N = numeric::get_msb(N);
 
-        auto u_challenge = this->random_evaluation_point(log_N);
+        std::vector<Fr> u_challenge = this->random_evaluation_point(log_N);
 
         // Construct some random multilinear polynomials f_i and their evaluations v_i = f_i(u)
         std::vector<Polynomial> f_polynomials; // unshifted polynomials
@@ -94,7 +92,7 @@ template <class Curve> class ZeroMorphTest : public CommitmentTest<Curve> {
         auto pairing_points = ZeroMorphVerifier::verify(
             f_commitments, g_commitments, v_evaluations, w_evaluations, u_challenge, verifier_transcript);
 
-        verified = this->vk()->pairing_check(pairing_points[0], pairing_points[1]);
+        bool verified = this->vk()->pairing_check(pairing_points[0], pairing_points[1]);
 
         // The prover and verifier manifests should agree
         EXPECT_EQ(prover_transcript->get_manifest(), verifier_transcript->get_manifest());
@@ -225,17 +223,6 @@ template <class Curve> class ZeroMorphWithConcatenationTest : public CommitmentT
         // Initialize an empty BaseTranscript
         auto prover_transcript = BaseTranscript::prover_init_empty();
 
-        std::vector<std::span<Fr>> concatenated_polynomials_views;
-        for (auto& poly : concatenated_polynomials) {
-            concatenated_polynomials_views.emplace_back(poly);
-        }
-
-        std::vector<std::vector<std::span<Fr>>> concatenation_groups_views(concatenation_groups.size());
-        for (auto [group_of_polys, group_of_views] : zip_view(concatenation_groups, concatenation_groups_views)) {
-            for (auto& poly : group_of_polys) {
-                group_of_views.emplace_back(poly);
-            }
-        }
         // Execute Prover protocol
         ZeroMorphProver::prove(f_polynomials, // unshifted
                                g_polynomials, // to-be-shifted
@@ -244,9 +231,9 @@ template <class Curve> class ZeroMorphWithConcatenationTest : public CommitmentT
                                u_challenge,
                                this->commitment_key,
                                prover_transcript,
-                               concatenated_polynomials_views,
+                               concatenated_polynomials,
                                c_evaluations,
-                               to_vector_of_ref_vectors(concatenation_groups_views));
+                               to_vector_of_ref_vectors(concatenation_groups));
 
         auto verifier_transcript = BaseTranscript::verifier_init_empty(prover_transcript);
 

barretenberg/cpp/src/barretenberg/common/ref_vector.hpp

@@ -139,11 +139,12 @@ template <typename T> RefVector<T> concatenate(const RefVector<T>& ref_vector, c
     return concatenated;
 }
 
+/* @details Ensures a nested vector holds reference objects */
 template <typename T> static std::vector<RefVector<T>> to_vector_of_ref_vectors(std::vector<std::vector<T>>& vec)
 {
     std::vector<RefVector<T>> result;
     for (std::vector<T>& inner : vec) {
         result.push_back(RefVector{ inner });
     }
     return result;
-}
+}

barretenberg/cpp/src/barretenberg/common/std_string.cpp

@@ -0,0 +1,49 @@
+#include "std_string.hpp"
+#include <algorithm>
+#include <cctype>
+#include <iostream>
+#include <locale>
+#include <sstream>
+#include <vector>
+
+namespace barretenberg::detail {
+std::vector<std::string> split(const std::string& str, char delimiter)
+{
+    std::vector<std::string> result;
+    std::istringstream iss(str);
+    std::string token;
+
+    while (std::getline(iss, token, delimiter)) {
+        result.push_back(token);
+    }
+
+    return result;
+}
+
+// trim from start (in place)
+void ltrim(std::string& s)
+{
+    s.erase(s.begin(), std::find_if(s.begin(), s.end(), [](unsigned char ch) { return !std::isspace(ch); }));
+}
+
+// trim from end (in place)
+void rtrim(std::string& s)
+{
+    s.erase(std::find_if(s.rbegin(), s.rend(), [](unsigned char ch) { return !std::isspace(ch); }).base(), s.end());
+}
+
+// trim from both ends (in place)
+void trim(std::string& s)
+{
+    rtrim(s);
+    ltrim(s);
+}
+std::vector<std::string> split_and_trim(const std::string& str, char delimiter)
+{
+    std::vector<std::string> ret = split(str, delimiter);
+    for (std::string& part : ret) {
+        trim(part);
+    }
+    return ret;
+}
+} // namespace barretenberg::detail

barretenberg/cpp/src/barretenberg/common/std_string.hpp

@@ -0,0 +1,15 @@
+#include <string>
+#include <vector>
+
+namespace barretenberg::detail {
+std::vector<std::string> split(const std::string& str, char delimiter);
+// trim from start (in place)
+void ltrim(std::string& s);
+// trim from end (in place)
+void rtrim(std::string& s);
+// trim from both ends (in place)
+void trim(std::string& s);
+
+// Used to extract variables from a macro #__VA_ARGS__
+std::vector<std::string> split_and_trim(const std::string& str, char delimiter);
+} // namespace barretenberg::detail

barretenberg/cpp/src/barretenberg/common/std_vector.hpp

@@ -0,0 +1,25 @@
+#pragma once
+#include <vector>
+
+// TODO(https://github.com/AztecProtocol/barretenberg/issues/794) namespace this once convenient
+/**
+ * @brief Concatenates multiple std::vector objects into a single std::vector.
+ *
+ * @tparam T The type of elements in the std::vector.
+ * @param vectors The std::vector objects to be concatenated.
+ * @return std::vector object containing all elements from the input vectors.
+ */
+template <typename T> std::vector<T> concatenate(const std::vector<T>& vector, const auto&... vectors)
+{
+    std::vector<T> concatenated;
+    // Reserve our final space
+    concatenated.reserve(vector.size() + (vectors.size() + ...));
+
+    auto append = [&](const auto& vec) { std::copy(vec.begin(), vec.end(), std::back_inserter(concatenated)); };
+
+    append(vector);
+    // Unpack and append each std::vector's elements to concatenated
+    (append(vectors), ...);
+
+    return concatenated;
+}