Skip to content

Commit d093266

Browse files
codyguntonledwards2225ludamad0ludamad
authored
feat: bb uses goblin (AztecProtocol#3636)
"vertical slice" of goblin integrating with ACIR and BB. Beginning of goblin ultra honk interface for bb. Can only run a basic test (assert_statement) for now and is not intended for use outside of the CI check that it's still working, for now. - adds bb and bb.js command `prove_and_verify_goblin`. Adds bb.js bindings for goblin prove/verify - modifies ACIR `dsl` folder to be able to take a goblin builder - adds CI calls to bb.js and bb that use `prove_and_verify_goblin` - allowing ability to load grumpkin SRS through memory, needed for bb and (especially) bb.js. This allows an alternate source of points to be used other than the default file-based grumpkin loader, which mostly only works in dev (though revisit: could this work for native bb?) --------- Co-authored-by: ledwards2225 <ledwards2225@users.noreply.github.com> Co-authored-by: ledwards2225 <l.edwards.d@gmail.com> Co-authored-by: ludamad <adam@aztecprotocol.com> Co-authored-by: ludamad <adam.domurad@gmail.com>
1 parent c0a24fb commit d093266

File tree

93 files changed

+1381
-454
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

93 files changed

+1381
-454
lines changed

‎barretenberg/acir_tests/Dockerfile.bb

+2
Original file line numberDiff line numberDiff line change
@@ -10,5 +10,7 @@ COPY . .
1010
# Run every acir test through native bb build prove_then_verify flow.
1111
# This ensures we test independent pk construction through real/garbage witness data paths.
1212
RUN FLOW=prove_then_verify ./run_acir_tests.sh
13+
# TODO(https://github.com/AztecProtocol/barretenberg/issues/811) make this able to run the default test
14+
RUN FLOW=prove_and_verify_goblin ./run_acir_tests.sh assert_statement
1315
# Run 1_mul through native bb build, all_cmds flow, to test all cli args.
1416
RUN VERBOSE=1 FLOW=all_cmds ./run_acir_tests.sh 1_mul

‎barretenberg/acir_tests/Dockerfile.bb.js

+2
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,8 @@ COPY . .
1414
ENV VERBOSE=1
1515
# Run double_verify_proof through bb.js on node to check 512k support.
1616
RUN BIN=../ts/dest/node/main.js FLOW=prove_then_verify ./run_acir_tests.sh double_verify_proof
17+
# TODO(https://github.com/AztecProtocol/barretenberg/issues/811) make this able to run double_verify_proof
18+
RUN BIN=../ts/dest/node/main.js FLOW=prove_and_verify_goblin ./run_acir_tests.sh assert_statement
1719
# Run 1_mul through bb.js build, all_cmds flow, to test all cli args.
1820
RUN BIN=../ts/dest/node/main.js FLOW=all_cmds ./run_acir_tests.sh 1_mul
1921
# Run double_verify_proof through bb.js on chrome testing multi-threaded browser support.

‎barretenberg/acir_tests/flows/prove_and_verify_goblin.sh

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
#!/bin/sh
2+
set -eu
3+
4+
VFLAG=${VERBOSE:+-v}
5+
6+
$BIN prove_and_verify_goblin $VFLAG -c $CRS_PATH -b ./target/acir.gz

‎barretenberg/cpp/.dockerignore

+1
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88

99
# Important srs_db files.
1010
!srs_db/download_ignition.sh
11+
!srs_db/download_grumpkin.sh
1112
!srs_db/ignition/checksums
1213

1314
# Source code.

‎barretenberg/cpp/dockerfiles/Dockerfile.wasm-linux-clang

+1
Original file line numberDiff line numberDiff line change
@@ -11,5 +11,6 @@ RUN ./scripts/strip-wasm.sh
1111
FROM scratch
1212
WORKDIR /usr/src/barretenberg/cpp
1313
COPY . .
14+
COPY --from=builder /usr/src/barretenberg/cpp/srs_db /usr/src/barretenberg/cpp/srs_db
1415
COPY --from=builder /usr/src/barretenberg/cpp/build-wasm/bin/barretenberg.wasm /usr/src/barretenberg/cpp/build-wasm/bin/barretenberg.wasm
1516
COPY --from=builder /usr/src/barretenberg/cpp/build-wasm-threads/bin/barretenberg.wasm /usr/src/barretenberg/cpp/build-wasm-threads/bin/barretenberg.wasm

‎barretenberg/cpp/dockerfiles/Dockerfile.x86_64-linux-clang-assert

+10-9
Original file line numberDiff line numberDiff line change
@@ -3,19 +3,20 @@ FROM alpine:3.17 AS builder
33
RUN apk update \
44
&& apk upgrade \
55
&& apk add --no-cache \
6-
build-base \
7-
clang15 \
8-
cmake \
9-
ninja \
10-
git \
11-
curl \
12-
perl \
13-
clang-extra-tools \
14-
bash
6+
build-base \
7+
clang15 \
8+
cmake \
9+
ninja \
10+
git \
11+
curl \
12+
perl \
13+
clang-extra-tools \
14+
bash
1515
WORKDIR /usr/src/barretenberg/cpp
1616
COPY . .
1717
# Build everything to ensure everything builds. All tests will be run from the result of this build.
1818
RUN ./format.sh check && cmake --preset default -DCMAKE_BUILD_TYPE=RelWithAssert -DCI=ON && cmake --build --preset default
19+
RUN srs_db/download_grumpkin.sh
1920

2021
FROM alpine:3.17
2122
RUN apk update && apk add curl libstdc++

‎barretenberg/cpp/scripts/bb-tests.sh

+2-2
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ TESTS_STR="${TESTS[@]}"
4444
docker run --rm -t $IMAGE_URI /bin/sh -c "\
4545
set -xe; \
4646
cd /usr/src/barretenberg/cpp; \
47-
(cd srs_db && ./download_ignition.sh 1); \
47+
srs_db/download_ignition.sh 1; \
48+
srs_db/download_grumpkin.sh; \
4849
cd build; \
49-
./bin/grumpkin_srs_gen 1048576; \
5050
for BIN in $TESTS_STR; do ./bin/\$BIN; done"

‎barretenberg/cpp/scripts/run_tests

+2-3
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,7 @@ fi
2727

2828
docker run --rm -t $IMAGE_URI /bin/sh -c "\
2929
set -xe; \
30-
cd /usr/src/barretenberg/cpp/srs_db; \
31-
./download_ignition.sh $NUM_TRANSCRIPTS; \
30+
/usr/src/barretenberg/cpp/srs_db/download_ignition.sh $NUM_TRANSCRIPTS; \
31+
/usr/src/barretenberg/cpp/srs_db/download_grumpkin.sh; \
3232
cd /usr/src/barretenberg/cpp/build; \
33-
./bin/grumpkin_srs_gen 1048576; \
3433
for BIN in $TESTS; do ./bin/\$BIN $@; done"

‎barretenberg/cpp/src/CMakeLists.txt

+20-35
Original file line numberDiff line numberDiff line change
@@ -96,9 +96,9 @@ include(GNUInstallDirs)
9696
# libbarretenberg + libwasi = a wasi "reactor" that implements it's own env (e.g. logstr), e.g. barretenberg.wasm.
9797
# libbarretenberg + env = a wasi "command" that expects a full wasi runtime (e.g. wasmtime), e.g. test binaries.
9898
message(STATUS "Compiling all-in-one barretenberg archive")
99-
add_library(
100-
barretenberg
101-
STATIC
99+
100+
set(BARRETENBERG_TARGET_OBJECTS
101+
$<TARGET_OBJECTS:commitment_schemes_objects>
102102
$<TARGET_OBJECTS:common_objects>
103103
$<TARGET_OBJECTS:crypto_aes128_objects>
104104
$<TARGET_OBJECTS:crypto_blake2s_objects>
@@ -111,12 +111,17 @@ add_library(
111111
$<TARGET_OBJECTS:crypto_sha256_objects>
112112
$<TARGET_OBJECTS:dsl_objects>
113113
$<TARGET_OBJECTS:ecc_objects>
114+
$<TARGET_OBJECTS:eccvm_objects>
114115
$<TARGET_OBJECTS:examples_objects>
116+
$<TARGET_OBJECTS:flavor_objects>
117+
$<TARGET_OBJECTS:goblin_objects>
115118
$<TARGET_OBJECTS:honk_objects>
116119
$<TARGET_OBJECTS:numeric_objects>
117120
$<TARGET_OBJECTS:plonk_objects>
118121
$<TARGET_OBJECTS:polynomials_objects>
119122
$<TARGET_OBJECTS:proof_system_objects>
123+
$<TARGET_OBJECTS:protogalaxy_objects>
124+
$<TARGET_OBJECTS:relations_objects>
120125
$<TARGET_OBJECTS:srs_objects>
121126
$<TARGET_OBJECTS:stdlib_aes128_objects>
122127
$<TARGET_OBJECTS:stdlib_blake2s_objects>
@@ -126,9 +131,18 @@ add_library(
126131
$<TARGET_OBJECTS:stdlib_pedersen_commitment_objects>
127132
$<TARGET_OBJECTS:stdlib_pedersen_hash_objects>
128133
$<TARGET_OBJECTS:stdlib_primitives_objects>
134+
$<TARGET_OBJECTS:stdlib_recursion_objects>
129135
$<TARGET_OBJECTS:stdlib_schnorr_objects>
130136
$<TARGET_OBJECTS:stdlib_sha256_objects>
137+
$<TARGET_OBJECTS:sumcheck_objects>
131138
$<TARGET_OBJECTS:transcript_objects>
139+
$<TARGET_OBJECTS:translator_vm_objects>
140+
$<TARGET_OBJECTS:ultra_honk_objects>)
141+
142+
add_library(
143+
barretenberg
144+
STATIC
145+
${BARRETENBERG_TARGET_OBJECTS}
132146
)
133147

134148
if(WASM)
@@ -144,36 +158,7 @@ if(WASM)
144158
# to implement the functions in env.
145159
add_executable(
146160
barretenberg.wasm
147-
$<TARGET_OBJECTS:common_objects>
148-
$<TARGET_OBJECTS:crypto_aes128_objects>
149-
$<TARGET_OBJECTS:crypto_blake2s_objects>
150-
$<TARGET_OBJECTS:crypto_blake3s_objects>
151-
$<TARGET_OBJECTS:crypto_ecdsa_objects>
152-
$<TARGET_OBJECTS:crypto_keccak_objects>
153-
$<TARGET_OBJECTS:crypto_pedersen_commitment_objects>
154-
$<TARGET_OBJECTS:crypto_pedersen_hash_objects>
155-
$<TARGET_OBJECTS:crypto_schnorr_objects>
156-
$<TARGET_OBJECTS:crypto_sha256_objects>
157-
$<TARGET_OBJECTS:dsl_objects>
158-
$<TARGET_OBJECTS:ecc_objects>
159-
$<TARGET_OBJECTS:examples_objects>
160-
$<TARGET_OBJECTS:honk_objects>
161-
$<TARGET_OBJECTS:numeric_objects>
162-
$<TARGET_OBJECTS:plonk_objects>
163-
$<TARGET_OBJECTS:polynomials_objects>
164-
$<TARGET_OBJECTS:proof_system_objects>
165-
$<TARGET_OBJECTS:srs_objects>
166-
$<TARGET_OBJECTS:stdlib_aes128_objects>
167-
$<TARGET_OBJECTS:stdlib_blake2s_objects>
168-
$<TARGET_OBJECTS:stdlib_blake3s_objects>
169-
$<TARGET_OBJECTS:stdlib_keccak_objects>
170-
$<TARGET_OBJECTS:stdlib_merkle_tree_objects>
171-
$<TARGET_OBJECTS:stdlib_pedersen_commitment_objects>
172-
$<TARGET_OBJECTS:stdlib_pedersen_hash_objects>
173-
$<TARGET_OBJECTS:stdlib_primitives_objects>
174-
$<TARGET_OBJECTS:stdlib_schnorr_objects>
175-
$<TARGET_OBJECTS:stdlib_sha256_objects>
176-
$<TARGET_OBJECTS:transcript_objects>
161+
${BARRETENBERG_TARGET_OBJECTS}
177162
$<TARGET_OBJECTS:wasi_objects>
178163
)
179164

@@ -194,12 +179,12 @@ if(WASM)
194179
target_link_options(
195180
barretenberg.wasm
196181
PRIVATE
197-
-nostartfiles -Wl,--no-entry,--export-dynamic,--allow-undefined
182+
-nostartfiles -Wl,--no-entry,--export-dynamic
198183
)
199184

200185
target_link_options(
201186
acvm_backend.wasm
202187
PRIVATE
203-
-nostartfiles -Wl,--no-entry,--export-dynamic,--allow-undefined
188+
-nostartfiles -Wl,--no-entry,--export-dynamic
204189
)
205190
endif()

‎barretenberg/cpp/src/barretenberg/bb/CMakeLists.txt

+2
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@ if (NOT(FUZZING))
22
add_executable(
33
bb
44
main.cpp
5+
get_bn254_crs.cpp
6+
get_grumpkin_crs.cpp
57
)
68

79
target_link_libraries(

‎barretenberg/cpp/src/barretenberg/bb/get_crs.hpp ‎barretenberg/cpp/src/barretenberg/bb/get_bn254_crs.cpp

+14-20
Original file line numberDiff line numberDiff line change
@@ -1,33 +1,27 @@
1-
#pragma once
2-
#include "exec_pipe.hpp"
3-
#include "file_io.hpp"
4-
#include "log.hpp"
5-
#include <barretenberg/ecc/curves/bn254/g1.hpp>
6-
#include <barretenberg/srs/io.hpp>
7-
#include <filesystem>
8-
#include <fstream>
9-
#include <ios>
1+
#include "get_bn254_crs.hpp"
102

113
// Gets the transcript URL from the BARRETENBERG_TRANSCRIPT_URL environment variable, if set.
124
// Otherwise returns the default URL.
13-
inline std::string getTranscriptURL()
5+
namespace {
6+
std::string get_bn254_transcript_url()
147
{
158
const char* ENV_VAR_NAME = "BARRETENBERG_TRANSCRIPT_URL";
169
const std::string DEFAULT_URL = "https://aztec-ignition.s3.amazonaws.com/MAIN%20IGNITION/monomial/transcript00.dat";
1710

1811
const char* env_url = std::getenv(ENV_VAR_NAME);
1912

20-
auto environment_variable_exists = (env_url && *env_url);
13+
auto environment_variable_exists = ((env_url != nullptr) && *env_url);
2114

2215
return environment_variable_exists ? std::string(env_url) : DEFAULT_URL;
2316
}
17+
} // namespace
2418

25-
inline std::vector<uint8_t> download_g1_data(size_t num_points)
19+
std::vector<uint8_t> download_bn254_g1_data(size_t num_points)
2620
{
2721
size_t g1_start = 28;
2822
size_t g1_end = g1_start + num_points * 64 - 1;
2923

30-
std::string url = getTranscriptURL();
24+
std::string url = get_bn254_transcript_url();
3125

3226
std::string command =
3327
"curl -s -H \"Range: bytes=" + std::to_string(g1_start) + "-" + std::to_string(g1_end) + "\" '" + url + "'";
@@ -38,23 +32,23 @@ inline std::vector<uint8_t> download_g1_data(size_t num_points)
3832
throw std::runtime_error("Failed to download g1 data.");
3933
}
4034

41-
return exec_pipe(command);
35+
return data;
4236
}
4337

44-
inline std::vector<uint8_t> download_g2_data()
38+
std::vector<uint8_t> download_bn254_g2_data()
4539
{
4640
size_t g2_start = 28 + 5040001 * 64;
4741
size_t g2_end = g2_start + 128 - 1;
4842

49-
std::string url = getTranscriptURL();
43+
std::string url = get_bn254_transcript_url();
5044

5145
std::string command =
5246
"curl -s -H \"Range: bytes=" + std::to_string(g2_start) + "-" + std::to_string(g2_end) + "\" '" + url + "'";
5347

5448
return exec_pipe(command);
5549
}
5650

57-
inline std::vector<barretenberg::g1::affine_element> get_g1_data(const std::filesystem::path& path, size_t num_points)
51+
std::vector<barretenberg::g1::affine_element> get_bn254_g1_data(const std::filesystem::path& path, size_t num_points)
5852
{
5953
std::filesystem::create_directories(path);
6054
std::ifstream size_file(path / "size");
@@ -74,7 +68,7 @@ inline std::vector<barretenberg::g1::affine_element> get_g1_data(const std::file
7468
}
7569

7670
vinfo("downloading crs...");
77-
auto data = download_g1_data(num_points);
71+
auto data = download_bn254_g1_data(num_points);
7872
write_file(path / "g1.dat", data);
7973

8074
std::ofstream new_size_file(path / "size");
@@ -90,7 +84,7 @@ inline std::vector<barretenberg::g1::affine_element> get_g1_data(const std::file
9084
return points;
9185
}
9286

93-
inline barretenberg::g2::affine_element get_g2_data(const std::filesystem::path& path)
87+
barretenberg::g2::affine_element get_bn254_g2_data(const std::filesystem::path& path)
9488
{
9589
std::filesystem::create_directories(path);
9690

@@ -100,7 +94,7 @@ inline barretenberg::g2::affine_element get_g2_data(const std::filesystem::path&
10094
barretenberg::srs::IO<curve::BN254>::read_affine_elements_from_buffer(&g2_point, (char*)data.data(), 128);
10195
return g2_point;
10296
} catch (std::exception&) {
103-
auto data = download_g2_data();
97+
auto data = download_bn254_g2_data();
10498
write_file(path / "g2.dat", data);
10599
barretenberg::g2::affine_element g2_point;
106100
barretenberg::srs::IO<curve::BN254>::read_affine_elements_from_buffer(&g2_point, (char*)data.data(), 128);

‎barretenberg/cpp/src/barretenberg/bb/get_bn254_crs.hpp

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
#pragma once
2+
#include "exec_pipe.hpp"
3+
#include "file_io.hpp"
4+
#include "log.hpp"
5+
#include <barretenberg/ecc/curves/bn254/g1.hpp>
6+
#include <barretenberg/srs/io.hpp>
7+
#include <filesystem>
8+
#include <fstream>
9+
#include <ios>
10+
11+
std::vector<barretenberg::g1::affine_element> get_bn254_g1_data(const std::filesystem::path& path, size_t num_points);
12+
barretenberg::g2::affine_element get_bn254_g2_data(const std::filesystem::path& path);

‎barretenberg/cpp/src/barretenberg/bb/get_grumpkin_crs.cpp

+72
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
#include "get_grumpkin_crs.hpp"
2+
3+
// Gets the transcript URL from the BARRETENBERG_GRUMPKIN_TRANSCRIPT_URL environment variable, if set.
4+
// Otherwise returns the default URL.
5+
namespace {
6+
std::string get_grumpkin_transcript_url()
7+
{
8+
const char* ENV_VAR_NAME = "BARRETENBERG_GRUMPKIN_TRANSCRIPT_URL";
9+
const std::string DEFAULT_URL = "https://aztec-ignition.s3.amazonaws.com/TEST%20GRUMPKIN/monomial/transcript00.dat";
10+
11+
const char* env_url = std::getenv(ENV_VAR_NAME);
12+
13+
auto environment_variable_exists = ((env_url != nullptr) && *env_url);
14+
15+
return environment_variable_exists ? env_url : DEFAULT_URL;
16+
}
17+
} // namespace
18+
19+
std::vector<uint8_t> download_grumpkin_g1_data(size_t num_points)
20+
{
21+
size_t g1_start = 28;
22+
size_t g1_end = g1_start + num_points * 64 - 1;
23+
24+
std::string url = get_grumpkin_transcript_url();
25+
26+
std::string command =
27+
"curl -s -H \"Range: bytes=" + std::to_string(g1_start) + "-" + std::to_string(g1_end) + "\" '" + url + "'";
28+
29+
auto data = exec_pipe(command);
30+
// Header + num_points * sizeof point.
31+
if (data.size() < g1_end - g1_start) {
32+
throw std::runtime_error("Failed to download grumpkin g1 data.");
33+
}
34+
35+
return data;
36+
}
37+
38+
std::vector<curve::Grumpkin::AffineElement> get_grumpkin_g1_data(const std::filesystem::path& path, size_t num_points)
39+
{
40+
std::filesystem::create_directories(path);
41+
std::ifstream size_file(path / "grumpkin_size");
42+
size_t size = 0;
43+
if (size_file) {
44+
size_file >> size;
45+
size_file.close();
46+
}
47+
if (size >= num_points) {
48+
vinfo("using cached crs at: ", path);
49+
auto data = read_file(path / "grumpkin_g1.dat", 28 + num_points * 64);
50+
auto points = std::vector<curve::Grumpkin::AffineElement>(num_points);
51+
auto size_of_points_in_bytes = num_points * 64;
52+
barretenberg::srs::IO<curve::Grumpkin>::read_affine_elements_from_buffer(
53+
points.data(), (char*)data.data(), size_of_points_in_bytes);
54+
return points;
55+
}
56+
57+
vinfo("downloading grumpkin crs...");
58+
auto data = download_grumpkin_g1_data(num_points);
59+
write_file(path / "grumpkin_g1.dat", data);
60+
61+
std::ofstream new_size_file(path / "grumpkin_size");
62+
if (!new_size_file) {
63+
throw std::runtime_error("Failed to open size file for writing");
64+
}
65+
new_size_file << num_points;
66+
new_size_file.close();
67+
68+
auto points = std::vector<curve::Grumpkin::AffineElement>(num_points);
69+
barretenberg::srs::IO<curve::Grumpkin>::read_affine_elements_from_buffer(
70+
points.data(), (char*)data.data(), data.size());
71+
return points;
72+
}

‎barretenberg/cpp/src/barretenberg/bb/get_grumpkin_crs.hpp

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
#pragma once
2+
#include "exec_pipe.hpp"
3+
#include "file_io.hpp"
4+
#include "log.hpp"
5+
#include <barretenberg/ecc/curves/bn254/g1.hpp>
6+
#include <barretenberg/srs/io.hpp>
7+
#include <filesystem>
8+
#include <fstream>
9+
#include <ios>
10+
11+
std::vector<curve::Grumpkin::AffineElement> get_grumpkin_g1_data(const std::filesystem::path& path, size_t num_points);

0 commit comments

Comments
 (0)