fix(bb.js): remove size metadata from UH proof (#12775)

Fixes #11829 

- Remove first 4 bytes from proof (metadata - length of "proof + PI" in
fields) returned from `UltraHonkBackend.generateProof()`
- `proof` returned is now 14080 bytes (440 fields) and can be directly
verified in solidity


Note: `proof` output from bb CLI also includes the size metadata in the
first 4 bytes. This should go away with #11024

---------

Co-authored-by: Tom French <15848336+TomAFrench@users.noreply.github.com>

Author: saleel

barretenberg/acir_tests/bbjs-test/src/index.ts

@@ -7,6 +7,10 @@ import assert from "assert";
 createDebug.enable("*");
 const debug = createDebug("bbjs-test");
 
+const UH_PROOF_FIELDS_LENGTH = 440;
+const BYTES_PER_FIELD = 32;
+const UH_PROOF_LENGTH_IN_BYTES = UH_PROOF_FIELDS_LENGTH * BYTES_PER_FIELD;
+
 const proofPath = (dir: string) => path.join(dir, "proof");
 const publicInputsPath = (dir: string) => path.join(dir, "public-inputs");
 const vkeyPath = (dir: string) => path.join(dir, "vk");
@@ -33,6 +37,7 @@ async function generateProof({
 
   const witness = await fs.readFile(witnessPath);
   const proof = await backend.generateProof(new Uint8Array(witness), { keccak: (oracleHash === "keccak") });
+  assert(proof.proof.length === UH_PROOF_LENGTH_IN_BYTES, `Unexpected proof length ${proof.proof.length} for ${bytecodePath}`);
 
   await fs.writeFile(proofPath(outputDirectory), Buffer.from(proof.proof));
   debug("Proof written to " + proofPath(outputDirectory));
@@ -53,6 +58,8 @@ async function verifyProof({ directory }: { directory: string }) {
   const verifier = new BarretenbergVerifier();
 
   const proof = await fs.readFile(proofPath(directory));
+  assert(proof.length === UH_PROOF_LENGTH_IN_BYTES, `Unexpected proof length ${proof.length}`);
+
   const publicInputs = JSON.parse(await fs.readFile(publicInputsPath(directory), "utf8"));
   const vkey = await fs.readFile(vkeyPath(directory));
 

barretenberg/acir_tests/flows/bb_prove_bbjs_verify.sh

@@ -32,17 +32,16 @@ $BIN write_vk \
 # bb.js expects proof and public inputs to be separate files, so we need to split them
 # this will not be needed after #11024
 
-# Save public inputs as separate file (first NUM_PUBLIC_INPUTS fields of proof_fields.json)
+# Save public inputs as a separate file (first NUM_PUBLIC_INPUTS fields of proof_fields.json)
 PROOF_FIELDS_LENGTH=$(jq 'length' $output_dir/proof_fields.json)
 UH_PROOF_FIELDS_LENGTH=440
 NUM_PUBLIC_INPUTS=$((PROOF_FIELDS_LENGTH - UH_PROOF_FIELDS_LENGTH))
 jq ".[:$NUM_PUBLIC_INPUTS]" $output_dir/proof_fields.json > $output_dir/public-inputs
 
-# Remove NUM_PUBLIC_INPUTS*32 bytes from the proof
+# Remove public inputs from the proof (first NUM_PUBLIC_INPUTS*32 bytes)
+# Also remove the first 4 bytes, which is the proof length in fields
 proof_hex=$(cat $output_dir/proof | xxd -p)
-proof_start=${proof_hex:0:8}
-proof_end=${proof_hex:$((8 + NUM_PUBLIC_INPUTS * 64))}
-echo -n $proof_start$proof_end | xxd -r -p > $output_dir/proof
+echo -n ${proof_hex:$((NUM_PUBLIC_INPUTS * 64 + 8))} | xxd -r -p > $output_dir/proof
 
 # Verify the proof with bb.js classes
 node ../../bbjs-test verify \

barretenberg/acir_tests/flows/bbjs_prove_bb_verify.sh

@@ -24,21 +24,22 @@ node ../../bbjs-test prove \
 # Join the proof and public inputs to a single file
 # this will not be needed after #11024
 
+NUM_PUBLIC_INPUTS=$(cat $output_dir/public-inputs | jq 'length')
+UH_PROOF_FIELDS_LENGTH=440
+PROOF_AND_PI_LENGTH_IN_FIELDS=$((NUM_PUBLIC_INPUTS + UH_PROOF_FIELDS_LENGTH))
+# First 4 bytes is PROOF_AND_PI_LENGTH_IN_FIELDS
+proof_header=$(printf "%08x" $PROOF_AND_PI_LENGTH_IN_FIELDS)
+
 proof_bytes=$(cat $output_dir/proof | xxd -p)
 public_inputs=$(cat $output_dir/public-inputs | jq -r '.[]')
-proof_start=${proof_bytes:0:8}
-proof_end=${proof_bytes:8}
 
 public_inputs_bytes=""
 for input in $public_inputs; do
   public_inputs_bytes+=$input
 done
 
-# Combine proof start, public inputs, and rest of proof
-echo -n $proof_start$public_inputs_bytes$proof_end | xxd -r -p > $output_dir/proof
-
-# Print the length of the proof file in bytes
-ls -l $output_dir/proof | awk '{print $5}'
+# Combine proof header, public inputs, and the proof to a single file
+echo -n $proof_header$public_inputs_bytes$proof_bytes | xxd -r -p > $output_dir/proof
 
 # Verify the proof with bb cli
 $BIN verify \

barretenberg/acir_tests/sol-test/src/index.js

@@ -248,12 +248,11 @@ try {
     );
 
     proofStr = proofStr.substring(32 * 2 * numPublicInputs); // Remove the publicInput bytes from the proof
-  }
 
-  // Honk proof have field length as the first 4 bytes
-  // This should go away in the future
-  if (testingHonk) {
-    proofStr = proofStr.substring(8);
+    // Honk proof from the CLI have field length as the first 4 bytes. This should go away in the future
+    if (testingHonk) {
+      proofStr = proofStr.substring(8);
+    }
   }
 
   proofStr = "0x" + proofStr;

barretenberg/ts/src/barretenberg/backend.ts

@@ -8,6 +8,7 @@ import {
   ProofDataForRecursion,
   reconstructHonkProof,
   reconstructUltraPlonkProof,
+  splitHonkProof,
 } from '../proof/index.js';
 
 export class AztecClientBackendError extends Error {
@@ -154,10 +155,6 @@ export class UltraPlonkBackend {
   }
 }
 
-// Buffers are prepended with their size. The size takes 4 bytes.
-const serializedBufferSize = 4;
-const fieldByteSize = 32;
-
 /**
  * Options for the UltraHonkBackend.
  */
@@ -222,22 +219,8 @@ export class UltraHonkBackend {
     // Item at index 1 in VK is the number of public inputs
     const numPublicInputs = Number(vkAsFields[1].toString());
 
-
-    // Account for the serialized buffer size at start
-    // Get the part before and after the public inputs
-    const proofStart = proofWithPublicInputs.slice(0, serializedBufferSize);
-    const publicInputsSplitIndex = numPublicInputs * fieldByteSize;
-    const proofEnd = proofWithPublicInputs.slice(serializedBufferSize + publicInputsSplitIndex);
-
-    // Construct the proof without the public inputs
-    const proof = new Uint8Array([...proofStart, ...proofEnd]);
-
-    // Fetch the number of public inputs out of the proof string
-    const publicInputsConcatenated = proofWithPublicInputs.slice(
-      serializedBufferSize,
-      serializedBufferSize + publicInputsSplitIndex,
-    );
-    const publicInputs = deflattenFields(publicInputsConcatenated);
+    const { proof, publicInputs: publicInputsBytes } = splitHonkProof(proofWithPublicInputs, numPublicInputs);
+    const publicInputs = deflattenFields(publicInputsBytes);
 
     return { proof, publicInputs };
   }
@@ -265,29 +248,15 @@ export class UltraHonkBackend {
     const vk = await writeVKUltraHonk(this.acirUncompressedBytecode, this.circuitOptions.recursive);
     const vkAsFields = await this.api.acirVkAsFieldsUltraHonk(new RawBuffer(vk));
 
-    // proofWithPublicInputs starts with a four-byte size
-    const numSerdeHeaderBytes = 4;
     // some public inputs are handled specially
     const numKZGAccumulatorFieldElements = 16;
     const publicInputsSizeIndex = 1; // index into VK for numPublicInputs
-
     const numPublicInputs = Number(vkAsFields[publicInputsSizeIndex].toString()) - numKZGAccumulatorFieldElements;
 
-    // Construct the proof without the public inputs
-    const numPublicInputsBytes = numPublicInputs * fieldByteSize;
-    const proofNoPIs = new Uint8Array(proofWithPublicInputs.length - numPublicInputsBytes);
-    // copy the elements before the public inputs
-    proofNoPIs.set(proofWithPublicInputs.subarray(0, numSerdeHeaderBytes), 0);
-    // copy the elements after the public inputs
-    proofNoPIs.set(proofWithPublicInputs.subarray(numSerdeHeaderBytes + numPublicInputsBytes), numSerdeHeaderBytes);
-    const proof: string[] = deflattenFields(proofNoPIs.slice(numSerdeHeaderBytes));
-
-    // Fetch the number of public inputs out of the proof string
-    const publicInputsConcatenated = proofWithPublicInputs.slice(
-      serializedBufferSize,
-      serializedBufferSize + numPublicInputsBytes,
-    );
-    const publicInputs = deflattenFields(publicInputsConcatenated);
+    const { proof: proofBytes, publicInputs: publicInputsBytes } = splitHonkProof(proofWithPublicInputs, numPublicInputs);
+
+    const publicInputs = deflattenFields(publicInputsBytes);
+    const proof = deflattenFields(proofBytes);
 
     return { proof, publicInputs };
   }

barretenberg/ts/src/proof/index.ts

@@ -1,3 +1,5 @@
+import { numToUInt32BE } from "../serialize/serialize.js";
+
 /**
  * @description
  * The representation of a proof
@@ -20,24 +22,19 @@ export type ProofDataForRecursion = {
   proof: string[];
 };
 
-// Buffers are prepended with their size. The size takes 4 bytes.
-const serializedBufferSize = 4;
+// Honk proofs start with 4 bytes for the size of the proof in fields
+const metadataOffset = 4;
 const fieldByteSize = 32;
 
 export function splitHonkProof(
   proofWithPublicInputs: Uint8Array,
   numPublicInputs: number,
 ): { publicInputs: Uint8Array; proof: Uint8Array } {
-  // Account for the serialized buffer size at start
-  // Get the part before and after the public inputs
-  const proofStart = proofWithPublicInputs.slice(0, serializedBufferSize);
-  const publicInputsSplitIndex = numPublicInputs * fieldByteSize;
-  const proofEnd = proofWithPublicInputs.slice(serializedBufferSize + publicInputsSplitIndex);
-  // Construct the proof without the public inputs
-  const proof = new Uint8Array([...proofStart, ...proofEnd]);
+  // Remove the metadata (proof size in fields)
+  const proofWithPI = proofWithPublicInputs.slice(metadataOffset);
 
-  // Fetch the number of public inputs out of the proof string
-  const publicInputs = proofWithPublicInputs.slice(serializedBufferSize, serializedBufferSize + publicInputsSplitIndex);
+  const publicInputs = proofWithPI.slice(0, numPublicInputs * fieldByteSize);
+  const proof = proofWithPI.slice(numPublicInputs * fieldByteSize);
 
   return {
     proof,
@@ -46,12 +43,10 @@ export function splitHonkProof(
 }
 
 export function reconstructHonkProof(publicInputs: Uint8Array, proof: Uint8Array): Uint8Array {
-  const proofStart = proof.slice(0, serializedBufferSize);
-  const proofEnd = proof.slice(serializedBufferSize);
-
-  // Concatenate publicInputs and proof
-  const proofWithPublicInputs = Uint8Array.from([...proofStart, ...publicInputs, ...proofEnd]);
+  // Append proofWithPublicInputs size in fields
+  const proofSize = numToUInt32BE((publicInputs.length + proof.length) / fieldByteSize);
 
+  const proofWithPublicInputs = Uint8Array.from([...proofSize, ...publicInputs, ...proof]);
   return proofWithPublicInputs;
 }