Skip to content

Commit 753f505

Browse files
Maddiaa0Maddiaa0
authored
feat(blob-lib): make blob lib and fix encoding test flake (#11782)
- Move blob lib out of foundation - Makes a check against valid first field- in the test this was flaking when the random fields generated ended up producing valid length encodings. It is now much stricter about the accepted encoding
1 parent a6fcdb0 commit 753f505

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

54 files changed

+230
-30
lines changed

‎yarn-project/archiver/package.json

+1
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@
6464
]
6565
},
6666
"dependencies": {
67+
"@aztec/blob-lib": "workspace:^",
6768
"@aztec/blob-sink": "workspace:^",
6869
"@aztec/circuit-types": "workspace:^",
6970
"@aztec/circuits.js": "workspace:^",

‎yarn-project/archiver/src/archiver/archiver.test.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
1+
import { Blob } from '@aztec/blob-lib';
12
import { type BlobSinkClientInterface } from '@aztec/blob-sink/client';
23
import { InboxLeaf, type L1RollupConstants, L2Block } from '@aztec/circuit-types';
34
import { GENESIS_ARCHIVE_ROOT, PrivateLog } from '@aztec/circuits.js';
45
import { DefaultL1ContractsConfig } from '@aztec/ethereum';
5-
import { Blob } from '@aztec/foundation/blob';
66
import { EthAddress } from '@aztec/foundation/eth-address';
77
import { Fr } from '@aztec/foundation/fields';
88
import { type Logger, createLogger } from '@aztec/foundation/log';

‎yarn-project/archiver/src/archiver/data_retrieval.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
1+
import { Blob, BlobDeserializationError } from '@aztec/blob-lib';
12
import { type BlobSinkClientInterface } from '@aztec/blob-sink/client';
23
import { Body, InboxLeaf, L2Block } from '@aztec/circuit-types';
34
import { AppendOnlyTreeSnapshot, BlockHeader, Fr, Proof } from '@aztec/circuits.js';
45
import { asyncPool } from '@aztec/foundation/async-pool';
5-
import { Blob, BlobDeserializationError } from '@aztec/foundation/blob';
66
import { type EthAddress } from '@aztec/foundation/eth-address';
77
import { type ViemSignature } from '@aztec/foundation/eth-signature';
88
import { type Logger, createLogger } from '@aztec/foundation/log';

‎yarn-project/archiver/tsconfig.json

+3
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,9 @@
66
"tsBuildInfoFile": ".tsbuildinfo"
77
},
88
"references": [
9+
{
10+
"path": "../blob-lib"
11+
},
912
{
1013
"path": "../blob-sink"
1114
},

‎yarn-project/blob-lib/.eslintrc.cjs

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
module.exports = require('@aztec/foundation/eslint');

‎yarn-project/blob-lib/package.json

+84
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
{
2+
"name": "@aztec/blob-lib",
3+
"version": "0.1.0",
4+
"type": "module",
5+
"exports": {
6+
".": "./dest/index.js"
7+
},
8+
"typedocOptions": {
9+
"entryPoints": [
10+
"./src/index.ts"
11+
],
12+
"name": "Blob Lib",
13+
"tsconfig": "./tsconfig.json"
14+
},
15+
"scripts": {
16+
"build": "yarn clean && tsc -b",
17+
"build:dev": "tsc -b --watch",
18+
"clean": "rm -rf ./dest .tsbuildinfo",
19+
"formatting": "run -T prettier --check ./src && run -T eslint ./src",
20+
"formatting:fix": "run -T eslint --fix ./src && run -T prettier -w ./src",
21+
"start:dev": "tsc-watch -p tsconfig.json --onSuccess 'yarn start'",
22+
"start": "node ./dest/index.js",
23+
"test": "HARDWARE_CONCURRENCY=${HARDWARE_CONCURRENCY:-16} RAYON_NUM_THREADS=${RAYON_NUM_THREADS:-4} NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=${JEST_MAX_WORKERS:-8}"
24+
},
25+
"inherits": [
26+
"../package.common.json"
27+
],
28+
"dependencies": {
29+
"@aztec/foundation": "workspace:^",
30+
"c-kzg": "4.0.0-alpha.1",
31+
"tslib": "^2.4.0"
32+
},
33+
"devDependencies": {
34+
"@jest/globals": "^29.5.0",
35+
"@types/jest": "^29.5.0",
36+
"@types/node": "^18.14.6",
37+
"get-port": "^7.1.0",
38+
"jest": "^29.5.0",
39+
"ts-node": "^10.9.1",
40+
"typescript": "^5.0.4"
41+
},
42+
"files": [
43+
"dest",
44+
"src",
45+
"!*.test.*"
46+
],
47+
"types": "./dest/index.d.ts",
48+
"jest": {
49+
"moduleNameMapper": {
50+
"^(\\.{1,2}/.*)\\.[cm]?js$": "$1"
51+
},
52+
"testRegex": "./src/.*\\.test\\.(js|mjs|ts)$",
53+
"rootDir": "./src",
54+
"transform": {
55+
"^.+\\.tsx?$": [
56+
"@swc/jest",
57+
{
58+
"jsc": {
59+
"parser": {
60+
"syntax": "typescript",
61+
"decorators": true
62+
},
63+
"transform": {
64+
"decoratorVersion": "2022-03"
65+
}
66+
}
67+
}
68+
]
69+
},
70+
"extensionsToTreatAsEsm": [
71+
".ts"
72+
],
73+
"reporters": [
74+
"default"
75+
],
76+
"testTimeout": 30000,
77+
"setupFiles": [
78+
"../../foundation/src/jest/setup.mjs"
79+
]
80+
},
81+
"engines": {
82+
"node": ">=18"
83+
}
84+
}

‎yarn-project/foundation/src/blob/blob.test.ts ‎yarn-project/blob-lib/src/blob.test.ts

+3-2
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
1+
import { poseidon2Hash } from '@aztec/foundation/crypto';
2+
import { Fr } from '@aztec/foundation/fields';
3+
14
import cKzg from 'c-kzg';
25
import type { Blob as BlobBuffer, Bytes48, KZGProof } from 'c-kzg';
36

4-
import { poseidon2Hash } from '../crypto/index.js';
5-
import { Fr } from '../fields/index.js';
67
import { Blob, makeEncodedBlob } from './index.js';
78

89
// Importing directly from 'c-kzg' does not work, ignoring import/no-named-as-default-member err:

‎yarn-project/foundation/src/blob/blob.ts ‎yarn-project/blob-lib/src/blob.ts

+4-3
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,11 @@
11
// Importing directly from 'c-kzg' does not work, ignoring import/no-named-as-default-member err:
2+
import { poseidon2Hash, sha256 } from '@aztec/foundation/crypto';
3+
import { Fr } from '@aztec/foundation/fields';
4+
import { BufferReader, serializeToBuffer } from '@aztec/foundation/serialize';
5+
26
import cKzg from 'c-kzg';
37
import type { Blob as BlobBuffer } from 'c-kzg';
48

5-
import { poseidon2Hash, sha256 } from '../crypto/index.js';
6-
import { Fr } from '../fields/index.js';
7-
import { BufferReader, serializeToBuffer } from '../serialize/index.js';
89
import { deserializeEncodedBlobToFields, extractBlobFieldsFromBuffer } from './encoding.js';
910
import { BlobDeserializationError } from './errors.js';
1011
import { type BlobJson } from './interface.js';

‎yarn-project/foundation/src/blob/encoding.ts ‎yarn-project/blob-lib/src/encoding.ts

+41
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,14 @@ import { BufferReader, FieldReader } from '@aztec/foundation/serialize';
33

44
import type { Blob as BlobBuffer } from 'c-kzg';
55

6+
// Note duplicated from circuit-types !
67
// This will appear as 0x74785f7374617274 in logs
78
export const TX_START_PREFIX = 8392562855083340404n;
89
// These are helper constants to decode tx effects from blob encoded fields
910
export const TX_START_PREFIX_BYTES_LENGTH = TX_START_PREFIX.toString(16).length / 2;
1011
// 7 bytes for: | 0 | txlen[0] | txlen[1] | 0 | REVERT_CODE_PREFIX | 0 | revertCode |
1112
export const TX_EFFECT_PREFIX_BYTE_LENGTH = TX_START_PREFIX_BYTES_LENGTH + 7;
13+
export const REVERT_CODE_PREFIX = 1;
1214

1315
/**
1416
* Deserializes a blob buffer into an array of field elements.
@@ -66,11 +68,50 @@ export function deserializeEncodedBlobToFields(blob: BlobBuffer): Fr[] {
6668
return array.slice(0, fieldReader.cursor);
6769
}
6870

71+
/**
72+
* Get the length of the transaction from the first field.
73+
*
74+
* @param firstField - The first field of the transaction.
75+
* @returns The length of the transaction.
76+
*
77+
* @throws If the first field does not include the correct prefix - encoding invalid.
78+
*/
6979
export function getLengthFromFirstField(firstField: Fr): number {
80+
// Check that the first field includes the correct prefix
81+
if (!isValidFirstField(firstField)) {
82+
throw new Error('Invalid prefix');
83+
}
7084
const buf = firstField.toBuffer().subarray(-TX_EFFECT_PREFIX_BYTE_LENGTH);
7185
return new Fr(buf.subarray(TX_START_PREFIX_BYTES_LENGTH + 1, TX_START_PREFIX_BYTES_LENGTH + 3)).toNumber();
7286
}
7387

88+
// NOTE: duplicated from circuit-types tx effect!
89+
/**
90+
* Determines whether a field is the first field of a tx effect
91+
*/
92+
export function isValidFirstField(field: Fr): boolean {
93+
const buf = field.toBuffer();
94+
if (
95+
!buf
96+
.subarray(0, field.size - TX_EFFECT_PREFIX_BYTE_LENGTH)
97+
.equals(Buffer.alloc(field.size - TX_EFFECT_PREFIX_BYTE_LENGTH))
98+
) {
99+
return false;
100+
}
101+
const sliced = buf.subarray(-TX_EFFECT_PREFIX_BYTE_LENGTH);
102+
if (
103+
// Checking we start with the correct prefix...
104+
!new Fr(sliced.subarray(0, TX_START_PREFIX_BYTES_LENGTH)).equals(new Fr(TX_START_PREFIX)) ||
105+
// ...and include the revert code prefix..
106+
sliced[sliced.length - 3] !== REVERT_CODE_PREFIX ||
107+
// ...and the following revert code is valid.
108+
sliced[sliced.length - 1] > 4
109+
) {
110+
return false;
111+
}
112+
return true;
113+
}
114+
74115
/**
75116
* Extract the fields from a blob buffer, but do not take into account encoding
76117
* that will include trailing zeros.

‎yarn-project/foundation/src/blob/errors.ts ‎yarn-project/blob-lib/src/errors.ts

File renamed without changes.

‎yarn-project/foundation/src/blob/index.ts ‎yarn-project/blob-lib/src/index.ts

File renamed without changes.

‎yarn-project/foundation/src/blob/interface.ts ‎yarn-project/blob-lib/src/interface.ts

File renamed without changes.

‎yarn-project/foundation/src/blob/mocks.ts ‎yarn-project/blob-lib/src/mocks.ts

File renamed without changes.

‎yarn-project/blob-lib/tsconfig.json

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
{
2+
"extends": "..",
3+
"compilerOptions": {
4+
"outDir": "dest",
5+
"rootDir": "src",
6+
"tsBuildInfoFile": ".tsbuildinfo"
7+
},
8+
"include": ["src"],
9+
"references": [
10+
{
11+
"path": "../foundation"
12+
}
13+
]
14+
}

‎yarn-project/blob-sink/package.json

+1
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@
5252
]
5353
},
5454
"dependencies": {
55+
"@aztec/blob-lib": "workspace:^",
5556
"@aztec/circuit-types": "workspace:^",
5657
"@aztec/foundation": "workspace:^",
5758
"@aztec/kv-store": "workspace:*",

‎yarn-project/blob-sink/src/blobstore/blob_store_test_suite.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { Blob } from '@aztec/foundation/blob';
1+
import { Blob } from '@aztec/blob-lib';
22
import { Fr } from '@aztec/foundation/fields';
33

44
import { BlobWithIndex } from '../types/index.js';

‎yarn-project/blob-sink/src/client/blob-sink-client-tests.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { makeEncodedBlob } from '@aztec/foundation/blob';
1+
import { makeEncodedBlob } from '@aztec/blob-lib';
22

33
import { type BlobSinkClientInterface } from './interface.js';
44

‎yarn-project/blob-sink/src/client/http.test.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { Blob, makeEncodedBlob, makeUnencodedBlob } from '@aztec/foundation/blob';
1+
import { Blob, makeEncodedBlob, makeUnencodedBlob } from '@aztec/blob-lib';
22
import { Fr } from '@aztec/foundation/fields';
33

44
import { jest } from '@jest/globals';

‎yarn-project/blob-sink/src/client/http.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { Blob, BlobDeserializationError, type BlobJson } from '@aztec/foundation/blob';
1+
import { Blob, BlobDeserializationError, type BlobJson } from '@aztec/blob-lib';
22
import { type Logger, createLogger } from '@aztec/foundation/log';
33
import { makeBackoff, retry } from '@aztec/foundation/retry';
44

‎yarn-project/blob-sink/src/client/interface.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { type Blob } from '@aztec/foundation/blob';
1+
import { type Blob } from '@aztec/blob-lib';
22

33
export interface BlobSinkClientInterface {
44
sendBlobsToBlobSink(blockId: string, blobs: Blob[]): Promise<boolean>;

‎yarn-project/blob-sink/src/client/local.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { type Blob } from '@aztec/foundation/blob';
1+
import { type Blob } from '@aztec/blob-lib';
22

33
import { type BlobStore } from '../blobstore/index.js';
44
import { BlobWithIndex } from '../types/blob_with_index.js';

‎yarn-project/blob-sink/src/server/server.test.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { Blob, makeEncodedBlob } from '@aztec/foundation/blob';
1+
import { Blob, makeEncodedBlob } from '@aztec/blob-lib';
22

33
import request from 'supertest';
44

‎yarn-project/blob-sink/src/server/server.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { Blob } from '@aztec/foundation/blob';
1+
import { Blob } from '@aztec/blob-lib';
22
import { type Logger, createLogger } from '@aztec/foundation/log';
33
import { type AztecAsyncKVStore } from '@aztec/kv-store';
44
import { type TelemetryClient, getTelemetryClient } from '@aztec/telemetry-client';

‎yarn-project/blob-sink/src/types/blob_with_index.test.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { Blob } from '@aztec/foundation/blob';
1+
import { Blob } from '@aztec/blob-lib';
22
import { Fr } from '@aztec/foundation/fields';
33

44
import { BlobWithIndex, BlobsWithIndexes } from './blob_with_index.js';

‎yarn-project/blob-sink/src/types/blob_with_index.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { Blob, type BlobJson } from '@aztec/foundation/blob';
1+
import { Blob, type BlobJson } from '@aztec/blob-lib';
22
import { BufferReader, serializeToBuffer } from '@aztec/foundation/serialize';
33

44
/** Serialized an array of blobs with their indexes to be stored at a given block id */

‎yarn-project/blob-sink/tsconfig.json

+3
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,9 @@
66
"tsBuildInfoFile": ".tsbuildinfo"
77
},
88
"references": [
9+
{
10+
"path": "../blob-lib"
11+
},
912
{
1013
"path": "../circuit-types"
1114
},

‎yarn-project/circuit-types/package.json

+1
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
"type": "module",
55
"exports": {
66
".": "./dest/index.js",
7+
"./blob": "./dest/blob/index.js",
78
"./stats": "./dest/stats/index.js",
89
"./jest": "./dest/jest/index.js",
910
"./interfaces": "./dest/interfaces/index.js",

‎yarn-project/circuits.js/package.json

+1
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@
4242
},
4343
"dependencies": {
4444
"@aztec/bb.js": "portal:../../barretenberg/ts",
45+
"@aztec/blob-lib": "workspace:^",
4546
"@aztec/ethereum": "workspace:^",
4647
"@aztec/foundation": "workspace:^",
4748
"@aztec/types": "workspace:^",

‎yarn-project/circuits.js/src/structs/blobs/blob_public_inputs.test.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { Blob } from '@aztec/foundation/blob';
1+
import { Blob } from '@aztec/blob-lib';
22
import { timesParallel } from '@aztec/foundation/collection';
33
import { randomInt } from '@aztec/foundation/crypto';
44

‎yarn-project/circuits.js/src/structs/blobs/blob_public_inputs.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
1+
import { type Blob } from '@aztec/blob-lib';
12
import { makeTuple } from '@aztec/foundation/array';
23
import { toBigIntBE, toBufferBE, toHex } from '@aztec/foundation/bigint-buffer';
3-
import { type Blob } from '@aztec/foundation/blob';
44
import { sha256, sha256Trunc } from '@aztec/foundation/crypto';
55
import { Fr } from '@aztec/foundation/fields';
66
import { BufferReader, FieldReader, type Tuple, serializeToBuffer } from '@aztec/foundation/serialize';

‎yarn-project/circuits.js/tsconfig.json

+3
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,9 @@
66
"tsBuildInfoFile": ".tsbuildinfo"
77
},
88
"references": [
9+
{
10+
"path": "../blob-lib"
11+
},
912
{
1013
"path": "../ethereum"
1114
},

‎yarn-project/end-to-end/.gitignore

+2-1
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,3 @@
11
joblog.txt
2-
results
2+
results
3+
web/main.js*

‎yarn-project/end-to-end/package.json

+1
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
"@aztec/aztec-node": "workspace:^",
3434
"@aztec/aztec.js": "workspace:^",
3535
"@aztec/bb-prover": "workspace:^",
36+
"@aztec/blob-lib": "workspace:^",
3637
"@aztec/blob-sink": "workspace:^",
3738
"@aztec/bot": "workspace:^",
3839
"@aztec/circuit-types": "workspace:^",

‎yarn-project/end-to-end/src/composed/integration_l1_publisher.test.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import { type ArchiveSource } from '@aztec/archiver';
22
import { getConfigEnvVars } from '@aztec/aztec-node';
33
import { AztecAddress, Fr, GlobalVariables, type L2Block, createLogger } from '@aztec/aztec.js';
4+
import { Blob } from '@aztec/blob-lib';
45
// eslint-disable-next-line no-restricted-imports
56
import { type L2Tips, type ProcessedTx } from '@aztec/circuit-types';
67
import { makeBloatedProcessedTx } from '@aztec/circuit-types/test';
@@ -27,7 +28,6 @@ import {
2728
} from '@aztec/ethereum';
2829
import { EthCheatCodesWithState } from '@aztec/ethereum/test';
2930
import { range } from '@aztec/foundation/array';
30-
import { Blob } from '@aztec/foundation/blob';
3131
import { timesParallel } from '@aztec/foundation/collection';
3232
import { sha256, sha256ToField } from '@aztec/foundation/crypto';
3333
import { TestDateProvider } from '@aztec/foundation/timer';

‎yarn-project/end-to-end/tsconfig.json

+3
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,9 @@
2121
{
2222
"path": "../bb-prover"
2323
},
24+
{
25+
"path": "../blob-lib"
26+
},
2427
{
2528
"path": "../blob-sink"
2629
},

‎yarn-project/ethereum/package.json

+1
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
"../package.common.json"
3333
],
3434
"dependencies": {
35+
"@aztec/blob-lib": "workspace:^",
3536
"@aztec/foundation": "workspace:^",
3637
"@aztec/l1-artifacts": "workspace:^",
3738
"@viem/anvil": "^0.0.10",

0 commit comments

Comments
 (0)