Skip to content

Commit a9d418c

Browse files
authored
feat: Process blocks in parallel during epoch proving (#10263)
Instead of processing blocks in sequencer during epoch proving and trigger proving jobs as each block is processed, we fetch the state prior to each block by forking off a world state following the pending (not proven) chain and process each block (ie execute public functions) in parallel. This means tx execution is less of a bottleneck for proving. Main change is that the epoch orchestrator now requires not a db, but something that can return forks at given block numbers of the db. It also means that the orchestrator accepts out-of-order operations for block building, so multiple blocks can be started, and their txs added in any order (though following the order within each block). Builds on #10174 Fixes #10265 Pending: - Ensuring the state after processing each block matches what we had used from world-state
1 parent ebd4165 commit a9d418c

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

46 files changed

+526
-482
lines changed

yarn-project/circuit-types/src/interfaces/epoch-prover.ts

+6-18
Original file line numberDiff line numberDiff line change
@@ -1,32 +1,20 @@
1-
import { type Fr, type Proof, type RootRollupPublicInputs } from '@aztec/circuits.js';
1+
import { type Fr, type Header, type Proof, type RootRollupPublicInputs } from '@aztec/circuits.js';
22

33
import { type L2Block } from '../l2_block.js';
44
import { type BlockBuilder } from './block-builder.js';
55

6-
/**
7-
* Coordinates the proving of an entire epoch.
8-
*
9-
* Expected usage:
10-
* ```
11-
* startNewEpoch
12-
* foreach block {
13-
* addNewBlock
14-
* foreach tx {
15-
* addTx
16-
* }
17-
* setBlockCompleted
18-
* }
19-
* finaliseEpoch
20-
* ```
21-
*/
22-
export interface EpochProver extends BlockBuilder {
6+
/** Coordinates the proving of an entire epoch. */
7+
export interface EpochProver extends Omit<BlockBuilder, 'setBlockCompleted'> {
238
/**
249
* Starts a new epoch. Must be the first method to be called.
2510
* @param epochNumber - The epoch number.
2611
* @param totalNumBlocks - The total number of blocks expected in the epoch (must be at least one).
2712
**/
2813
startNewEpoch(epochNumber: number, totalNumBlocks: number): void;
2914

15+
/** Pads the block with empty txs if it hasn't reached the declared number of txs. */
16+
setBlockCompleted(blockNumber: number, expectedBlockHeader?: Header): Promise<L2Block>;
17+
3018
/** Pads the epoch with empty block roots if needed and blocks until proven. Throws if proving has failed. */
3119
finaliseEpoch(): Promise<{ publicInputs: RootRollupPublicInputs; proof: Proof }>;
3220

yarn-project/circuit-types/src/interfaces/prover-client.ts

+1-2
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@ import { z } from 'zod';
66

77
import { type TxHash } from '../tx/tx_hash.js';
88
import { type EpochProver } from './epoch-prover.js';
9-
import { type MerkleTreeReadOperations } from './merkle_tree_operations.js';
109
import { type ProvingJobConsumer } from './prover-broker.js';
1110
import { type ProvingJobStatus } from './proving-job.js';
1211

@@ -105,7 +104,7 @@ export interface ProverCache {
105104
* Provides the ability to generate proofs and build rollups.
106105
*/
107106
export interface EpochProverManager {
108-
createEpochProver(db: MerkleTreeReadOperations, cache?: ProverCache): EpochProver;
107+
createEpochProver(cache?: ProverCache): EpochProver;
109108

110109
start(): Promise<void>;
111110

yarn-project/circuit-types/src/interfaces/world_state.ts

+11-15
Original file line numberDiff line numberDiff line change
@@ -25,10 +25,17 @@ export interface WorldStateSynchronizerStatus {
2525
syncedToL2Block: L2BlockId;
2626
}
2727

28-
/**
29-
* Defines the interface for a world state synchronizer.
30-
*/
31-
export interface WorldStateSynchronizer {
28+
/** Provides writeable forks of the world state at a given block number. */
29+
export interface ForkMerkleTreeOperations {
30+
/** Forks the world state at the given block number, defaulting to the latest one. */
31+
fork(block?: number): Promise<MerkleTreeWriteOperations>;
32+
33+
/** Gets a handle that allows reading the state as it was at the given block number. */
34+
getSnapshot(blockNumber: number): MerkleTreeReadOperations;
35+
}
36+
37+
/** Defines the interface for a world state synchronizer. */
38+
export interface WorldStateSynchronizer extends ForkMerkleTreeOperations {
3239
/**
3340
* Starts the synchronizer.
3441
* @returns A promise that resolves once the initial sync is completed.
@@ -53,19 +60,8 @@ export interface WorldStateSynchronizer {
5360
*/
5461
syncImmediate(minBlockNumber?: number): Promise<number>;
5562

56-
/**
57-
* Forks the current in-memory state based off the current committed state, and returns an instance that cannot modify the underlying data store.
58-
*/
59-
fork(block?: number): Promise<MerkleTreeWriteOperations>;
60-
6163
/**
6264
* Returns an instance of MerkleTreeAdminOperations that will not include uncommitted data.
6365
*/
6466
getCommitted(): MerkleTreeReadOperations;
65-
66-
/**
67-
* Returns a readonly instance of MerkleTreeAdminOperations where the state is as it was at the given block number
68-
* @param block - The block number to look at
69-
*/
70-
getSnapshot(block: number): MerkleTreeReadOperations;
7167
}

yarn-project/circuit-types/src/test/factories.ts

+1-8
Original file line numberDiff line numberDiff line change
@@ -53,14 +53,7 @@ export function makeBloatedProcessedTx({
5353
privateOnly?: boolean;
5454
} = {}) {
5555
seed *= 0x1000; // Avoid clashing with the previous mock values if seed only increases by 1.
56-
57-
if (!header) {
58-
if (db) {
59-
header = db.getInitialHeader();
60-
} else {
61-
header = makeHeader(seed);
62-
}
63-
}
56+
header ??= db?.getInitialHeader() ?? makeHeader(seed);
6457

6558
const txConstantData = TxConstantData.empty();
6659
txConstantData.historicalHeader = header;

yarn-project/end-to-end/src/composed/integration_l1_publisher.test.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ import { OutboxAbi, RollupAbi } from '@aztec/l1-artifacts';
2525
import { SHA256Trunc, StandardTree } from '@aztec/merkle-tree';
2626
import { getVKTreeRoot } from '@aztec/noir-protocol-circuits-types';
2727
import { protocolContractTreeRoot } from '@aztec/protocol-contracts';
28+
import { LightweightBlockBuilder } from '@aztec/prover-client/block-builder';
2829
import { L1Publisher } from '@aztec/sequencer-client';
2930
import { NoopTelemetryClient } from '@aztec/telemetry-client/noop';
3031
import {
@@ -52,7 +53,6 @@ import {
5253
} from 'viem';
5354
import { type PrivateKeyAccount, privateKeyToAccount } from 'viem/accounts';
5455

55-
import { LightweightBlockBuilder } from '../../../sequencer-client/src/block_builder/light.js';
5656
import { sendL1ToL2Message } from '../fixtures/l1_to_l2_messaging.js';
5757
import { setupL1Contracts } from '../fixtures/utils.js';
5858

yarn-project/end-to-end/src/e2e_prover/e2e_prover_test.ts

+1
Original file line numberDiff line numberDiff line change
@@ -269,6 +269,7 @@ export class FullProverTest {
269269
proverAgentCount: 2,
270270
publisherPrivateKey: `0x${proverNodePrivateKey!.toString('hex')}`,
271271
proverNodeMaxPendingJobs: 100,
272+
proverNodeMaxParallelBlocksPerEpoch: 32,
272273
proverNodePollingIntervalMs: 100,
273274
quoteProviderBasisPointFee: 100,
274275
quoteProviderBondAmount: 1000n,

yarn-project/end-to-end/src/fixtures/utils.ts

+1
Original file line numberDiff line numberDiff line change
@@ -682,6 +682,7 @@ export async function createAndSyncProverNode(
682682
proverAgentCount: 2,
683683
publisherPrivateKey: proverNodePrivateKey,
684684
proverNodeMaxPendingJobs: 10,
685+
proverNodeMaxParallelBlocksPerEpoch: 32,
685686
proverNodePollingIntervalMs: 200,
686687
quoteProviderBasisPointFee: 100,
687688
quoteProviderBondAmount: 1000n,

yarn-project/foundation/package.json

+2-1
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
"./prettier": "./.prettierrc.json",
1212
"./abi": "./dest/abi/index.js",
1313
"./async-map": "./dest/async-map/index.js",
14+
"./async-pool": "./dest/async-pool/index.js",
1415
"./aztec-address": "./dest/aztec-address/index.js",
1516
"./collection": "./dest/collection/index.js",
1617
"./config": "./dest/config/index.js",
@@ -163,4 +164,4 @@
163164
"engines": {
164165
"node": ">=18"
165166
}
166-
}
167+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
/*
2+
* Adapted from https://github.com/rxaviers/async-pool/blob/1.x/lib/es6.js
3+
*
4+
* Copyright (c) 2017 Rafael Xavier de Souza http://rafael.xavier.blog.br
5+
*
6+
* Permission is hereby granted, free of charge, to any person
7+
* obtaining a copy of this software and associated documentation
8+
* files (the "Software"), to deal in the Software without
9+
* restriction, including without limitation the rights to use,
10+
* copy, modify, merge, publish, distribute, sublicense, and/or sell
11+
* copies of the Software, and to permit persons to whom the
12+
* Software is furnished to do so, subject to the following
13+
* conditions:
14+
*
15+
* The above copyright notice and this permission notice shall be
16+
* included in all copies or substantial portions of the Software.
17+
*
18+
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
19+
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
20+
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
21+
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
22+
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
23+
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
24+
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
25+
* OTHER DEALINGS IN THE SOFTWARE.
26+
*/
27+
28+
/** Executes the given async function over the iterable, up to a determined number of promises in parallel. */
29+
export function asyncPool<T, R>(poolLimit: number, iterable: T[], iteratorFn: (item: T, iterable: T[]) => Promise<R>) {
30+
let i = 0;
31+
const ret: Promise<R>[] = [];
32+
const executing: Set<Promise<R>> = new Set();
33+
const enqueue = (): Promise<any> => {
34+
if (i === iterable.length) {
35+
return Promise.resolve();
36+
}
37+
const item = iterable[i++];
38+
const p = Promise.resolve().then(() => iteratorFn(item, iterable));
39+
ret.push(p);
40+
executing.add(p);
41+
const clean = () => executing.delete(p);
42+
p.then(clean).catch(clean);
43+
let r: Promise<any> = Promise.resolve();
44+
if (executing.size >= poolLimit) {
45+
r = Promise.race(executing);
46+
}
47+
return r.then(() => enqueue());
48+
};
49+
return enqueue().then(() => Promise.all(ret));
50+
}

yarn-project/foundation/src/collection/array.test.ts

+19-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { compactArray, removeArrayPaddingEnd, times, unique } from './array.js';
1+
import { compactArray, maxBy, removeArrayPaddingEnd, times, unique } from './array.js';
22

33
describe('times', () => {
44
it('should return an array with the result from all executions', () => {
@@ -61,3 +61,21 @@ describe('unique', () => {
6161
expect(unique([1n, 2n, 1n])).toEqual([1n, 2n]);
6262
});
6363
});
64+
65+
describe('maxBy', () => {
66+
it('returns the max value', () => {
67+
expect(maxBy([1, 2, 3], x => x)).toEqual(3);
68+
});
69+
70+
it('returns the first max value', () => {
71+
expect(maxBy([{ a: 1 }, { a: 3, b: 1 }, { a: 3, b: 2 }], ({ a }) => a)).toEqual({ a: 3, b: 1 });
72+
});
73+
74+
it('returns undefined for an empty array', () => {
75+
expect(maxBy([], x => x)).toBeUndefined();
76+
});
77+
78+
it('applies the mapping function', () => {
79+
expect(maxBy([1, 2, 3], x => -x)).toEqual(1);
80+
});
81+
});

yarn-project/foundation/src/collection/array.ts

+24
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,20 @@ export function times<T>(n: number, fn: (i: number) => T): T[] {
7575
return [...Array(n).keys()].map(i => fn(i));
7676
}
7777

78+
/**
79+
* Executes the given async function n times and returns the results in an array. Awaits each execution before starting the next one.
80+
* @param n - How many times to repeat.
81+
* @param fn - Mapper from index to value.
82+
* @returns The array with the result from all executions.
83+
*/
84+
export async function timesAsync<T>(n: number, fn: (i: number) => Promise<T>): Promise<T[]> {
85+
const results: T[] = [];
86+
for (let i = 0; i < n; i++) {
87+
results.push(await fn(i));
88+
}
89+
return results;
90+
}
91+
7892
/**
7993
* Returns the serialized size of all non-empty items in an array.
8094
* @param arr - Array
@@ -121,3 +135,13 @@ export function areArraysEqual<T>(a: T[], b: T[], eq: (a: T, b: T) => boolean =
121135
}
122136
return true;
123137
}
138+
139+
/**
140+
* Returns the element of the array that has the maximum value of the given function.
141+
* In case of a tie, returns the first element with the maximum value.
142+
* @param arr - The array.
143+
* @param fn - The function to get the value to compare.
144+
*/
145+
export function maxBy<T>(arr: T[], fn: (x: T) => number): T | undefined {
146+
return arr.reduce((max, x) => (fn(x) > fn(max) ? x : max), arr[0]);
147+
}

yarn-project/foundation/src/config/env_var.ts

+1
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,7 @@ export type EnvVar =
115115
| 'PROVER_JOB_TIMEOUT_MS'
116116
| 'PROVER_NODE_POLLING_INTERVAL_MS'
117117
| 'PROVER_NODE_MAX_PENDING_JOBS'
118+
| 'PROVER_NODE_MAX_PARALLEL_BLOCKS_PER_EPOCH'
118119
| 'PROVER_PUBLISH_RETRY_INTERVAL_MS'
119120
| 'PROVER_PUBLISHER_PRIVATE_KEY'
120121
| 'PROVER_REAL_PROOFS'

yarn-project/prover-client/package.json

+2-1
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
"type": "module",
55
"exports": {
66
".": "./dest/index.js",
7+
"./block-builder": "./dest/block_builder/index.js",
78
"./broker": "./dest/proving_broker/index.js",
89
"./prover-agent": "./dest/prover-agent/index.js",
910
"./orchestrator": "./dest/orchestrator/index.js",
@@ -103,4 +104,4 @@
103104
"engines": {
104105
"node": ">=18"
105106
}
106-
}
107+
}

yarn-project/sequencer-client/src/block_builder/index.ts yarn-project/prover-client/src/block_builder/index.ts

-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
import { type BlockBuilder, type MerkleTreeReadOperations } from '@aztec/circuit-types';
22

3-
export * from './orchestrator.js';
43
export * from './light.js';
54
export interface BlockBuilderFactory {
65
create(db: MerkleTreeReadOperations): BlockBuilder;

yarn-project/sequencer-client/src/block_builder/light.test.ts yarn-project/prover-client/src/block_builder/light.test.ts

+6-6
Original file line numberDiff line numberDiff line change
@@ -47,18 +47,18 @@ import {
4747
getVKTreeRoot,
4848
} from '@aztec/noir-protocol-circuits-types';
4949
import { protocolContractTreeRoot } from '@aztec/protocol-contracts';
50+
import { NoopTelemetryClient } from '@aztec/telemetry-client/noop';
51+
import { type MerkleTreeAdminDatabase, NativeWorldStateService } from '@aztec/world-state';
52+
53+
import { jest } from '@jest/globals';
54+
5055
import {
5156
buildBaseRollupHints,
5257
buildHeaderFromCircuitOutputs,
5358
getRootTreeSiblingPath,
5459
getSubtreeSiblingPath,
5560
getTreeSnapshot,
56-
} from '@aztec/prover-client/helpers';
57-
import { NoopTelemetryClient } from '@aztec/telemetry-client/noop';
58-
import { type MerkleTreeAdminDatabase, NativeWorldStateService } from '@aztec/world-state';
59-
60-
import { jest } from '@jest/globals';
61-
61+
} from '../orchestrator/block-building-helpers.js';
6262
import { LightweightBlockBuilder } from './light.js';
6363

6464
jest.setTimeout(50_000);

yarn-project/sequencer-client/src/block_builder/light.ts yarn-project/prover-client/src/block_builder/light.ts

+27-2
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
import { createDebugLogger } from '@aztec/aztec.js';
21
import {
32
type BlockBuilder,
43
L2Block,
@@ -9,14 +8,20 @@ import {
98
} from '@aztec/circuit-types';
109
import { Fr, type GlobalVariables, NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP } from '@aztec/circuits.js';
1110
import { padArrayEnd } from '@aztec/foundation/collection';
11+
import { createDebugLogger } from '@aztec/foundation/log';
1212
import { getVKTreeRoot } from '@aztec/noir-protocol-circuits-types';
1313
import { protocolContractTreeRoot } from '@aztec/protocol-contracts';
14-
import { buildBaseRollupHints, buildHeaderAndBodyFromTxs, getTreeSnapshot } from '@aztec/prover-client/helpers';
1514
import { type TelemetryClient } from '@aztec/telemetry-client';
1615
import { NoopTelemetryClient } from '@aztec/telemetry-client/noop';
1716

1817
import { inspect } from 'util';
1918

19+
import {
20+
buildBaseRollupHints,
21+
buildHeaderAndBodyFromTxs,
22+
getTreeSnapshot,
23+
} from '../orchestrator/block-building-helpers.js';
24+
2025
/**
2126
* Builds a block and its header from a set of processed tx without running any circuits.
2227
*/
@@ -90,3 +95,23 @@ export class LightweightBlockBuilderFactory {
9095
return new LightweightBlockBuilder(db, this.telemetry ?? new NoopTelemetryClient());
9196
}
9297
}
98+
99+
/**
100+
* Creates a block builder under the hood with the given txs and messages and creates a block.
101+
* Automatically adds padding txs to get to a minimum of 2 txs in the block.
102+
* @param db - A db fork to use for block building.
103+
*/
104+
export async function buildBlock(
105+
txs: ProcessedTx[],
106+
globalVariables: GlobalVariables,
107+
l1ToL2Messages: Fr[],
108+
db: MerkleTreeWriteOperations,
109+
telemetry: TelemetryClient = new NoopTelemetryClient(),
110+
) {
111+
const builder = new LightweightBlockBuilder(db, telemetry);
112+
await builder.startNewBlock(Math.max(txs.length, 2), globalVariables, l1ToL2Messages);
113+
for (const tx of txs) {
114+
await builder.addNewTx(tx);
115+
}
116+
return await builder.setBlockCompleted();
117+
}
+1-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
export { EpochProverManager } from '@aztec/circuit-types';
22

3-
export * from './tx-prover/tx-prover.js';
3+
export * from './prover-client/index.js';
44
export * from './config.js';
5-
export * from './tx-prover/factory.js';
65
export * from './proving_broker/prover_cache/memory.js';

0 commit comments

Comments
 (0)