diff --git a/yarn-project/end-to-end/src/e2e_p2p_network.test.ts b/yarn-project/end-to-end/src/e2e_p2p_network.test.ts
index 7c782e6edf9..cf414d6df7f 100644
--- a/yarn-project/end-to-end/src/e2e_p2p_network.test.ts
+++ b/yarn-project/end-to-end/src/e2e_p2p_network.test.ts
@@ -150,7 +150,7 @@ describe('e2e_p2p_network', () => {
numTxs: number,
): Promise<NodeContext> => {
const rpcConfig = getRpcConfig();
- const pxeService = await createPXEService(node, rpcConfig, {}, true);
+ const pxeService = await createPXEService(node, rpcConfig, true);
const keyPair = ConstantKeyPair.random(new Grumpkin());
const completeAddress = CompleteAddress.fromPrivateKeyAndPartialAddress(keyPair.getPrivateKey(), Fr.random());
diff --git a/yarn-project/end-to-end/src/fixtures/utils.ts b/yarn-project/end-to-end/src/fixtures/utils.ts
index 978147ab5e0..3ac20b8339f 100644
--- a/yarn-project/end-to-end/src/fixtures/utils.ts
+++ b/yarn-project/end-to-end/src/fixtures/utils.ts
@@ -151,7 +151,7 @@ export async function setupPXEService(
logger: DebugLogger;
}> {
const pxeServiceConfig = getPXEServiceConfig();
- const pxe = await createPXEService(aztecNode, pxeServiceConfig, {}, useLogSuffix);
+ const pxe = await createPXEService(aztecNode, pxeServiceConfig, useLogSuffix);
const wallets = await createAccounts(pxe, numberOfAccounts);
diff --git a/yarn-project/foundation/src/serialize/buffer_reader.ts b/yarn-project/foundation/src/serialize/buffer_reader.ts
index 1b8749d6358..b54a2d1b52b 100644
--- a/yarn-project/foundation/src/serialize/buffer_reader.ts
+++ b/yarn-project/foundation/src/serialize/buffer_reader.ts
@@ -31,8 +31,16 @@ export class BufferReader {
* @param bufferOrReader - A Buffer or BufferReader to initialize the BufferReader.
* @returns An instance of BufferReader.
*/
- public static asReader(bufferOrReader: Buffer | BufferReader) {
- return Buffer.isBuffer(bufferOrReader) ? new BufferReader(bufferOrReader) : bufferOrReader;
+ public static asReader(bufferOrReader: Uint8Array | Buffer | BufferReader): BufferReader {
+ if (bufferOrReader instanceof BufferReader) {
+ return bufferOrReader;
+ }
+
+ const buf = Buffer.isBuffer(bufferOrReader)
+ ? bufferOrReader
+ : Buffer.from(bufferOrReader.buffer, bufferOrReader.byteOffset, bufferOrReader.byteLength);
+
+ return new BufferReader(buf);
}
/**
diff --git a/yarn-project/key-store/package.json b/yarn-project/key-store/package.json
index bbcdabc0de8..a19af464fd2 100644
--- a/yarn-project/key-store/package.json
+++ b/yarn-project/key-store/package.json
@@ -32,6 +32,7 @@
"dependencies": {
"@aztec/circuits.js": "workspace:^",
"@aztec/foundation": "workspace:^",
+ "@aztec/kv-store": "workspace:^",
"@aztec/types": "workspace:^",
"tslib": "^2.4.0"
},
diff --git a/yarn-project/key-store/src/test_key_store.ts b/yarn-project/key-store/src/test_key_store.ts
index 865fb9d75ee..65d64fd0231 100644
--- a/yarn-project/key-store/src/test_key_store.ts
+++ b/yarn-project/key-store/src/test_key_store.ts
@@ -1,5 +1,6 @@
-import { GrumpkinPrivateKey } from '@aztec/circuits.js';
+import { GrumpkinPrivateKey, GrumpkinScalar, Point } from '@aztec/circuits.js';
import { Grumpkin } from '@aztec/circuits.js/barretenberg';
+import { AztecKVStore, AztecMap } from '@aztec/kv-store';
import { KeyPair, KeyStore, PublicKey } from '@aztec/types';
import { ConstantKeyPair } from './key_pair.js';
@@ -9,30 +10,27 @@ import { ConstantKeyPair } from './key_pair.js';
* It should be utilized in testing scenarios where secure key management is not required, and ease-of-use is prioritized.
*/
export class TestKeyStore implements KeyStore {
- private accounts: KeyPair[] = [];
- constructor(private curve: Grumpkin) {}
+ #keys: AztecMap<string, Buffer>;
- public addAccount(privKey: GrumpkinPrivateKey): PublicKey {
- const keyPair = ConstantKeyPair.fromPrivateKey(this.curve, privKey);
-
- // check if private key has already been used
- const account = this.accounts.find(a => a.getPublicKey().equals(keyPair.getPublicKey()));
- if (account) {
- return account.getPublicKey();
- }
+ constructor(private curve: Grumpkin, database: AztecKVStore) {
+ this.#keys = database.createMap('key_store');
+ }
- this.accounts.push(keyPair);
+ public async addAccount(privKey: GrumpkinPrivateKey): Promise<PublicKey> {
+ const keyPair = ConstantKeyPair.fromPrivateKey(this.curve, privKey);
+ await this.#keys.setIfNotExists(keyPair.getPublicKey().toString(), keyPair.getPrivateKey().toBuffer());
return keyPair.getPublicKey();
}
- public createAccount(): Promise<PublicKey> {
+ public async createAccount(): Promise<PublicKey> {
const keyPair = ConstantKeyPair.random(this.curve);
- this.accounts.push(keyPair);
- return Promise.resolve(keyPair.getPublicKey());
+ await this.#keys.set(keyPair.getPublicKey().toString(), keyPair.getPrivateKey().toBuffer());
+ return keyPair.getPublicKey();
}
public getAccounts(): Promise<PublicKey[]> {
- return Promise.resolve(this.accounts.map(a => a.getPublicKey()));
+ const range = Array.from(this.#keys.keys());
+ return Promise.resolve(range.map(key => Point.fromString(key)));
}
public getAccountPrivateKey(pubKey: PublicKey): Promise<GrumpkinPrivateKey> {
@@ -48,13 +46,13 @@ export class TestKeyStore implements KeyStore {
* @param pubKey - The public key of the account to retrieve.
* @returns The KeyPair object associated with the provided key.
*/
- private getAccount(pubKey: PublicKey) {
- const account = this.accounts.find(a => a.getPublicKey().equals(pubKey));
- if (!account) {
+ private getAccount(pubKey: PublicKey): KeyPair {
+ const privKey = this.#keys.get(pubKey.toString());
+ if (!privKey) {
throw new Error(
'Unknown account.\nSee docs for context: https://docs.aztec.network/dev_docs/contracts/common_errors#unknown-contract-error',
);
}
- return account;
+ return ConstantKeyPair.fromPrivateKey(this.curve, GrumpkinScalar.fromBuffer(privKey));
}
}
diff --git a/yarn-project/key-store/tsconfig.json b/yarn-project/key-store/tsconfig.json
index 1820488d409..76107a492b5 100644
--- a/yarn-project/key-store/tsconfig.json
+++ b/yarn-project/key-store/tsconfig.json
@@ -6,6 +6,9 @@
"tsBuildInfoFile": ".tsbuildinfo"
},
"references": [
+ {
+ "path": "../kv-store"
+ },
{
"path": "../circuits.js"
},
diff --git a/yarn-project/kv-store/.eslintrc.cjs b/yarn-project/kv-store/.eslintrc.cjs
new file mode 100644
index 00000000000..e659927475c
--- /dev/null
+++ b/yarn-project/kv-store/.eslintrc.cjs
@@ -0,0 +1 @@
+module.exports = require('@aztec/foundation/eslint');
diff --git a/yarn-project/kv-store/README.md b/yarn-project/kv-store/README.md
new file mode 100644
index 00000000000..37c15c72f12
--- /dev/null
+++ b/yarn-project/kv-store/README.md
@@ -0,0 +1,10 @@
+# KV Store
+
+The Aztec KV store is an implementation of a durable key-value database with a pluggable backend. THe only supported backend right now is LMDB by using the [`lmdb-js` package](https://github.com/kriszyp/lmdb-js).
+
+This package exports a number of primitive data structures that can be used to build domain-specific databases in each node component (e.g. a PXE database or an Archiver database). The data structures supported:
+
+- singleton - holds a single value. Great for when a value needs to be stored but it's not a collection (e.g. the latest block header or the length of an array)
+- array - works like a normal in-memory JS array. It can't contain holes and it can be used as a stack (push-pop mechanics).
+- map - a hashmap where keys can be numbers or strings
+- multi-map - just like a map but each key holds multiple values. Can be used for indexing into other data structures
diff --git a/yarn-project/kv-store/package.json b/yarn-project/kv-store/package.json
new file mode 100644
index 00000000000..8aea043378d
--- /dev/null
+++ b/yarn-project/kv-store/package.json
@@ -0,0 +1,50 @@
+{
+ "name": "@aztec/kv-store",
+ "version": "0.1.0",
+ "type": "module",
+ "exports": "./dest/index.js",
+ "scripts": {
+ "build": "yarn clean && tsc -b",
+ "build:dev": "tsc -b --watch",
+ "clean": "rm -rf ./dest .tsbuildinfo",
+ "formatting": "run -T prettier --check ./src && run -T eslint ./src",
+ "formatting:fix": "run -T eslint --fix ./src && run -T prettier -w ./src",
+ "test": "NODE_NO_WARNINGS=1 node --experimental-vm-modules $(yarn bin jest) --passWithNoTests",
+ "start": "DEBUG='aztec:*' && node ./dest/bin/index.js"
+ },
+ "inherits": [
+ "../package.common.json"
+ ],
+ "jest": {
+ "preset": "ts-jest/presets/default-esm",
+ "moduleNameMapper": {
+ "^(\\.{1,2}/.*)\\.[cm]?js$": "$1"
+ },
+ "testRegex": "./src/.*\\.test\\.(js|mjs|ts)$",
+ "rootDir": "./src",
+ "workerThreads": true
+ },
+ "dependencies": {
+ "@aztec/foundation": "workspace:^",
+ "lmdb": "^2.9.1"
+ },
+ "devDependencies": {
+ "@jest/globals": "^29.5.0",
+ "@types/jest": "^29.5.0",
+ "@types/node": "^18.7.23",
+ "jest": "^29.5.0",
+ "jest-mock-extended": "^3.0.3",
+ "ts-jest": "^29.1.0",
+ "ts-node": "^10.9.1",
+ "typescript": "^5.0.4"
+ },
+ "files": [
+ "dest",
+ "src",
+ "!*.test.*"
+ ],
+ "types": "./dest/index.d.ts",
+ "engines": {
+ "node": ">=18"
+ }
+}
diff --git a/yarn-project/kv-store/src/index.ts b/yarn-project/kv-store/src/index.ts
new file mode 100644
index 00000000000..2a71333f9e6
--- /dev/null
+++ b/yarn-project/kv-store/src/index.ts
@@ -0,0 +1,5 @@
+export * from './interfaces/array.js';
+export * from './interfaces/map.js';
+export * from './interfaces/singleton.js';
+export * from './interfaces/store.js';
+export * from './lmdb/store.js';
diff --git a/yarn-project/kv-store/src/interfaces/array.ts b/yarn-project/kv-store/src/interfaces/array.ts
new file mode 100644
index 00000000000..e2492204212
--- /dev/null
+++ b/yarn-project/kv-store/src/interfaces/array.ts
@@ -0,0 +1,54 @@
+/**
+ * An array backed by a persistent store. Can not have any holes in it.
+ */
+export interface AztecArray<T> {
+ /**
+ * The size of the array
+ */
+ length: number;
+
+ /**
+ * Pushes values to the end of the array
+ * @param vals - The values to push to the end of the array
+ * @returns The new length of the array
+ */
+ push(...vals: T[]): Promise<number>;
+
+ /**
+ * Pops a value from the end of the array.
+ * @returns The value that was popped, or undefined if the array was empty
+ */
+ pop(): Promise<T | undefined>;
+
+ /**
+ * Gets the value at the given index. Index can be in the range [-length, length - 1).
+ * If the index is negative, it will be treated as an offset from the end of the array.
+ *
+ * @param index - The index to get the value from
+ * @returns The value at the given index or undefined if the index is out of bounds
+ */
+ at(index: number): T | undefined;
+
+ /**
+ * Updates the value at the given index. Index can be in the range [-length, length - 1).
+ * @param index - The index to set the value at
+ * @param val - The value to set
+ * @returns Whether the value was set
+ */
+ setAt(index: number, val: T): Promise<boolean>;
+
+ /**
+ * Iterates over the array with indexes.
+ */
+ entries(): IterableIterator<[number, T]>;
+
+ /**
+ * Iterates over the array.
+ */
+ values(): IterableIterator<T>;
+
+ /**
+ * Iterates over the array.
+ */
+ [Symbol.iterator](): IterableIterator<T>;
+}
diff --git a/yarn-project/kv-store/src/interfaces/map.ts b/yarn-project/kv-store/src/interfaces/map.ts
new file mode 100644
index 00000000000..8de773837b7
--- /dev/null
+++ b/yarn-project/kv-store/src/interfaces/map.ts
@@ -0,0 +1,70 @@
+/**
+ * A map backed by a persistent store.
+ */
+export interface AztecMap<K extends string | number, V> {
+ /**
+ * Gets the value at the given key.
+ * @param key - The key to get the value from
+ */
+ get(key: K): V | undefined;
+
+ /**
+ * Checks if a key exists in the map.
+ * @param key - The key to check
+ * @returns True if the key exists, false otherwise
+ */
+ has(key: K): boolean;
+
+ /**
+ * Sets the value at the given key.
+ * @param key - The key to set the value at
+ * @param val - The value to set
+ */
+ set(key: K, val: V): Promise<boolean>;
+
+ /**
+ * Sets the value at the given key if it does not already exist.
+ * @param key - The key to set the value at
+ * @param val - The value to set
+ */
+ setIfNotExists(key: K, val: V): Promise<boolean>;
+
+ /**
+ * Deletes the value at the given key.
+ * @param key - The key to delete the value at
+ */
+ delete(key: K): Promise<boolean>;
+
+ /**
+ * Iterates over the map's key-value entries
+ */
+ entries(): IterableIterator<[K, V]>;
+
+ /**
+ * Iterates over the map's values
+ */
+ values(): IterableIterator<V>;
+
+ /**
+ * Iterates over the map's keys
+ */
+ keys(): IterableIterator<K>;
+}
+
+/**
+ * A map backed by a persistent store that can have multiple values for a single key.
+ */
+export interface AztecMultiMap<K extends string | number, V> extends AztecMap<K, V> {
+ /**
+ * Gets all the values at the given key.
+ * @param key - The key to get the values from
+ */
+ getValues(key: K): IterableIterator<V>;
+
+ /**
+ * Deletes a specific value at the given key.
+ * @param key - The key to delete the value at
+ * @param val - The value to delete
+ */
+ deleteValue(key: K, val: V): Promise<void>;
+}
diff --git a/yarn-project/kv-store/src/interfaces/singleton.ts b/yarn-project/kv-store/src/interfaces/singleton.ts
new file mode 100644
index 00000000000..43b34aa0ad8
--- /dev/null
+++ b/yarn-project/kv-store/src/interfaces/singleton.ts
@@ -0,0 +1,20 @@
+/**
+ * Represents a singleton value in the database.
+ */
+export interface AztecSingleton<T> {
+ /**
+ * Gets the value.
+ */
+ get(): T | undefined;
+
+ /**
+ * Sets the value.
+ * @param val - The new value
+ */
+ set(val: T): Promise<boolean>;
+
+ /**
+ * Deletes the value.
+ */
+ delete(): Promise<boolean>;
+}
diff --git a/yarn-project/kv-store/src/interfaces/store.ts b/yarn-project/kv-store/src/interfaces/store.ts
new file mode 100644
index 00000000000..d7ccfa3cd29
--- /dev/null
+++ b/yarn-project/kv-store/src/interfaces/store.ts
@@ -0,0 +1,40 @@
+import { AztecArray } from './array.js';
+import { AztecMap, AztecMultiMap } from './map.js';
+import { AztecSingleton } from './singleton.js';
+
+/** A key-value store */
+export interface AztecKVStore {
+ /**
+ * Creates a new map.
+ * @param name - The name of the map
+ * @returns The map
+ */
+ createMap<K extends string | number, V>(name: string): AztecMap<K, V>;
+
+ /**
+ * Creates a new multi-map.
+ * @param name - The name of the multi-map
+ * @returns The multi-map
+ */
+ createMultiMap<K extends string | number, V>(name: string): AztecMultiMap<K, V>;
+
+ /**
+ * Creates a new array.
+ * @param name - The name of the array
+ * @returns The array
+ */
+ createArray<T>(name: string): AztecArray<T>;
+
+ /**
+ * Creates a new singleton.
+ * @param name - The name of the singleton
+ * @returns The singleton
+ */
+ createSingleton<T>(name: string): AztecSingleton<T>;
+
+ /**
+ * Starts a transaction. All calls to read/write data while in a transaction are queued and executed atomically.
+ * @param callback - The callback to execute in a transaction
+ */
+ transaction<T extends Exclude<any, Promise<any>>>(callback: () => T): Promise<T>;
+}
diff --git a/yarn-project/kv-store/src/lmdb/array.test.ts b/yarn-project/kv-store/src/lmdb/array.test.ts
new file mode 100644
index 00000000000..3058302e87f
--- /dev/null
+++ b/yarn-project/kv-store/src/lmdb/array.test.ts
@@ -0,0 +1,91 @@
+import { Database, open } from 'lmdb';
+
+import { LmdbAztecArray } from './array.js';
+
+describe('LmdbAztecArray', () => {
+ let db: Database;
+ let arr: LmdbAztecArray<number>;
+
+ beforeEach(() => {
+ db = open({} as any);
+ arr = new LmdbAztecArray(db, 'test');
+ });
+
+ it('should be able to push and pop values', async () => {
+ await arr.push(1);
+ await arr.push(2);
+ await arr.push(3);
+
+ expect(arr.length).toEqual(3);
+ expect(await arr.pop()).toEqual(3);
+ expect(await arr.pop()).toEqual(2);
+ expect(await arr.pop()).toEqual(1);
+ expect(await arr.pop()).toEqual(undefined);
+ });
+
+ it('should be able to get values by index', async () => {
+ await arr.push(1);
+ await arr.push(2);
+ await arr.push(3);
+
+ expect(arr.at(0)).toEqual(1);
+ expect(arr.at(1)).toEqual(2);
+ expect(arr.at(2)).toEqual(3);
+ expect(arr.at(3)).toEqual(undefined);
+ expect(arr.at(-1)).toEqual(3);
+ expect(arr.at(-2)).toEqual(2);
+ expect(arr.at(-3)).toEqual(1);
+ expect(arr.at(-4)).toEqual(undefined);
+ });
+
+ it('should be able to set values by index', async () => {
+ await arr.push(1);
+ await arr.push(2);
+ await arr.push(3);
+
+ expect(await arr.setAt(0, 4)).toEqual(true);
+ expect(await arr.setAt(1, 5)).toEqual(true);
+ expect(await arr.setAt(2, 6)).toEqual(true);
+
+ expect(await arr.setAt(3, 7)).toEqual(false);
+
+ expect(arr.at(0)).toEqual(4);
+ expect(arr.at(1)).toEqual(5);
+ expect(arr.at(2)).toEqual(6);
+ expect(arr.at(3)).toEqual(undefined);
+
+ expect(await arr.setAt(-1, 8)).toEqual(true);
+ expect(await arr.setAt(-2, 9)).toEqual(true);
+ expect(await arr.setAt(-3, 10)).toEqual(true);
+
+ expect(await arr.setAt(-4, 11)).toEqual(false);
+
+ expect(arr.at(-1)).toEqual(8);
+ expect(arr.at(-2)).toEqual(9);
+ expect(arr.at(-3)).toEqual(10);
+ expect(arr.at(-4)).toEqual(undefined);
+ });
+
+ it('should be able to iterate over values', async () => {
+ await arr.push(1);
+ await arr.push(2);
+ await arr.push(3);
+
+ expect([...arr.values()]).toEqual([1, 2, 3]);
+ expect([...arr.entries()]).toEqual([
+ [0, 1],
+ [1, 2],
+ [2, 3],
+ ]);
+ });
+
+ it('should be able to restore state', async () => {
+ await arr.push(1);
+ await arr.push(2);
+ await arr.push(3);
+
+ const arr2 = new LmdbAztecArray(db, 'test');
+ expect(arr2.length).toEqual(3);
+ expect([...arr2.values()]).toEqual([...arr.values()]);
+ });
+});
diff --git a/yarn-project/kv-store/src/lmdb/array.ts b/yarn-project/kv-store/src/lmdb/array.ts
new file mode 100644
index 00000000000..1da3676aac0
--- /dev/null
+++ b/yarn-project/kv-store/src/lmdb/array.ts
@@ -0,0 +1,109 @@
+import { Database, Key } from 'lmdb';
+
+import { AztecArray } from '../interfaces/array.js';
+import { LmdbAztecSingleton } from './singleton.js';
+
+/** The shape of a key that stores a value in an array */
+type ArrayIndexSlot = ['array', string, 'slot', number];
+
+/**
+ * An persistent array backed by LMDB.
+ */
+export class LmdbAztecArray<T> implements AztecArray<T> {
+ #db: Database<T, ArrayIndexSlot>;
+ #name: string;
+ #length: LmdbAztecSingleton<number>;
+
+ constructor(db: Database<unknown, Key>, arrName: string) {
+ this.#name = arrName;
+ this.#length = new LmdbAztecSingleton(db, `${arrName}:meta:length`);
+ this.#db = db as Database<T, ArrayIndexSlot>;
+ }
+
+ get length(): number {
+ return this.#length.get() ?? 0;
+ }
+
+ push(...vals: T[]): Promise<number> {
+ return this.#db.childTransaction(() => {
+ let length = this.length;
+ for (const val of vals) {
+ void this.#db.put(this.#slot(length), val);
+ length += 1;
+ }
+
+ void this.#length.set(length);
+
+ return length;
+ });
+ }
+
+ pop(): Promise<T | undefined> {
+ return this.#db.childTransaction(() => {
+ const length = this.length;
+ if (length === 0) {
+ return undefined;
+ }
+
+ const slot = this.#slot(length - 1);
+ const val = this.#db.get(slot) as T;
+
+ void this.#db.remove(slot);
+ void this.#length.set(length - 1);
+
+ return val;
+ });
+ }
+
+ at(index: number): T | undefined {
+ if (index < 0) {
+ index = this.length + index;
+ }
+
+ // the Array API only accepts indexes in the range [-this.length, this.length)
+ // so if after normalizing the index is still out of range, return undefined
+ if (index < 0 || index >= this.length) {
+ return undefined;
+ }
+
+ return this.#db.get(this.#slot(index));
+ }
+
+ setAt(index: number, val: T): Promise<boolean> {
+ if (index < 0) {
+ index = this.length + index;
+ }
+
+ if (index < 0 || index >= this.length) {
+ return Promise.resolve(false);
+ }
+
+ return this.#db.put(this.#slot(index), val);
+ }
+
+ *entries(): IterableIterator<[number, T]> {
+ const values = this.#db.getRange({
+ start: this.#slot(0),
+ limit: this.length,
+ });
+
+ for (const { key, value } of values) {
+ const index = key[3];
+ yield [index, value];
+ }
+ }
+
+ *values(): IterableIterator<T> {
+ for (const [_, value] of this.entries()) {
+ yield value;
+ }
+ }
+
+ [Symbol.iterator](): IterableIterator<T> {
+ return this.values();
+ }
+
+ #slot(index: number): ArrayIndexSlot {
+ return ['array', this.#name, 'slot', index];
+ }
+}
diff --git a/yarn-project/kv-store/src/lmdb/map.test.ts b/yarn-project/kv-store/src/lmdb/map.test.ts
new file mode 100644
index 00000000000..5319e0a26c3
--- /dev/null
+++ b/yarn-project/kv-store/src/lmdb/map.test.ts
@@ -0,0 +1,72 @@
+import { Database, open } from 'lmdb';
+
+import { LmdbAztecMap } from './map.js';
+
+describe('LmdbAztecMap', () => {
+ let db: Database;
+ let map: LmdbAztecMap<string, string>;
+
+ beforeEach(() => {
+ db = open({ dupSort: true } as any);
+ map = new LmdbAztecMap(db, 'test');
+ });
+
+ it('should be able to set and get values', async () => {
+ await map.set('foo', 'bar');
+ await map.set('baz', 'qux');
+
+ expect(map.get('foo')).toEqual('bar');
+ expect(map.get('baz')).toEqual('qux');
+ expect(map.get('quux')).toEqual(undefined);
+ });
+
+ it('should be able to set values if they do not exist', async () => {
+ expect(await map.setIfNotExists('foo', 'bar')).toEqual(true);
+ expect(await map.setIfNotExists('foo', 'baz')).toEqual(false);
+
+ expect(map.get('foo')).toEqual('bar');
+ });
+
+ it('should be able to delete values', async () => {
+ await map.set('foo', 'bar');
+ await map.set('baz', 'qux');
+
+ expect(await map.delete('foo')).toEqual(true);
+
+ expect(map.get('foo')).toEqual(undefined);
+ expect(map.get('baz')).toEqual('qux');
+ });
+
+ it('should be able to iterate over entries', async () => {
+ await map.set('foo', 'bar');
+ await map.set('baz', 'qux');
+
+ expect([...map.entries()]).toEqual(
+ expect.arrayContaining([
+ ['foo', 'bar'],
+ ['baz', 'qux'],
+ ]),
+ );
+ });
+
+ it('should be able to iterate over values', async () => {
+ await map.set('foo', 'bar');
+ await map.set('baz', 'qux');
+
+ expect([...map.values()]).toEqual(expect.arrayContaining(['bar', 'qux']));
+ });
+
+ it('should be able to iterate over keys', async () => {
+ await map.set('foo', 'bar');
+ await map.set('baz', 'qux');
+
+ expect([...map.keys()]).toEqual(expect.arrayContaining(['foo', 'baz']));
+ });
+
+ it('should be able to get multiple values for a single key', async () => {
+ await map.set('foo', 'bar');
+ await map.set('foo', 'baz');
+
+ expect([...map.getValues('foo')]).toEqual(['bar', 'baz']);
+ });
+});
diff --git a/yarn-project/kv-store/src/lmdb/map.ts b/yarn-project/kv-store/src/lmdb/map.ts
new file mode 100644
index 00000000000..b883b809738
--- /dev/null
+++ b/yarn-project/kv-store/src/lmdb/map.ts
@@ -0,0 +1,88 @@
+import { Database, Key } from 'lmdb';
+
+import { AztecMultiMap } from '../interfaces/map.js';
+
+/** The slot where a key-value entry would be stored */
+type MapKeyValueSlot<K extends string | number> = ['map', string, 'slot', K];
+
+/**
+ * A map backed by LMDB.
+ */
+export class LmdbAztecMap<K extends string | number, V> implements AztecMultiMap<K, V> {
+ protected db: Database<V, MapKeyValueSlot<K>>;
+ protected name: string;
+
+ constructor(rootDb: Database<unknown, Key>, mapName: string) {
+ this.name = mapName;
+ this.db = rootDb as Database<V, MapKeyValueSlot<K>>;
+ }
+
+ close(): Promise<void> {
+ return this.db.close();
+ }
+
+ get(key: K): V | undefined {
+ return this.db.get(this.#slot(key)) as V | undefined;
+ }
+
+ *getValues(key: K): IterableIterator<V> {
+ const values = this.db.getValues(this.#slot(key));
+ for (const value of values) {
+ yield value;
+ }
+ }
+
+ has(key: K): boolean {
+ return this.db.doesExist(this.#slot(key));
+ }
+
+ set(key: K, val: V): Promise<boolean> {
+ return this.db.put(this.#slot(key), val);
+ }
+
+ setIfNotExists(key: K, val: V): Promise<boolean> {
+ const slot = this.#slot(key);
+ return this.db.ifNoExists(slot, () => {
+ void this.db.put(slot, val);
+ });
+ }
+
+ delete(key: K): Promise<boolean> {
+ return this.db.remove(this.#slot(key));
+ }
+
+ async deleteValue(key: K, val: V): Promise<void> {
+ await this.db.remove(this.#slot(key), val);
+ }
+
+ *entries(): IterableIterator<[K, V]> {
+ const iterator = this.db.getRange({
+ start: ['map', this.name, 'slot'],
+ });
+
+ for (const { key, value } of iterator) {
+ if (key[0] !== 'map' || key[1] !== this.name) {
+ break;
+ }
+
+ const originalKey = key[3];
+ yield [originalKey, value];
+ }
+ }
+
+ *values(): IterableIterator<V> {
+ for (const [_, value] of this.entries()) {
+ yield value;
+ }
+ }
+
+ *keys(): IterableIterator<K> {
+ for (const [key, _] of this.entries()) {
+ yield key;
+ }
+ }
+
+ #slot(key: K): MapKeyValueSlot<K> {
+ return ['map', this.name, 'slot', key];
+ }
+}
diff --git a/yarn-project/kv-store/src/lmdb/singleton.test.ts b/yarn-project/kv-store/src/lmdb/singleton.test.ts
new file mode 100644
index 00000000000..de1eefae462
--- /dev/null
+++ b/yarn-project/kv-store/src/lmdb/singleton.test.ts
@@ -0,0 +1,25 @@
+import { open } from 'lmdb';
+
+import { LmdbAztecSingleton } from './singleton.js';
+
+describe('LmdbAztecSingleton', () => {
+ let singleton: LmdbAztecSingleton<string>;
+ beforeEach(() => {
+ singleton = new LmdbAztecSingleton(open({} as any), 'test');
+ });
+
+ it('returns undefined if the value is not set', () => {
+ expect(singleton.get()).toEqual(undefined);
+ });
+
+ it('should be able to set and get values', async () => {
+ expect(await singleton.set('foo')).toEqual(true);
+ expect(singleton.get()).toEqual('foo');
+ });
+
+ it('overwrites the value if it is set again', async () => {
+ expect(await singleton.set('foo')).toEqual(true);
+ expect(await singleton.set('bar')).toEqual(true);
+ expect(singleton.get()).toEqual('bar');
+ });
+});
diff --git a/yarn-project/kv-store/src/lmdb/singleton.ts b/yarn-project/kv-store/src/lmdb/singleton.ts
new file mode 100644
index 00000000000..0fa4ffe69e4
--- /dev/null
+++ b/yarn-project/kv-store/src/lmdb/singleton.ts
@@ -0,0 +1,31 @@
+import { Database, Key } from 'lmdb';
+
+import { AztecSingleton } from '../interfaces/singleton.js';
+
+/** The slot where this singleton will store its value */
+type ValueSlot = ['singleton', string, 'value'];
+
+/**
+ * Stores a single value in LMDB.
+ */
+export class LmdbAztecSingleton<T> implements AztecSingleton<T> {
+ #db: Database<T, ValueSlot>;
+ #slot: ValueSlot;
+
+ constructor(db: Database<unknown, Key>, name: string) {
+ this.#db = db as Database<T, ValueSlot>;
+ this.#slot = ['singleton', name, 'value'];
+ }
+
+ get(): T | undefined {
+ return this.#db.get(this.#slot);
+ }
+
+ set(val: T): Promise<boolean> {
+ return this.#db.put(this.#slot, val);
+ }
+
+ delete(): Promise<boolean> {
+ return this.#db.remove(this.#slot);
+ }
+}
diff --git a/yarn-project/kv-store/src/lmdb/store.ts b/yarn-project/kv-store/src/lmdb/store.ts
new file mode 100644
index 00000000000..9ede111a875
--- /dev/null
+++ b/yarn-project/kv-store/src/lmdb/store.ts
@@ -0,0 +1,131 @@
+import { EthAddress } from '@aztec/foundation/eth-address';
+import { Logger, createDebugLogger } from '@aztec/foundation/log';
+
+import { Database, Key, RootDatabase, open } from 'lmdb';
+
+import { AztecArray } from '../interfaces/array.js';
+import { AztecMap, AztecMultiMap } from '../interfaces/map.js';
+import { AztecSingleton } from '../interfaces/singleton.js';
+import { AztecKVStore } from '../interfaces/store.js';
+import { LmdbAztecArray } from './array.js';
+import { LmdbAztecMap } from './map.js';
+import { LmdbAztecSingleton } from './singleton.js';
+
+/**
+ * A key-value store backed by LMDB.
+ */
+export class AztecLmdbStore implements AztecKVStore {
+ #rootDb: RootDatabase;
+ #data: Database<unknown, Key>;
+ #multiMapData: Database<unknown, Key>;
+ #rollupAddress: AztecSingleton<string>;
+ #log: Logger;
+
+ constructor(rootDb: RootDatabase, log: Logger) {
+ this.#rootDb = rootDb;
+ this.#log = log;
+
+ // big bucket to store all the data
+ this.#data = rootDb.openDB('data', {
+ encoding: 'msgpack',
+ keyEncoding: 'ordered-binary',
+ });
+
+ this.#multiMapData = rootDb.openDB('data_dup_sort', {
+ encoding: 'msgpack',
+ keyEncoding: 'ordered-binary',
+ dupSort: true,
+ });
+
+ this.#rollupAddress = this.createSingleton('rollupAddress');
+ }
+
+ /**
+ * Creates a new AztecKVStore backed by LMDB. The path to the database is optional. If not provided,
+ * the database will be stored in a temporary location and be deleted when the process exists.
+ *
+ * The `rollupAddress` passed is checked against what is stored in the database. If they do not match,
+ * the database is cleared before returning the store. This way data is not accidentally shared between
+ * different rollup instances.
+ *
+ * @param rollupAddress - The ETH address of the rollup contract
+ * @param path - A path on the disk to store the database. Optional
+ * @param log - A logger to use. Optional
+ * @returns The store
+ */
+ static async create(
+ rollupAddress: EthAddress,
+ path?: string,
+ log = createDebugLogger('aztec:kv-store:lmdb'),
+ ): Promise<AztecLmdbStore> {
+ log.info(`Opening LMDB database at ${path || 'temporary location'}`);
+
+ const rootDb = open({
+ path,
+ });
+
+ const db = new AztecLmdbStore(rootDb, log);
+ await db.#init(rollupAddress);
+
+ return db;
+ }
+
+ /**
+ * Creates a new AztecMap in the store.
+ * @param name - Name of the map
+ * @returns A new AztecMap
+ */
+ createMap<K extends string | number, V>(name: string): AztecMap<K, V> {
+ return new LmdbAztecMap(this.#data, name);
+ }
+
+ /**
+ * Creates a new AztecMultiMap in the store. A multi-map stores multiple values for a single key automatically.
+ * @param name - Name of the map
+ * @returns A new AztecMultiMap
+ */
+ createMultiMap<K extends string | number, V>(name: string): AztecMultiMap<K, V> {
+ return new LmdbAztecMap(this.#multiMapData, name);
+ }
+
+ /**
+ * Creates a new AztecArray in the store.
+ * @param name - Name of the array
+ * @returns A new AztecArray
+ */
+ createArray<T>(name: string): AztecArray<T> {
+ return new LmdbAztecArray(this.#data, name);
+ }
+
+ /**
+ * Creates a new AztecSingleton in the store.
+ * @param name - Name of the singleton
+ * @returns A new AztecSingleton
+ */
+ createSingleton<T>(name: string): AztecSingleton<T> {
+ return new LmdbAztecSingleton(this.#data, name);
+ }
+
+ /**
+ * Runs a callback in a transaction.
+ * @param callback - Function to execute in a transaction
+ * @returns A promise that resolves to the return value of the callback
+ */
+ transaction<T>(callback: () => T): Promise<T> {
+ return this.#rootDb.transaction(callback);
+ }
+
+ async #init(rollupAddress: EthAddress): Promise<void> {
+ const storedRollupAddress = this.#rollupAddress.get();
+ const rollupAddressString = rollupAddress.toString();
+
+ if (typeof storedRollupAddress === 'string' && rollupAddressString !== storedRollupAddress) {
+ this.#log.warn(
+ `Rollup address mismatch: expected ${rollupAddress}, found ${storedRollupAddress}. Clearing entire database...`,
+ );
+ await this.#rootDb.clearAsync();
+ }
+
+ await this.#rollupAddress.set(rollupAddressString);
+ }
+}
diff --git a/yarn-project/kv-store/tsconfig.json b/yarn-project/kv-store/tsconfig.json
new file mode 100644
index 00000000000..63f8ab3e9f7
--- /dev/null
+++ b/yarn-project/kv-store/tsconfig.json
@@ -0,0 +1,14 @@
+{
+ "extends": "..",
+ "compilerOptions": {
+ "outDir": "dest",
+ "rootDir": "src",
+ "tsBuildInfoFile": ".tsbuildinfo"
+ },
+ "references": [
+ {
+ "path": "../foundation"
+ }
+ ],
+ "include": ["src"]
+}
diff --git a/yarn-project/package.json b/yarn-project/package.json
index 089ce942376..dd6de60bbd8 100644
--- a/yarn-project/package.json
+++ b/yarn-project/package.json
@@ -47,7 +47,8 @@
"scripts",
"types",
"world-state",
- "yarn-project-base"
+ "yarn-project-base",
+ "kv-store"
],
"prettier": "@aztec/foundation/prettier",
"devDependencies": {
diff --git a/yarn-project/pxe/package.json b/yarn-project/pxe/package.json
index b426db2c3ed..b1892b3b1f3 100644
--- a/yarn-project/pxe/package.json
+++ b/yarn-project/pxe/package.json
@@ -29,7 +29,8 @@
"^(\\.{1,2}/.*)\\.[cm]?js$": "$1"
},
"testRegex": "./src/.*\\.test\\.(js|mjs|ts)$",
- "rootDir": "./src"
+ "rootDir": "./src",
+ "workerThreads": true
},
"dependencies": {
"@aztec/acir-simulator": "workspace:^",
@@ -37,6 +38,7 @@
"@aztec/ethereum": "workspace:^",
"@aztec/foundation": "workspace:^",
"@aztec/key-store": "workspace:^",
+ "@aztec/kv-store": "workspace:^",
"@aztec/noir-compiler": "workspace:^",
"@aztec/noir-protocol-circuits": "workspace:^",
"@aztec/types": "workspace:^",
diff --git a/yarn-project/pxe/src/config/index.ts b/yarn-project/pxe/src/config/index.ts
index 8c511ce4fa0..e96a5e64519 100644
--- a/yarn-project/pxe/src/config/index.ts
+++ b/yarn-project/pxe/src/config/index.ts
@@ -12,17 +12,21 @@ export interface PXEServiceConfig {
l2BlockPollingIntervalMS: number;
/** L2 block to start scanning from */
l2StartingBlock: number;
+
+ /** Where to store PXE data. If not set will store in memory */
+ dataDirectory?: string;
}
/**
* Creates an instance of PXEServiceConfig out of environment variables using sensible defaults for integration testing if not set.
*/
export function getPXEServiceConfig(): PXEServiceConfig {
- const { PXE_BLOCK_POLLING_INTERVAL_MS, PXE_L2_STARTING_BLOCK } = process.env;
+ const { PXE_BLOCK_POLLING_INTERVAL_MS, PXE_L2_STARTING_BLOCK, DATA_DIRECTORY } = process.env;
return {
l2BlockPollingIntervalMS: PXE_BLOCK_POLLING_INTERVAL_MS ? +PXE_BLOCK_POLLING_INTERVAL_MS : 1000,
l2StartingBlock: PXE_L2_STARTING_BLOCK ? +PXE_L2_STARTING_BLOCK : INITIAL_L2_BLOCK_NUM,
+ dataDirectory: DATA_DIRECTORY,
};
}
diff --git a/yarn-project/pxe/src/contract_tree/index.ts b/yarn-project/pxe/src/contract_tree/index.ts
index 2c3f9de54dc..8078e310f3d 100644
--- a/yarn-project/pxe/src/contract_tree/index.ts
+++ b/yarn-project/pxe/src/contract_tree/index.ts
@@ -93,12 +93,7 @@ export class ContractTree {
const completeAddress = computeCompleteAddress(from, contractAddressSalt, root, constructorHash);
- const contractDao: ContractDao = {
- ...artifact,
- completeAddress,
- functions,
- portalContract,
- };
+ const contractDao = new ContractDao(artifact, completeAddress, portalContract);
const NewContractConstructor = {
functionData,
vkHash,
diff --git a/yarn-project/pxe/src/database/index.ts b/yarn-project/pxe/src/database/index.ts
index d1306dbafe0..35d4e000a20 100644
--- a/yarn-project/pxe/src/database/index.ts
+++ b/yarn-project/pxe/src/database/index.ts
@@ -1,2 +1,2 @@
-export * from './database.js';
+export * from './pxe_database.js';
export * from './memory_db.js';
diff --git a/yarn-project/pxe/src/database/kv_pxe_database.test.ts b/yarn-project/pxe/src/database/kv_pxe_database.test.ts
new file mode 100644
index 00000000000..a9054af0719
--- /dev/null
+++ b/yarn-project/pxe/src/database/kv_pxe_database.test.ts
@@ -0,0 +1,15 @@
+import { EthAddress } from '@aztec/circuits.js';
+import { AztecLmdbStore } from '@aztec/kv-store';
+
+import { KVPxeDatabase } from './kv_pxe_database.js';
+import { describePxeDatabase } from './pxe_database_test_suite.js';
+
+describe('KVPxeDatabase', () => {
+ let database: KVPxeDatabase;
+
+ beforeEach(async () => {
+ database = new KVPxeDatabase(await AztecLmdbStore.create(EthAddress.random()));
+ });
+
+ describePxeDatabase(() => database);
+});
diff --git a/yarn-project/pxe/src/database/kv_pxe_database.ts b/yarn-project/pxe/src/database/kv_pxe_database.ts
new file mode 100644
index 00000000000..8e5e27c2b8b
--- /dev/null
+++ b/yarn-project/pxe/src/database/kv_pxe_database.ts
@@ -0,0 +1,288 @@
+import { AztecAddress, BlockHeader, CompleteAddress } from '@aztec/circuits.js';
+import { Fr } from '@aztec/foundation/fields';
+import { AztecArray, AztecKVStore, AztecMap, AztecMultiMap, AztecSingleton } from '@aztec/kv-store';
+import { ContractDao, MerkleTreeId, NoteFilter, PublicKey } from '@aztec/types';
+
+import { NoteDao } from './note_dao.js';
+import { PxeDatabase } from './pxe_database.js';
+
+/** Serialized structure of a block header */
+type SerializedBlockHeader = {
+ /** The tree roots when the block was created */
+ roots: Record<MerkleTreeId, string>;
+ /** The hash of the global variables */
+ globalVariablesHash: string;
+};
+
+/**
+ * A PXE database backed by LMDB.
+ */
+export class KVPxeDatabase implements PxeDatabase {
+ #blockHeader: AztecSingleton<SerializedBlockHeader>;
+ #addresses: AztecArray<Buffer>;
+ #addressIndex: AztecMap<string, number>;
+ #authWitnesses: AztecMap<string, Buffer[]>;
+ #capsules: AztecArray<Buffer[]>;
+ #contracts: AztecMap<string, Buffer>;
+ #notes: AztecArray<Buffer>;
+ #nullifiedNotes: AztecMap<number, boolean>;
+ #notesByContract: AztecMultiMap<string, number>;
+ #notesByStorageSlot: AztecMultiMap<string, number>;
+ #notesByTxHash: AztecMultiMap<string, number>;
+ #notesByOwner: AztecMultiMap<string, number>;
+ #db: AztecKVStore;
+
+ constructor(db: AztecKVStore) {
+ this.#db = db;
+
+ this.#addresses = db.createArray('addresses');
+ this.#addressIndex = db.createMap('address_index');
+
+ this.#authWitnesses = db.createMap('auth_witnesses');
+ this.#capsules = db.createArray('capsules');
+ this.#blockHeader = db.createSingleton('block_header');
+ this.#contracts = db.createMap('contracts');
+
+ this.#notes = db.createArray('notes');
+ this.#nullifiedNotes = db.createMap('nullified_notes');
+
+ this.#notesByContract = db.createMultiMap('notes_by_contract');
+ this.#notesByStorageSlot = db.createMultiMap('notes_by_storage_slot');
+ this.#notesByTxHash = db.createMultiMap('notes_by_tx_hash');
+ this.#notesByOwner = db.createMultiMap('notes_by_owner');
+ }
+
+ async addAuthWitness(messageHash: Fr, witness: Fr[]): Promise<void> {
+ await this.#authWitnesses.set(
+ messageHash.toString(),
+ witness.map(w => w.toBuffer()),
+ );
+ }
+
+ getAuthWitness(messageHash: Fr): Promise<Fr[] | undefined> {
+ const witness = this.#authWitnesses.get(messageHash.toString());
+ return Promise.resolve(witness?.map(w => Fr.fromBuffer(w)));
+ }
+
+ async addCapsule(capsule: Fr[]): Promise<void> {
+ await this.#capsules.push(capsule.map(c => c.toBuffer()));
+ }
+
+ async popCapsule(): Promise<Fr[] | undefined> {
+ const val = await this.#capsules.pop();
+ return val?.map(b => Fr.fromBuffer(b));
+ }
+
+ async addNote(note: NoteDao): Promise<void> {
+ await this.addNotes([note]);
+ }
+
+ async addNotes(notes: NoteDao[]): Promise<void> {
+ const newLength = await this.#notes.push(...notes.map(note => note.toBuffer()));
+ for (const [index, note] of notes.entries()) {
+ const noteId = newLength - notes.length + index;
+ await Promise.all([
+ this.#notesByContract.set(note.contractAddress.toString(), noteId),
+ this.#notesByStorageSlot.set(note.storageSlot.toString(), noteId),
+ this.#notesByTxHash.set(note.txHash.toString(), noteId),
+ this.#notesByOwner.set(note.publicKey.toString(), noteId),
+ ]);
+ }
+ }
+
+ *#getAllNonNullifiedNotes(): IterableIterator<NoteDao> {
+ for (const [index, serialized] of this.#notes.entries()) {
+ if (this.#nullifiedNotes.has(index)) {
+ continue;
+ }
+
+ yield NoteDao.fromBuffer(serialized);
+ }
+ }
+
+ async getNotes(filter: NoteFilter): Promise<NoteDao[]> {
+ const publicKey: PublicKey | undefined = filter.owner
+ ? (await this.getCompleteAddress(filter.owner))?.publicKey
+ : undefined;
+
+ const initialNoteIds = publicKey
+ ? this.#notesByOwner.getValues(publicKey.toString())
+ : filter.txHash
+ ? this.#notesByTxHash.getValues(filter.txHash.toString())
+ : filter.contractAddress
+ ? this.#notesByContract.getValues(filter.contractAddress.toString())
+ : filter.storageSlot
+ ? this.#notesByStorageSlot.getValues(filter.storageSlot.toString())
+ : undefined;
+
+ if (!initialNoteIds) {
+ return Array.from(this.#getAllNonNullifiedNotes());
+ }
+
+ const result: NoteDao[] = [];
+ for (const noteId of initialNoteIds) {
+ const serializedNote = this.#notes.at(noteId);
+ if (!serializedNote) {
+ continue;
+ }
+
+ const note = NoteDao.fromBuffer(serializedNote);
+ if (filter.contractAddress && !note.contractAddress.equals(filter.contractAddress)) {
+ continue;
+ }
+
+ if (filter.txHash && !note.txHash.equals(filter.txHash)) {
+ continue;
+ }
+
+ if (filter.storageSlot && !note.storageSlot.equals(filter.storageSlot!)) {
+ continue;
+ }
+
+ if (publicKey && !note.publicKey.equals(publicKey)) {
+ continue;
+ }
+
+ result.push(note);
+ }
+
+ return result;
+ }
+
+ removeNullifiedNotes(nullifiers: Fr[], account: PublicKey): Promise<NoteDao[]> {
+ const nullifierSet = new Set(nullifiers.map(n => n.toString()));
+ return this.#db.transaction(() => {
+ const notesIds = this.#notesByOwner.getValues(account.toString());
+ const nullifiedNotes: NoteDao[] = [];
+
+ for (const noteId of notesIds) {
+ const note = NoteDao.fromBuffer(this.#notes.at(noteId)!);
+ if (nullifierSet.has(note.siloedNullifier.toString())) {
+ nullifiedNotes.push(note);
+
+ void this.#nullifiedNotes.set(noteId, true);
+ void this.#notesByOwner.deleteValue(account.toString(), noteId);
+ void this.#notesByTxHash.deleteValue(note.txHash.toString(), noteId);
+ void this.#notesByContract.deleteValue(note.contractAddress.toString(), noteId);
+ void this.#notesByStorageSlot.deleteValue(note.storageSlot.toString(), noteId);
+ }
+ }
+
+ return nullifiedNotes;
+ });
+ }
+
+ getTreeRoots(): Record<MerkleTreeId, Fr> {
+ const roots = this.#blockHeader.get()?.roots;
+ if (!roots) {
+ throw new Error(`Tree roots not set`);
+ }
+
+ return {
+ [MerkleTreeId.ARCHIVE]: Fr.fromString(roots[MerkleTreeId.ARCHIVE]),
+ [MerkleTreeId.CONTRACT_TREE]: Fr.fromString(roots[MerkleTreeId.CONTRACT_TREE].toString()),
+ [MerkleTreeId.L1_TO_L2_MESSAGES_TREE]: Fr.fromString(roots[MerkleTreeId.L1_TO_L2_MESSAGES_TREE].toString()),
+ [MerkleTreeId.NOTE_HASH_TREE]: Fr.fromString(roots[MerkleTreeId.NOTE_HASH_TREE].toString()),
+ [MerkleTreeId.PUBLIC_DATA_TREE]: Fr.fromString(roots[MerkleTreeId.PUBLIC_DATA_TREE].toString()),
+ [MerkleTreeId.NULLIFIER_TREE]: Fr.fromString(roots[MerkleTreeId.NULLIFIER_TREE].toString()),
+ };
+ }
+
+ async setBlockHeader(blockHeader: BlockHeader): Promise<void> {
+ await this.#blockHeader.set({
+ globalVariablesHash: blockHeader.globalVariablesHash.toString(),
+ roots: {
+ [MerkleTreeId.NOTE_HASH_TREE]: blockHeader.noteHashTreeRoot.toString(),
+ [MerkleTreeId.NULLIFIER_TREE]: blockHeader.nullifierTreeRoot.toString(),
+ [MerkleTreeId.CONTRACT_TREE]: blockHeader.contractTreeRoot.toString(),
+ [MerkleTreeId.L1_TO_L2_MESSAGES_TREE]: blockHeader.l1ToL2MessagesTreeRoot.toString(),
+ [MerkleTreeId.ARCHIVE]: blockHeader.archiveRoot.toString(),
+ [MerkleTreeId.PUBLIC_DATA_TREE]: blockHeader.publicDataTreeRoot.toString(),
+ },
+ });
+ }
+
+ getBlockHeader(): BlockHeader {
+ const value = this.#blockHeader.get();
+ if (!value) {
+ throw new Error(`Block header not set`);
+ }
+
+ const blockHeader = new BlockHeader(
+ Fr.fromString(value.roots[MerkleTreeId.NOTE_HASH_TREE]),
+ Fr.fromString(value.roots[MerkleTreeId.NULLIFIER_TREE]),
+ Fr.fromString(value.roots[MerkleTreeId.CONTRACT_TREE]),
+ Fr.fromString(value.roots[MerkleTreeId.L1_TO_L2_MESSAGES_TREE]),
+ Fr.fromString(value.roots[MerkleTreeId.ARCHIVE]),
+ Fr.ZERO, // todo: private kernel vk tree root
+ Fr.fromString(value.roots[MerkleTreeId.PUBLIC_DATA_TREE]),
+ Fr.fromString(value.globalVariablesHash),
+ );
+
+ return blockHeader;
+ }
+
+ addCompleteAddress(completeAddress: CompleteAddress): Promise<boolean> {
+ return this.#db.transaction(() => {
+ const addressString = completeAddress.address.toString();
+ const buffer = completeAddress.toBuffer();
+ const existing = this.#addressIndex.get(addressString);
+ if (typeof existing === 'undefined') {
+ const index = this.#addresses.length;
+ void this.#addresses.push(buffer);
+ void this.#addressIndex.set(addressString, index);
+
+ return true;
+ } else {
+ const existingBuffer = this.#addresses.at(existing);
+
+ if (existingBuffer?.equals(buffer)) {
+ return false;
+ }
+
+ throw new Error(
+ `Complete address with aztec address ${addressString} but different public key or partial key already exists in memory database`,
+ );
+ }
+ });
+ }
+
+ getCompleteAddress(address: AztecAddress): Promise<CompleteAddress | undefined> {
+ const index = this.#addressIndex.get(address.toString());
+ if (typeof index === 'undefined') {
+ return Promise.resolve(undefined);
+ }
+
+ const value = this.#addresses.at(index);
+ return Promise.resolve(value ? CompleteAddress.fromBuffer(value) : undefined);
+ }
+
+ getCompleteAddresses(): Promise<CompleteAddress[]> {
+ return Promise.resolve(Array.from(this.#addresses).map(v => CompleteAddress.fromBuffer(v)));
+ }
+
+ estimateSize(): number {
+ const notesSize = Array.from(this.#getAllNonNullifiedNotes()).reduce((sum, note) => sum + note.getSize(), 0);
+ const authWitsSize = Array.from(this.#authWitnesses.values()).reduce(
+ (sum, value) => sum + value.length * Fr.SIZE_IN_BYTES,
+ 0,
+ );
+ const addressesSize = this.#addresses.length * CompleteAddress.SIZE_IN_BYTES;
+ const treeRootsSize = Object.keys(MerkleTreeId).length * Fr.SIZE_IN_BYTES;
+
+ return notesSize + treeRootsSize + authWitsSize + addressesSize;
+ }
+
+ async addContract(contract: ContractDao): Promise<void> {
+ await this.#contracts.set(contract.completeAddress.address.toString(), contract.toBuffer());
+ }
+
+ getContract(address: AztecAddress): Promise<ContractDao | undefined> {
+ const contract = this.#contracts.get(address.toString());
+ return Promise.resolve(contract ? ContractDao.fromBuffer(contract) : undefined);
+ }
+
+ getContracts(): Promise<ContractDao[]> {
+ return Promise.resolve(Array.from(this.#contracts.values()).map(c => ContractDao.fromBuffer(c)));
+ }
+}
diff --git a/yarn-project/pxe/src/database/memory_db.test.ts b/yarn-project/pxe/src/database/memory_db.test.ts
index 077f705167a..f505efa4a79 100644
--- a/yarn-project/pxe/src/database/memory_db.test.ts
+++ b/yarn-project/pxe/src/database/memory_db.test.ts
@@ -2,6 +2,7 @@ import { AztecAddress, Fr } from '@aztec/circuits.js';
import { MemoryDB } from './memory_db.js';
import { randomNoteDao } from './note_dao.test.js';
+import { describePxeDatabase } from './pxe_database_test_suite.js';
describe('Memory DB', () => {
let db: MemoryDB;
@@ -10,6 +11,8 @@ describe('Memory DB', () => {
db = new MemoryDB();
});
+ describePxeDatabase(() => db);
+
describe('NoteDao', () => {
const contractAddress = AztecAddress.random();
const storageSlot = Fr.random();
diff --git a/yarn-project/pxe/src/database/memory_db.ts b/yarn-project/pxe/src/database/memory_db.ts
index 4f2849b42e5..641603b0210 100644
--- a/yarn-project/pxe/src/database/memory_db.ts
+++ b/yarn-project/pxe/src/database/memory_db.ts
@@ -5,8 +5,8 @@ import { createDebugLogger } from '@aztec/foundation/log';
import { MerkleTreeId, NoteFilter } from '@aztec/types';
import { MemoryContractDatabase } from '../contract_database/index.js';
-import { Database } from './database.js';
import { NoteDao } from './note_dao.js';
+import { PxeDatabase } from './pxe_database.js';
/**
* The MemoryDB class provides an in-memory implementation of a database to manage transactions and auxiliary data.
@@ -14,7 +14,7 @@ import { NoteDao } from './note_dao.js';
* The class offers methods to add, fetch, and remove transaction records and auxiliary data based on various filters such as transaction hash, address, and storage slot.
* As an in-memory database, the stored data will not persist beyond the life of the application instance.
*/
-export class MemoryDB extends MemoryContractDatabase implements Database {
+export class MemoryDB extends MemoryContractDatabase implements PxeDatabase {
private notesTable: NoteDao[] = [];
private treeRoots: Record<MerkleTreeId, Fr> | undefined;
private globalVariablesHash: Fr | undefined;
@@ -43,7 +43,7 @@ export class MemoryDB extends MemoryContractDatabase implements Database {
* @param messageHash - The message hash.
* @returns A Promise that resolves to an array of field elements representing the auth witness.
*/
- public getAuthWitness(messageHash: Fr): Promise<Fr[]> {
+ public getAuthWitness(messageHash: Fr): Promise<Fr[] | undefined> {
return Promise.resolve(this.authWitnesses[messageHash.toString()]);
}
@@ -113,9 +113,8 @@ export class MemoryDB extends MemoryContractDatabase implements Database {
return roots;
}
- public setTreeRoots(roots: Record<MerkleTreeId, Fr>) {
+ private setTreeRoots(roots: Record<MerkleTreeId, Fr>) {
this.treeRoots = roots;
- return Promise.resolve();
}
public getBlockHeader(): BlockHeader {
@@ -135,9 +134,9 @@ export class MemoryDB extends MemoryContractDatabase implements Database {
);
}
- public async setBlockHeader(blockHeader: BlockHeader): Promise<void> {
+ public setBlockHeader(blockHeader: BlockHeader): Promise<void> {
this.globalVariablesHash = blockHeader.globalVariablesHash;
- await this.setTreeRoots({
+ this.setTreeRoots({
[MerkleTreeId.NOTE_HASH_TREE]: blockHeader.noteHashTreeRoot,
[MerkleTreeId.NULLIFIER_TREE]: blockHeader.nullifierTreeRoot,
[MerkleTreeId.CONTRACT_TREE]: blockHeader.contractTreeRoot,
@@ -145,6 +144,8 @@ export class MemoryDB extends MemoryContractDatabase implements Database {
[MerkleTreeId.ARCHIVE]: blockHeader.archiveRoot,
[MerkleTreeId.PUBLIC_DATA_TREE]: blockHeader.publicDataTreeRoot,
});
+
+ return Promise.resolve();
}
public addCompleteAddress(completeAddress: CompleteAddress): Promise<boolean> {
@@ -154,8 +155,10 @@ export class MemoryDB extends MemoryContractDatabase implements Database {
return Promise.resolve(false);
}
- throw new Error(
- `Complete address with aztec address ${completeAddress.address.toString()} but different public key or partial key already exists in memory database`,
+ return Promise.reject(
+ new Error(
+ `Complete address with aztec address ${completeAddress.address.toString()} but different public key or partial key already exists in memory database`,
+ ),
);
}
this.addresses.push(completeAddress);
diff --git a/yarn-project/pxe/src/database/database.ts b/yarn-project/pxe/src/database/pxe_database.ts
similarity index 88%
rename from yarn-project/pxe/src/database/database.ts
rename to yarn-project/pxe/src/database/pxe_database.ts
index 72f50a1c576..0c7f1551770 100644
--- a/yarn-project/pxe/src/database/database.ts
+++ b/yarn-project/pxe/src/database/pxe_database.ts
@@ -9,7 +9,7 @@ import { NoteDao } from './note_dao.js';
* A database interface that provides methods for retrieving, adding, and removing transactional data related to Aztec
* addresses, storage slots, and nullifiers.
*/
-export interface Database extends ContractDatabase {
+export interface PxeDatabase extends ContractDatabase {
/**
* Add a auth witness to the database.
* @param messageHash - The message hash.
@@ -22,7 +22,7 @@ export interface Database extends ContractDatabase {
* @param messageHash - The message hash.
* @returns A Promise that resolves to an array of field elements representing the auth witness.
*/
- getAuthWitness(messageHash: Fr): Promise<Fr[]>;
+ getAuthWitness(messageHash: Fr): Promise<Fr[] | undefined>;
/**
* Adding a capsule to the capsule dispenser.
@@ -79,17 +79,6 @@ export interface Database extends ContractDatabase {
*/
getTreeRoots(): Record<MerkleTreeId, Fr>;
- /**
- * Set the tree roots for the Merkle trees in the database.
- * This function updates the 'treeRoots' property of the instance
- * with the provided 'roots' object containing MerkleTreeId and Fr pairs.
- * Note that this will overwrite any existing tree roots in the database.
- *
- * @param roots - A Record object mapping MerkleTreeIds to their corresponding Fr root values.
- * @returns A Promise that resolves when the tree roots have been successfully updated in the database.
- */
- setTreeRoots(roots: Record<MerkleTreeId, Fr>): Promise<void>;
-
/**
* Retrieve the stored Block Header from the database.
* The function returns a Promise that resolves to the Block Header.
diff --git a/yarn-project/pxe/src/database/pxe_database_test_suite.ts b/yarn-project/pxe/src/database/pxe_database_test_suite.ts
new file mode 100644
index 00000000000..69eaff032b9
--- /dev/null
+++ b/yarn-project/pxe/src/database/pxe_database_test_suite.ts
@@ -0,0 +1,220 @@
+import { AztecAddress, BlockHeader, CompleteAddress } from '@aztec/circuits.js';
+import { Fr, Point } from '@aztec/foundation/fields';
+import { MerkleTreeId, NoteFilter, randomTxHash } from '@aztec/types';
+
+import { NoteDao } from './note_dao.js';
+import { randomNoteDao } from './note_dao.test.js';
+import { PxeDatabase } from './pxe_database.js';
+
+/**
+ * A common test suite for a PXE database.
+ * @param getDatabase - A function that returns a database instance.
+ */
+export function describePxeDatabase(getDatabase: () => PxeDatabase) {
+ let database: PxeDatabase;
+
+ beforeEach(() => {
+ database = getDatabase();
+ });
+
+ describe('Database', () => {
+ describe('auth witnesses', () => {
+ it('stores and retrieves auth witnesses', async () => {
+ const messageHash = Fr.random();
+ const witness = [Fr.random(), Fr.random()];
+
+ await database.addAuthWitness(messageHash, witness);
+ await expect(database.getAuthWitness(messageHash)).resolves.toEqual(witness);
+ });
+
+ it("returns undefined if it doesn't have auth witnesses for the message", async () => {
+ const messageHash = Fr.random();
+ await expect(database.getAuthWitness(messageHash)).resolves.toBeUndefined();
+ });
+
+ it.skip('refuses to overwrite auth witnesses for the same message', async () => {
+ const messageHash = Fr.random();
+ const witness = [Fr.random(), Fr.random()];
+
+ await database.addAuthWitness(messageHash, witness);
+ await expect(database.addAuthWitness(messageHash, witness)).rejects.toThrow();
+ });
+ });
+
+ describe('capsules', () => {
+ it('stores and retrieves capsules', async () => {
+ const capsule = [Fr.random(), Fr.random()];
+
+ await database.addCapsule(capsule);
+ await expect(database.popCapsule()).resolves.toEqual(capsule);
+ });
+
+ it("returns undefined if it doesn't have capsules", async () => {
+ await expect(database.popCapsule()).resolves.toBeUndefined();
+ });
+
+ it('behaves like a stack when storing capsules', async () => {
+ const capsule1 = [Fr.random(), Fr.random()];
+ const capsule2 = [Fr.random(), Fr.random()];
+
+ await database.addCapsule(capsule1);
+ await database.addCapsule(capsule2);
+ await expect(database.popCapsule()).resolves.toEqual(capsule2);
+ await expect(database.popCapsule()).resolves.toEqual(capsule1);
+ });
+ });
+
+ describe('notes', () => {
+ let owners: CompleteAddress[];
+ let contractAddresses: AztecAddress[];
+ let storageSlots: Fr[];
+ let notes: NoteDao[];
+
+ const filteringTests: [() => NoteFilter, () => NoteDao[]][] = [
+ [() => ({}), () => notes],
+
+ [
+ () => ({ contractAddress: contractAddresses[0] }),
+ () => notes.filter(note => note.contractAddress.equals(contractAddresses[0])),
+ ],
+ [() => ({ contractAddress: AztecAddress.random() }), () => []],
+
+ [
+ () => ({ storageSlot: storageSlots[0] }),
+ () => notes.filter(note => note.storageSlot.equals(storageSlots[0])),
+ ],
+ [() => ({ storageSlot: Fr.random() }), () => []],
+
+ [() => ({ txHash: notes[0].txHash }), () => [notes[0]]],
+ [() => ({ txHash: randomTxHash() }), () => []],
+
+ [() => ({ owner: owners[0].address }), () => notes.filter(note => note.publicKey.equals(owners[0].publicKey))],
+
+ [
+ () => ({ contractAddress: contractAddresses[0], storageSlot: storageSlots[0] }),
+ () =>
+ notes.filter(
+ note => note.contractAddress.equals(contractAddresses[0]) && note.storageSlot.equals(storageSlots[0]),
+ ),
+ ],
+ [() => ({ contractAddress: contractAddresses[0], storageSlot: storageSlots[1] }), () => []],
+ ];
+
+ beforeEach(() => {
+ owners = Array.from({ length: 2 }).map(() => CompleteAddress.random());
+ contractAddresses = Array.from({ length: 2 }).map(() => AztecAddress.random());
+ storageSlots = Array.from({ length: 2 }).map(() => Fr.random());
+
+ notes = Array.from({ length: 10 }).map((_, i) =>
+ randomNoteDao({
+ contractAddress: contractAddresses[i % contractAddresses.length],
+ storageSlot: storageSlots[i % storageSlots.length],
+ publicKey: owners[i % owners.length].publicKey,
+ }),
+ );
+ });
+
+ beforeEach(async () => {
+ for (const owner of owners) {
+ await database.addCompleteAddress(owner);
+ }
+ });
+
+ it.each(filteringTests)('stores notes in bulk and retrieves notes', async (getFilter, getExpected) => {
+ await database.addNotes(notes);
+ await expect(database.getNotes(getFilter())).resolves.toEqual(getExpected());
+ });
+
+ it.each(filteringTests)('stores notes one by one and retrieves notes', async (getFilter, getExpected) => {
+ for (const note of notes) {
+ await database.addNote(note);
+ }
+ await expect(database.getNotes(getFilter())).resolves.toEqual(getExpected());
+ });
+
+ it('removes nullified notes', async () => {
+ const notesToNullify = notes.filter(note => note.publicKey.equals(owners[0].publicKey));
+ const nullifiers = notesToNullify.map(note => note.siloedNullifier);
+
+ await database.addNotes(notes);
+
+ await expect(database.removeNullifiedNotes(nullifiers, notesToNullify[0].publicKey)).resolves.toEqual(
+ notesToNullify,
+ );
+ await expect(
+ database.getNotes({
+ owner: owners[0].address,
+ }),
+ ).resolves.toEqual([]);
+ await expect(database.getNotes({})).resolves.toEqual(notes.filter(note => !notesToNullify.includes(note)));
+ });
+ });
+
+ describe('block header', () => {
+ it('stores and retrieves the block header', async () => {
+ const blockHeader = BlockHeader.random();
+ blockHeader.privateKernelVkTreeRoot = Fr.zero();
+
+ await database.setBlockHeader(blockHeader);
+ expect(database.getBlockHeader()).toEqual(blockHeader);
+ });
+
+ it('retrieves the merkle tree roots from the block', async () => {
+ const blockHeader = BlockHeader.random();
+ await database.setBlockHeader(blockHeader);
+ expect(database.getTreeRoots()).toEqual({
+ [MerkleTreeId.NOTE_HASH_TREE]: blockHeader.noteHashTreeRoot,
+ [MerkleTreeId.NULLIFIER_TREE]: blockHeader.nullifierTreeRoot,
+ [MerkleTreeId.CONTRACT_TREE]: blockHeader.contractTreeRoot,
+ [MerkleTreeId.L1_TO_L2_MESSAGES_TREE]: blockHeader.l1ToL2MessagesTreeRoot,
+ [MerkleTreeId.ARCHIVE]: blockHeader.archiveRoot,
+ [MerkleTreeId.PUBLIC_DATA_TREE]: blockHeader.publicDataTreeRoot,
+ });
+ });
+
+ it('rejects getting merkle tree roots if no block set', () => {
+ expect(() => database.getTreeRoots()).toThrow();
+ });
+ });
+
+ describe('addresses', () => {
+ it('stores and retrieves addresses', async () => {
+ const address = CompleteAddress.random();
+ await expect(database.addCompleteAddress(address)).resolves.toBe(true);
+ await expect(database.getCompleteAddress(address.address)).resolves.toEqual(address);
+ });
+
+ it('silently ignores an address it already knows about', async () => {
+ const address = CompleteAddress.random();
+ await expect(database.addCompleteAddress(address)).resolves.toBe(true);
+ await expect(database.addCompleteAddress(address)).resolves.toBe(false);
+ });
+
+ it.skip('refuses to overwrite an address with a different public key', async () => {
+ const address = CompleteAddress.random();
+ const otherAddress = new CompleteAddress(address.address, Point.random(), address.partialAddress);
+
+ await database.addCompleteAddress(address);
+ await expect(database.addCompleteAddress(otherAddress)).rejects.toThrow();
+ });
+
+ it('returns all addresses', async () => {
+ const addresses = Array.from({ length: 10 }).map(() => CompleteAddress.random());
+ for (const address of addresses) {
+ await database.addCompleteAddress(address);
+ }
+
+ const result = await database.getCompleteAddresses();
+ expect(result).toEqual(expect.arrayContaining(addresses));
+ });
+
+ it("returns an empty array if it doesn't have addresses", async () => {
+ expect(await database.getCompleteAddresses()).toEqual([]);
+ });
+
+ it("returns undefined if it doesn't have an address", async () => {
+ expect(await database.getCompleteAddress(CompleteAddress.random().address)).toBeUndefined();
+ });
+ });
+ });
+}
diff --git a/yarn-project/pxe/src/note_processor/note_processor.test.ts b/yarn-project/pxe/src/note_processor/note_processor.test.ts
index d766a04c336..a317c24cddd 100644
--- a/yarn-project/pxe/src/note_processor/note_processor.test.ts
+++ b/yarn-project/pxe/src/note_processor/note_processor.test.ts
@@ -1,9 +1,10 @@
import { AcirSimulator } from '@aztec/acir-simulator';
-import { Fr, MAX_NEW_COMMITMENTS_PER_TX } from '@aztec/circuits.js';
+import { EthAddress, Fr, MAX_NEW_COMMITMENTS_PER_TX } from '@aztec/circuits.js';
import { Grumpkin } from '@aztec/circuits.js/barretenberg';
import { pedersenHash } from '@aztec/foundation/crypto';
import { Point } from '@aztec/foundation/fields';
import { ConstantKeyPair } from '@aztec/key-store';
+import { AztecLmdbStore } from '@aztec/kv-store';
import {
AztecNode,
FunctionL2Logs,
@@ -21,7 +22,8 @@ import {
import { jest } from '@jest/globals';
import { MockProxy, mock } from 'jest-mock-extended';
-import { Database, MemoryDB } from '../database/index.js';
+import { PxeDatabase } from '../database/index.js';
+import { KVPxeDatabase } from '../database/kv_pxe_database.js';
import { NoteDao } from '../database/note_dao.js';
import { NoteProcessor } from './note_processor.js';
@@ -29,7 +31,7 @@ const TXS_PER_BLOCK = 4;
describe('Note Processor', () => {
let grumpkin: Grumpkin;
- let database: Database;
+ let database: PxeDatabase;
let aztecNode: ReturnType<typeof mock<AztecNode>>;
let addNotesSpy: any;
let noteProcessor: NoteProcessor;
@@ -114,8 +116,8 @@ describe('Note Processor', () => {
owner = ConstantKeyPair.random(grumpkin);
});
- beforeEach(() => {
- database = new MemoryDB();
+ beforeEach(async () => {
+ database = new KVPxeDatabase(await AztecLmdbStore.create(EthAddress.random()));
addNotesSpy = jest.spyOn(database, 'addNotes');
aztecNode = mock<AztecNode>();
diff --git a/yarn-project/pxe/src/note_processor/note_processor.ts b/yarn-project/pxe/src/note_processor/note_processor.ts
index 0f07830436f..436085df589 100644
--- a/yarn-project/pxe/src/note_processor/note_processor.ts
+++ b/yarn-project/pxe/src/note_processor/note_processor.ts
@@ -7,7 +7,7 @@ import { Timer } from '@aztec/foundation/timer';
import { AztecNode, KeyStore, L1NotePayload, L2BlockContext, L2BlockL2Logs } from '@aztec/types';
import { NoteProcessorStats } from '@aztec/types/stats';
-import { Database } from '../database/index.js';
+import { PxeDatabase } from '../database/index.js';
import { NoteDao } from '../database/note_dao.js';
import { getAcirSimulator } from '../simulator/index.js';
@@ -45,7 +45,7 @@ export class NoteProcessor {
*/
public readonly publicKey: PublicKey,
private keyStore: KeyStore,
- private db: Database,
+ private db: PxeDatabase,
private node: AztecNode,
private startingBlock: number,
private simulator = getAcirSimulator(db, node, keyStore),
diff --git a/yarn-project/pxe/src/pxe_service/create_pxe_service.ts b/yarn-project/pxe/src/pxe_service/create_pxe_service.ts
index 04eddd7db48..f2942d8a800 100644
--- a/yarn-project/pxe/src/pxe_service/create_pxe_service.ts
+++ b/yarn-project/pxe/src/pxe_service/create_pxe_service.ts
@@ -1,25 +1,14 @@
import { Grumpkin } from '@aztec/circuits.js/barretenberg';
import { TestKeyStore } from '@aztec/key-store';
-import { AztecNode, KeyStore } from '@aztec/types';
+import { AztecLmdbStore } from '@aztec/kv-store';
+import { AztecNode } from '@aztec/types';
+
+import { join } from 'path';
import { PXEServiceConfig } from '../config/index.js';
-import { Database, MemoryDB } from '../database/index.js';
+import { KVPxeDatabase } from '../database/kv_pxe_database.js';
import { PXEService } from './pxe_service.js';
-/**
- * Optional information for creating an PXEService.
- */
-interface CreatePXEServiceOptions {
- /**
- * A secure storage for cryptographic keys.
- */
- keyStore?: KeyStore;
- /**
- * Storage for the PXE.
- */
- db?: Database;
-}
-
/**
* Create and start an PXEService instance with the given AztecNode.
* If no keyStore or database is provided, it will use TestKeyStore and MemoryDB as default values.
@@ -33,7 +22,6 @@ interface CreatePXEServiceOptions {
export async function createPXEService(
aztecNode: AztecNode,
config: PXEServiceConfig,
- { keyStore, db }: CreatePXEServiceOptions = {},
useLogSuffix: string | boolean | undefined = undefined,
) {
const logSuffix =
@@ -43,10 +31,18 @@ export async function createPXEService(
: undefined
: useLogSuffix;
- keyStore = keyStore || new TestKeyStore(new Grumpkin());
- db = db || new MemoryDB(logSuffix);
+ const pxeDbPath = config.dataDirectory ? join(config.dataDirectory, 'pxe_data') : undefined;
+ const keyStorePath = config.dataDirectory ? join(config.dataDirectory, 'pxe_key_store') : undefined;
+ const l1Contracts = await aztecNode.getL1ContractAddresses();
+
+ const keyStore = new TestKeyStore(
+ new Grumpkin(),
+ await AztecLmdbStore.create(l1Contracts.rollupAddress, keyStorePath),
+ );
+ const db = new KVPxeDatabase(await AztecLmdbStore.create(l1Contracts.rollupAddress, pxeDbPath));
const server = new PXEService(keyStore, aztecNode, db, config, logSuffix);
+
await server.start();
return server;
}
diff --git a/yarn-project/pxe/src/pxe_service/pxe_service.ts b/yarn-project/pxe/src/pxe_service/pxe_service.ts
index 37f4eb7dcb3..f4cf6eb29d9 100644
--- a/yarn-project/pxe/src/pxe_service/pxe_service.ts
+++ b/yarn-project/pxe/src/pxe_service/pxe_service.ts
@@ -50,12 +50,11 @@ import {
TxStatus,
getNewContractPublicFunctions,
isNoirCallStackUnresolved,
- toContractDao,
} from '@aztec/types';
import { PXEServiceConfig, getPackageInfo } from '../config/index.js';
import { ContractDataOracle } from '../contract_data_oracle/index.js';
-import { Database } from '../database/index.js';
+import { PxeDatabase } from '../database/index.js';
import { NoteDao } from '../database/note_dao.js';
import { KernelOracle } from '../kernel_oracle/index.js';
import { KernelProver } from '../kernel_prover/kernel_prover.js';
@@ -75,7 +74,7 @@ export class PXEService implements PXE {
constructor(
private keyStore: KeyStore,
private node: AztecNode,
- private db: Database,
+ private db: PxeDatabase,
private config: PXEServiceConfig,
logSuffix?: string,
) {
@@ -128,7 +127,7 @@ export class PXEService implements PXE {
const completeAddress = CompleteAddress.fromPrivateKeyAndPartialAddress(privKey, partialAddress);
const wasAdded = await this.db.addCompleteAddress(completeAddress);
if (wasAdded) {
- const pubKey = this.keyStore.addAccount(privKey);
+ const pubKey = await this.keyStore.addAccount(privKey);
this.synchronizer.addAccount(pubKey, this.keyStore, this.config.l2StartingBlock);
this.log.info(`Registered account ${completeAddress.address.toString()}`);
this.log.debug(`Registered account\n ${completeAddress.toReadableString()}`);
@@ -178,7 +177,7 @@ export class PXEService implements PXE {
}
public async addContracts(contracts: DeployedContract[]) {
- const contractDaos = contracts.map(c => toContractDao(c.artifact, c.completeAddress, c.portalContract));
+ const contractDaos = contracts.map(c => new ContractDao(c.artifact, c.completeAddress, c.portalContract));
await Promise.all(contractDaos.map(c => this.db.addContract(c)));
for (const contract of contractDaos) {
const portalInfo =
diff --git a/yarn-project/pxe/src/pxe_service/test/pxe_service.test.ts b/yarn-project/pxe/src/pxe_service/test/pxe_service.test.ts
index e265d99a898..e55560280a0 100644
--- a/yarn-project/pxe/src/pxe_service/test/pxe_service.test.ts
+++ b/yarn-project/pxe/src/pxe_service/test/pxe_service.test.ts
@@ -2,19 +2,22 @@ import { Grumpkin } from '@aztec/circuits.js/barretenberg';
import { L1ContractAddresses } from '@aztec/ethereum';
import { EthAddress } from '@aztec/foundation/eth-address';
import { TestKeyStore } from '@aztec/key-store';
+import { AztecLmdbStore } from '@aztec/kv-store';
import { AztecNode, INITIAL_L2_BLOCK_NUM, L2Tx, PXE, mockTx } from '@aztec/types';
import { MockProxy, mock } from 'jest-mock-extended';
-import { MemoryDB } from '../../database/memory_db.js';
+import { KVPxeDatabase } from '../../database/kv_pxe_database.js';
+import { PxeDatabase } from '../../database/pxe_database.js';
import { PXEServiceConfig } from '../../index.js';
import { PXEService } from '../pxe_service.js';
import { pxeTestSuite } from './pxe_test_suite.js';
-function createPXEService(): Promise<PXE> {
- const keyStore = new TestKeyStore(new Grumpkin());
+async function createPXEService(): Promise<PXE> {
+ const kvStore = await AztecLmdbStore.create(EthAddress.random());
+ const keyStore = new TestKeyStore(new Grumpkin(), kvStore);
const node = mock<AztecNode>();
- const db = new MemoryDB();
+ const db = new KVPxeDatabase(kvStore);
const config: PXEServiceConfig = { l2BlockPollingIntervalMS: 100, l2StartingBlock: INITIAL_L2_BLOCK_NUM };
// Setup the relevant mocks
@@ -39,13 +42,14 @@ pxeTestSuite('PXEService', createPXEService);
describe('PXEService', () => {
let keyStore: TestKeyStore;
let node: MockProxy<AztecNode>;
- let db: MemoryDB;
+ let db: PxeDatabase;
let config: PXEServiceConfig;
- beforeEach(() => {
- keyStore = new TestKeyStore(new Grumpkin());
+ beforeEach(async () => {
+ const kvStore = await AztecLmdbStore.create(EthAddress.random());
+ keyStore = new TestKeyStore(new Grumpkin(), kvStore);
node = mock<AztecNode>();
- db = new MemoryDB();
+ db = new KVPxeDatabase(kvStore);
config = { l2BlockPollingIntervalMS: 100, l2StartingBlock: INITIAL_L2_BLOCK_NUM };
});
diff --git a/yarn-project/pxe/src/simulator/index.ts b/yarn-project/pxe/src/simulator/index.ts
index 0ffc34a35ef..64abb774d20 100644
--- a/yarn-project/pxe/src/simulator/index.ts
+++ b/yarn-project/pxe/src/simulator/index.ts
@@ -2,14 +2,14 @@ import { AcirSimulator } from '@aztec/acir-simulator';
import { KeyStore, StateInfoProvider } from '@aztec/types';
import { ContractDataOracle } from '../contract_data_oracle/index.js';
-import { Database } from '../database/database.js';
+import { PxeDatabase } from '../database/pxe_database.js';
import { SimulatorOracle } from '../simulator_oracle/index.js';
/**
* Helper method to create an instance of the acir simulator.
*/
export function getAcirSimulator(
- db: Database,
+ db: PxeDatabase,
stateInfoProvider: StateInfoProvider,
keyStore: KeyStore,
contractDataOracle?: ContractDataOracle,
diff --git a/yarn-project/pxe/src/simulator_oracle/index.ts b/yarn-project/pxe/src/simulator_oracle/index.ts
index 2eb485fd287..4b3e21aee64 100644
--- a/yarn-project/pxe/src/simulator_oracle/index.ts
+++ b/yarn-project/pxe/src/simulator_oracle/index.ts
@@ -13,7 +13,7 @@ import { createDebugLogger } from '@aztec/foundation/log';
import { KeyStore, L2Block, MerkleTreeId, NullifierMembershipWitness, StateInfoProvider } from '@aztec/types';
import { ContractDataOracle } from '../contract_data_oracle/index.js';
-import { Database } from '../database/index.js';
+import { PxeDatabase } from '../database/index.js';
/**
* A data oracle that provides information needed for simulating a transaction.
@@ -21,7 +21,7 @@ import { Database } from '../database/index.js';
export class SimulatorOracle implements DBOracle {
constructor(
private contractDataOracle: ContractDataOracle,
- private db: Database,
+ private db: PxeDatabase,
private keyStore: KeyStore,
private stateInfoProvider: StateInfoProvider,
private log = createDebugLogger('aztec:pxe:simulator_oracle'),
diff --git a/yarn-project/pxe/src/synchronizer/synchronizer.test.ts b/yarn-project/pxe/src/synchronizer/synchronizer.test.ts
index 95893f10810..5270b0d7d1e 100644
--- a/yarn-project/pxe/src/synchronizer/synchronizer.test.ts
+++ b/yarn-project/pxe/src/synchronizer/synchronizer.test.ts
@@ -1,22 +1,24 @@
-import { BlockHeader, CompleteAddress, Fr, GrumpkinScalar } from '@aztec/circuits.js';
+import { BlockHeader, CompleteAddress, EthAddress, Fr, GrumpkinScalar } from '@aztec/circuits.js';
import { Grumpkin } from '@aztec/circuits.js/barretenberg';
import { TestKeyStore } from '@aztec/key-store';
+import { AztecLmdbStore } from '@aztec/kv-store';
import { AztecNode, INITIAL_L2_BLOCK_NUM, L2Block, MerkleTreeId } from '@aztec/types';
import { MockProxy, mock } from 'jest-mock-extended';
import omit from 'lodash.omit';
-import { Database, MemoryDB } from '../database/index.js';
+import { PxeDatabase } from '../database/index.js';
+import { KVPxeDatabase } from '../database/kv_pxe_database.js';
import { Synchronizer } from './synchronizer.js';
describe('Synchronizer', () => {
let aztecNode: MockProxy<AztecNode>;
- let database: Database;
+ let database: PxeDatabase;
let synchronizer: TestSynchronizer;
let roots: Record<MerkleTreeId, Fr>;
let blockHeader: BlockHeader;
- beforeEach(() => {
+ beforeEach(async () => {
blockHeader = BlockHeader.random();
roots = {
[MerkleTreeId.CONTRACT_TREE]: blockHeader.contractTreeRoot,
@@ -28,7 +30,7 @@ describe('Synchronizer', () => {
};
aztecNode = mock<AztecNode>();
- database = new MemoryDB();
+ database = new KVPxeDatabase(await AztecLmdbStore.create(EthAddress.random()));
synchronizer = new TestSynchronizer(aztecNode, database);
});
@@ -102,9 +104,9 @@ describe('Synchronizer', () => {
aztecNode.getBlockNumber.mockResolvedValueOnce(1);
// Manually adding account to database so that we can call synchronizer.isAccountStateSynchronized
- const keyStore = new TestKeyStore(new Grumpkin());
+ const keyStore = new TestKeyStore(new Grumpkin(), await AztecLmdbStore.create(EthAddress.random()));
const privateKey = GrumpkinScalar.random();
- keyStore.addAccount(privateKey);
+ await keyStore.addAccount(privateKey);
const completeAddress = CompleteAddress.fromPrivateKeyAndPartialAddress(privateKey, Fr.random());
await database.addCompleteAddress(completeAddress);
diff --git a/yarn-project/pxe/src/synchronizer/synchronizer.ts b/yarn-project/pxe/src/synchronizer/synchronizer.ts
index 1c557386c18..2346ae7d613 100644
--- a/yarn-project/pxe/src/synchronizer/synchronizer.ts
+++ b/yarn-project/pxe/src/synchronizer/synchronizer.ts
@@ -5,7 +5,7 @@ import { InterruptibleSleep } from '@aztec/foundation/sleep';
import { AztecNode, INITIAL_L2_BLOCK_NUM, KeyStore, L2BlockContext, L2BlockL2Logs, LogType } from '@aztec/types';
import { NoteProcessorCaughtUpStats } from '@aztec/types/stats';
-import { Database } from '../database/index.js';
+import { PxeDatabase } from '../database/index.js';
import { NoteProcessor } from '../note_processor/index.js';
/**
@@ -25,7 +25,7 @@ export class Synchronizer {
private log: DebugLogger;
private noteProcessorsToCatchUp: NoteProcessor[] = [];
- constructor(private node: AztecNode, private db: Database, logSuffix = '') {
+ constructor(private node: AztecNode, private db: PxeDatabase, logSuffix = '') {
this.log = createDebugLogger(logSuffix ? `aztec:pxe_synchronizer_${logSuffix}` : 'aztec:pxe_synchronizer');
}
diff --git a/yarn-project/pxe/tsconfig.json b/yarn-project/pxe/tsconfig.json
index 3106fcd762f..c282504efe6 100644
--- a/yarn-project/pxe/tsconfig.json
+++ b/yarn-project/pxe/tsconfig.json
@@ -9,6 +9,9 @@
{
"path": "../acir-simulator"
},
+ {
+ "path": "../kv-store"
+ },
{
"path": "../circuits.js"
},
diff --git a/yarn-project/types/src/contract_dao.test.ts b/yarn-project/types/src/contract_dao.test.ts
new file mode 100644
index 00000000000..5cfa8ea7e92
--- /dev/null
+++ b/yarn-project/types/src/contract_dao.test.ts
@@ -0,0 +1,54 @@
+import { CompleteAddress, EthAddress } from '@aztec/circuits.js';
+import { ABIParameterVisibility, ContractArtifact, FunctionSelector, FunctionType } from '@aztec/foundation/abi';
+
+import { ContractDao } from './contract_dao.js';
+import { randomContractArtifact } from './mocks.js';
+
+describe('ContractDao', () => {
+ it('serializes / deserializes correctly', () => {
+ const artifact = randomContractArtifact();
+ const dao = new ContractDao(artifact, CompleteAddress.random(), EthAddress.random());
+
+ expect(ContractDao.fromBuffer(dao.toBuffer())).toEqual(dao);
+ });
+
+ it('extracts function data', () => {
+ const artifact: ContractArtifact = {
+ name: 'test',
+ functions: [
+ {
+ name: 'bar',
+ functionType: FunctionType.SECRET,
+ isInternal: false,
+ parameters: [
+ {
+ name: 'value',
+ type: {
+ kind: 'field',
+ },
+ visibility: ABIParameterVisibility.PUBLIC,
+ },
+ {
+ name: 'value',
+ type: {
+ kind: 'field',
+ },
+ visibility: ABIParameterVisibility.SECRET,
+ },
+ ],
+ returnTypes: [],
+ bytecode: '0af',
+ },
+ ],
+ events: [],
+ };
+
+ const dao = new ContractDao(artifact, CompleteAddress.random(), EthAddress.random());
+
+ expect(dao.functions[0]).toEqual({
+ ...artifact.functions[0],
+ // number representing bar((Field),Field)
+ selector: new FunctionSelector(4138634513),
+ });
+ });
+});
diff --git a/yarn-project/types/src/contract_dao.ts b/yarn-project/types/src/contract_dao.ts
index d07e2d66a58..10cb955d743 100644
--- a/yarn-project/types/src/contract_dao.ts
+++ b/yarn-project/types/src/contract_dao.ts
@@ -1,53 +1,67 @@
import { CompleteAddress, ContractFunctionDao } from '@aztec/circuits.js';
-import { ContractArtifact, FunctionSelector, FunctionType } from '@aztec/foundation/abi';
+import { ContractArtifact, DebugMetadata, EventAbi, FunctionSelector, FunctionType } from '@aztec/foundation/abi';
import { EthAddress } from '@aztec/foundation/eth-address';
+import { prefixBufferWithLength } from '@aztec/foundation/serialize';
-import { EncodedContractFunction } from './contract_data.js';
+import { BufferReader, EncodedContractFunction } from './contract_data.js';
/**
* A contract Data Access Object (DAO).
* Contains the contract's address, portal contract address, and an array of ContractFunctionDao objects.
* Each ContractFunctionDao object includes FunctionAbi data and the function selector buffer.
*/
-export interface ContractDao extends ContractArtifact {
- /**
- * The complete address representing the contract on L2.
- */
- completeAddress: CompleteAddress;
- /**
- * The Ethereum address of the L1 contract serving as a bridge for cross-layer interactions.
- */
- portalContract: EthAddress;
- /**
- * An array of contract functions with additional selector property.
- */
- functions: ContractFunctionDao[];
-}
+export class ContractDao implements ContractArtifact {
+ /** An array of contract functions with additional selector property. */
+ public readonly functions: ContractFunctionDao[];
+ constructor(
+ private contractArtifact: ContractArtifact,
+ /** The complete address representing the contract on L2. */
+ public readonly completeAddress: CompleteAddress,
+ /** The Ethereum address of the L1 contract serving as a bridge for cross-layer interactions. */
+ public readonly portalContract: EthAddress,
+ ) {
+ this.functions = contractArtifact.functions.map(f => ({
+ ...f,
+ selector: FunctionSelector.fromNameAndParameters(f.name, f.parameters),
+ }));
+ }
-/**
- * Converts the given contract artifact into a ContractDao object that includes additional properties
- * such as the address, portal contract, and function selectors.
- *
- * @param artifact - The contract artifact.
- * @param completeAddress - The AztecAddress representing the contract's address.
- * @param portalContract - The EthAddress representing the address of the associated portal contract.
- * @returns A ContractDao object containing the provided information along with generated function selectors.
- */
-export function toContractDao(
- artifact: ContractArtifact,
- completeAddress: CompleteAddress,
- portalContract: EthAddress,
-): ContractDao {
- const functions = artifact.functions.map(f => ({
- ...f,
- selector: FunctionSelector.fromNameAndParameters(f.name, f.parameters),
- }));
- return {
- ...artifact,
- completeAddress,
- functions,
- portalContract,
- };
+ get aztecNrVersion() {
+ return this.contractArtifact.aztecNrVersion;
+ }
+
+ get name(): string {
+ return this.contractArtifact.name;
+ }
+
+ get events(): EventAbi[] {
+ return this.contractArtifact.events;
+ }
+
+ get debug(): DebugMetadata | undefined {
+ return this.contractArtifact.debug;
+ }
+
+ toBuffer(): Buffer {
+ // the contract artifact was originally emitted to a JSON file by Noir
+ // should be safe to JSON.stringify it (i.e. it doesn't contain BigInts)
+ const contractArtifactJson = JSON.stringify(this.contractArtifact);
+ const buf = Buffer.concat([
+ this.completeAddress.toBuffer(),
+ this.portalContract.toBuffer20(),
+ prefixBufferWithLength(Buffer.from(contractArtifactJson, 'utf-8')),
+ ]);
+
+ return buf;
+ }
+
+ static fromBuffer(buf: Uint8Array | BufferReader) {
+ const reader = BufferReader.asReader(buf);
+ const completeAddress = CompleteAddress.fromBuffer(reader);
+ const portalContract = new EthAddress(reader.readBytes(EthAddress.SIZE_IN_BYTES));
+ const contractArtifact = JSON.parse(reader.readString());
+ return new ContractDao(contractArtifact, completeAddress, portalContract);
+ }
}
/**
diff --git a/yarn-project/types/src/keys/key_store.ts b/yarn-project/types/src/keys/key_store.ts
index bce41ab0163..682aa89862e 100644
--- a/yarn-project/types/src/keys/key_store.ts
+++ b/yarn-project/types/src/keys/key_store.ts
@@ -18,7 +18,7 @@ export interface KeyStore {
* @param privKey - The private key of the account.
* @returns - The account's public key.
*/
- addAccount(privKey: GrumpkinPrivateKey): PublicKey;
+ addAccount(privKey: GrumpkinPrivateKey): Promise<PublicKey>;
/**
* Retrieves the public keys of all accounts stored.
diff --git a/yarn-project/yarn.lock b/yarn-project/yarn.lock
index 935cf87829a..56d05bdb6cd 100644
--- a/yarn-project/yarn.lock
+++ b/yarn-project/yarn.lock
@@ -522,6 +522,7 @@ __metadata:
dependencies:
"@aztec/circuits.js": "workspace:^"
"@aztec/foundation": "workspace:^"
+ "@aztec/kv-store": "workspace:^"
"@aztec/types": "workspace:^"
"@jest/globals": ^29.5.0
"@types/jest": ^29.5.0
@@ -534,6 +535,23 @@ __metadata:
languageName: unknown
linkType: soft
+"@aztec/kv-store@workspace:^, @aztec/kv-store@workspace:kv-store":
+ version: 0.0.0-use.local
+ resolution: "@aztec/kv-store@workspace:kv-store"
+ dependencies:
+ "@aztec/foundation": "workspace:^"
+ "@jest/globals": ^29.5.0
+ "@types/jest": ^29.5.0
+ "@types/node": ^18.7.23
+ jest: ^29.5.0
+ jest-mock-extended: ^3.0.3
+ lmdb: ^2.9.1
+ ts-jest: ^29.1.0
+ ts-node: ^10.9.1
+ typescript: ^5.0.4
+ languageName: unknown
+ linkType: soft
+
"@aztec/l1-artifacts@workspace:^, @aztec/l1-artifacts@workspace:l1-artifacts":
version: 0.0.0-use.local
resolution: "@aztec/l1-artifacts@workspace:l1-artifacts"
@@ -741,6 +759,7 @@ __metadata:
"@aztec/ethereum": "workspace:^"
"@aztec/foundation": "workspace:^"
"@aztec/key-store": "workspace:^"
+ "@aztec/kv-store": "workspace:^"
"@aztec/noir-compiler": "workspace:^"
"@aztec/noir-protocol-circuits": "workspace:^"
"@aztec/types": "workspace:^"