feat: in memory kv store.

yarn-project/aztec-node/src/aztec-node/server.ts

@@ -47,6 +47,7 @@ import { padArrayEnd } from '@aztec/foundation/collection';
 import { createDebugLogger } from '@aztec/foundation/log';
 import { type AztecKVStore } from '@aztec/kv-store';
 import { AztecLmdbStore } from '@aztec/kv-store/lmdb';
+import { AztecMemStore } from '@aztec/kv-store/mem';
 import { initStoreForRollup, openTmpStore } from '@aztec/kv-store/utils';
 import { SHA256Trunc, StandardTree } from '@aztec/merkle-tree';
 import { AztecKVTxPool, type P2P, createP2PClient } from '@aztec/p2p';
@@ -121,11 +122,11 @@ export class AztecNodeService implements AztecNode {
 
     const log = createDebugLogger('aztec:node');
     const storeLog = createDebugLogger('aztec:node:lmdb');
-    const store = await initStoreForRollup(
-      AztecLmdbStore.open(config.dataDirectory, false, storeLog),
-      config.l1Contracts.rollupAddress,
-      storeLog,
-    );
+    const storeDb = config.dataDirectory
+      ? AztecLmdbStore.open(config.dataDirectory, false, storeLog)
+      : new AztecMemStore();
+
+    const store = await initStoreForRollup(storeDb, config.l1Contracts.rollupAddress, storeLog);
 
     let archiver: ArchiveSource;
     if (!config.archiverUrl) {

yarn-project/aztec/src/cli/cmds/start_archiver.ts

@@ -5,9 +5,10 @@ import {
   createArchiverRpcServer,
   getConfigEnvVars as getArchiverConfigEnvVars,
 } from '@aztec/archiver';
-import { createDebugLogger } from '@aztec/aztec.js';
 import { type ServerList } from '@aztec/foundation/json-rpc/server';
+import { createDebugLogger } from '@aztec/foundation/log';
 import { AztecLmdbStore } from '@aztec/kv-store/lmdb';
+import { AztecMemStore } from '@aztec/kv-store/mem';
 import { initStoreForRollup } from '@aztec/kv-store/utils';
 
 import { mergeEnvVarsAndCliOptions, parseModuleOptions } from '../util.js';
@@ -23,11 +24,10 @@ export const startArchiver = async (options: any, signalHandlers: (() => Promise
   const archiverConfig = mergeEnvVarsAndCliOptions<ArchiverConfig>(archiverConfigEnvVars, archiverCliOptions, true);
 
   const storeLog = createDebugLogger('aztec:archiver:lmdb');
-  const store = await initStoreForRollup(
-    AztecLmdbStore.open(archiverConfig.dataDirectory, false, storeLog),
-    archiverConfig.l1Contracts.rollupAddress,
-    storeLog,
-  );
+  const storeDb = archiverConfig.dataDirectory
+    ? AztecLmdbStore.open(archiverConfig.dataDirectory, false, storeLog)
+    : new AztecMemStore();
+  const store = await initStoreForRollup(storeDb, archiverConfig.l1Contracts.rollupAddress, storeLog);
   const archiverStore = new KVArchiverDataStore(store, archiverConfig.maxLogs);
 
   const archiver = await Archiver.createAndSync(archiverConfig, archiverStore, true);

yarn-project/kv-store/package.json

@@ -5,6 +5,7 @@
   "exports": {
     ".": "./dest/interfaces/index.js",
     "./lmdb": "./dest/lmdb/index.js",
+    "./mem": "./dest/mem/index.js",
     "./utils": "./dest/utils.js"
   },
   "scripts": {

yarn-project/kv-store/src/lmdb/array.test.ts

@@ -1,91 +1,14 @@
 import { type Database, open } from 'lmdb';
 
+import { addArrayTests } from '../tests/aztec_array_tests.js';
 import { LmdbAztecArray } from './array.js';
 
 describe('LmdbAztecArray', () => {
   let db: Database;
-  let arr: LmdbAztecArray<number>;
 
   beforeEach(() => {
     db = open({} as any);
-    arr = new LmdbAztecArray(db, 'test');
   });
 
-  it('should be able to push and pop values', async () => {
-    await arr.push(1);
-    await arr.push(2);
-    await arr.push(3);
-
-    expect(arr.length).toEqual(3);
-    expect(await arr.pop()).toEqual(3);
-    expect(await arr.pop()).toEqual(2);
-    expect(await arr.pop()).toEqual(1);
-    expect(await arr.pop()).toEqual(undefined);
-  });
-
-  it('should be able to get values by index', async () => {
-    await arr.push(1);
-    await arr.push(2);
-    await arr.push(3);
-
-    expect(arr.at(0)).toEqual(1);
-    expect(arr.at(1)).toEqual(2);
-    expect(arr.at(2)).toEqual(3);
-    expect(arr.at(3)).toEqual(undefined);
-    expect(arr.at(-1)).toEqual(3);
-    expect(arr.at(-2)).toEqual(2);
-    expect(arr.at(-3)).toEqual(1);
-    expect(arr.at(-4)).toEqual(undefined);
-  });
-
-  it('should be able to set values by index', async () => {
-    await arr.push(1);
-    await arr.push(2);
-    await arr.push(3);
-
-    expect(await arr.setAt(0, 4)).toEqual(true);
-    expect(await arr.setAt(1, 5)).toEqual(true);
-    expect(await arr.setAt(2, 6)).toEqual(true);
-
-    expect(await arr.setAt(3, 7)).toEqual(false);
-
-    expect(arr.at(0)).toEqual(4);
-    expect(arr.at(1)).toEqual(5);
-    expect(arr.at(2)).toEqual(6);
-    expect(arr.at(3)).toEqual(undefined);
-
-    expect(await arr.setAt(-1, 8)).toEqual(true);
-    expect(await arr.setAt(-2, 9)).toEqual(true);
-    expect(await arr.setAt(-3, 10)).toEqual(true);
-
-    expect(await arr.setAt(-4, 11)).toEqual(false);
-
-    expect(arr.at(-1)).toEqual(8);
-    expect(arr.at(-2)).toEqual(9);
-    expect(arr.at(-3)).toEqual(10);
-    expect(arr.at(-4)).toEqual(undefined);
-  });
-
-  it('should be able to iterate over values', async () => {
-    await arr.push(1);
-    await arr.push(2);
-    await arr.push(3);
-
-    expect([...arr.values()]).toEqual([1, 2, 3]);
-    expect([...arr.entries()]).toEqual([
-      [0, 1],
-      [1, 2],
-      [2, 3],
-    ]);
-  });
-
-  it('should be able to restore state', async () => {
-    await arr.push(1);
-    await arr.push(2);
-    await arr.push(3);
-
-    const arr2 = new LmdbAztecArray(db, 'test');
-    expect(arr2.length).toEqual(3);
-    expect([...arr2.values()]).toEqual([...arr.values()]);
-  });
+  addArrayTests(() => new LmdbAztecArray(db, 'test'));
 });

yarn-project/kv-store/src/lmdb/map.test.ts

@@ -1,99 +1,14 @@
 import { type Database, open } from 'lmdb';
 
+import { addMapTests } from '../tests/aztec_map_tests.js';
 import { LmdbAztecMap } from './map.js';
 
 describe('LmdbAztecMap', () => {
   let db: Database;
-  let map: LmdbAztecMap<string, string>;
 
   beforeEach(() => {
     db = open({ dupSort: true } as any);
-    map = new LmdbAztecMap(db, 'test');
   });
 
-  it('should be able to set and get values', async () => {
-    await map.set('foo', 'bar');
-    await map.set('baz', 'qux');
-
-    expect(map.get('foo')).toEqual('bar');
-    expect(map.get('baz')).toEqual('qux');
-    expect(map.get('quux')).toEqual(undefined);
-  });
-
-  it('should be able to set values if they do not exist', async () => {
-    expect(await map.setIfNotExists('foo', 'bar')).toEqual(true);
-    expect(await map.setIfNotExists('foo', 'baz')).toEqual(false);
-
-    expect(map.get('foo')).toEqual('bar');
-  });
-
-  it('should be able to delete values', async () => {
-    await map.set('foo', 'bar');
-    await map.set('baz', 'qux');
-
-    await map.delete('foo');
-
-    expect(map.get('foo')).toEqual(undefined);
-    expect(map.get('baz')).toEqual('qux');
-  });
-
-  it('should be able to iterate over entries', async () => {
-    await map.set('foo', 'bar');
-    await map.set('baz', 'qux');
-
-    expect([...map.entries()]).toEqual([
-      ['baz', 'qux'],
-      ['foo', 'bar'],
-    ]);
-  });
-
-  it('should be able to iterate over values', async () => {
-    await map.set('foo', 'bar');
-    await map.set('baz', 'quux');
-
-    expect([...map.values()]).toEqual(['quux', 'bar']);
-  });
-
-  it('should be able to iterate over keys', async () => {
-    await map.set('foo', 'bar');
-    await map.set('baz', 'qux');
-
-    expect([...map.keys()]).toEqual(['baz', 'foo']);
-  });
-
-  it('should be able to get multiple values for a single key', async () => {
-    await map.set('foo', 'bar');
-    await map.set('foo', 'baz');
-
-    expect([...map.getValues('foo')]).toEqual(['bar', 'baz']);
-  });
-
-  it('supports tuple keys', async () => {
-    const map = new LmdbAztecMap<[number, string], string>(db, 'test');
-
-    await map.set([5, 'bar'], 'val');
-    await map.set([0, 'foo'], 'val');
-
-    expect([...map.keys()]).toEqual([
-      [0, 'foo'],
-      [5, 'bar'],
-    ]);
-
-    expect(map.get([5, 'bar'])).toEqual('val');
-  });
-
-  it('supports range queries', async () => {
-    await map.set('a', 'a');
-    await map.set('b', 'b');
-    await map.set('c', 'c');
-    await map.set('d', 'd');
-
-    expect([...map.keys({ start: 'b', end: 'c' })]).toEqual(['b']);
-    expect([...map.keys({ start: 'b' })]).toEqual(['b', 'c', 'd']);
-    expect([...map.keys({ end: 'c' })]).toEqual(['a', 'b']);
-    expect([...map.keys({ start: 'b', end: 'c', reverse: true })]).toEqual(['c']);
-    expect([...map.keys({ start: 'b', limit: 1 })]).toEqual(['b']);
-    expect([...map.keys({ start: 'b', reverse: true })]).toEqual(['d', 'c']);
-    expect([...map.keys({ end: 'b', reverse: true })]).toEqual(['b', 'a']);
-  });
+  addMapTests(() => new LmdbAztecMap(db, 'test'));
 });

yarn-project/kv-store/src/lmdb/store.test.ts

@@ -0,0 +1,6 @@
+import { addStoreTests } from '../tests/aztec_store_tests.js';
+import { AztecLmdbStore } from './store.js';
+
+describe('AztecLmdbStore', () => {
+  addStoreTests(() => AztecLmdbStore.open());
+});

yarn-project/kv-store/src/mem/array.test.ts

@@ -0,0 +1,7 @@
+import { addArrayTests } from '../tests/aztec_array_tests.js';
+import { MemAztecArray } from './array.js';
+import { MemDb } from './mem_db.js';
+
+describe('MemAztecArray', () => {
+  addArrayTests(() => new MemAztecArray('test', new MemDb()));
+});

yarn-project/kv-store/src/mem/array.ts

@@ -0,0 +1,72 @@
+import { type AztecArray } from '../interfaces/array.js';
+import type { MemDb } from './mem_db.js';
+
+/**
+ * An persistent array backed by mem.
+ */
+export class MemAztecArray<T> implements AztecArray<T> {
+  private slot: string;
+
+  constructor(private name: string, private db: MemDb) {
+    this.slot = JSON.stringify(['array', this.name]);
+  }
+
+  get length(): number {
+    return this.db.get(this.slot)?.length || 0;
+  }
+
+  push(...vals: T[]): Promise<number> {
+    const arr = this.db.get(this.slot);
+    if (arr) {
+      this.db.set(this.slot, [...arr, ...vals]);
+    } else {
+      this.db.set(this.slot, [...vals]);
+    }
+    return Promise.resolve(this.length);
+  }
+
+  pop(): Promise<T | undefined> {
+    const arr = [...this.db.get(this.slot)];
+    const result = arr.pop();
+    this.db.set(this.slot, arr);
+    return Promise.resolve(result);
+  }
+
+  at(index: number): T | undefined {
+    const arr = this.db.get(this.slot) || [];
+    if (index < 0) {
+      return arr[arr.length + index];
+    } else {
+      return arr[index];
+    }
+  }
+
+  setAt(index: number, val: T): Promise<boolean> {
+    if (index < 0) {
+      index = this.length + index;
+    }
+
+    if (index < 0 || index >= this.length) {
+      return Promise.resolve(false);
+    }
+
+    const arr = [...this.db.get(this.slot)];
+    arr[index] = val;
+    this.db.set(this.slot, arr);
+    return Promise.resolve(true);
+  }
+
+  entries(): IterableIterator<[number, T]> {
+    const arr = this.db.get(this.slot) || [];
+    return arr.entries();
+  }
+
+  values(): IterableIterator<T> {
+    const arr = this.db.get(this.slot) || [];
+    return arr.values();
+  }
+
+  [Symbol.iterator](): IterableIterator<T> {
+    return this.values();
+  }
+}

yarn-project/kv-store/src/mem/counter.test.ts

@@ -0,0 +1,7 @@
+import { addCounterTests } from '../tests/aztec_counter_tests.js';
+import { MemAztecCounter } from './counter.js';
+import { MemDb } from './mem_db.js';
+
+describe('MemAztecCounter', () => {
+  addCounterTests(() => new MemAztecCounter('test', new MemDb()));
+});