feat: add vatstoreGetAfter syscall to enable iterating vatstore keys

FUDCo · FUDCo · commit 63c7d9759875 · 2021-12-03T18:44:31.000-08:00
diff --git a/packages/SwingSet/src/kernel/kernelSyscall.js b/packages/SwingSet/src/kernel/kernelSyscall.js
@@ -51,6 +51,16 @@ export function makeKernelSyscallHandler(tools) {
     return `${vatID}.vs.${key}`;
   }
 
+  let workingKeyPrefix;
+  let workingPriorKey;
+  let workingKeyIterator;
+
+  function clearVatStoreIteration() {
+    workingKeyPrefix = undefined;
+    workingPriorKey = undefined;
+    workingKeyIterator = undefined;
+  }
+
   function vatstoreGet(vatID, key) {
     const actualKey = vatstoreKeyKey(vatID, key);
     kernelKeeper.incStat('syscalls');
@@ -64,14 +74,59 @@ export function makeKernelSyscallHandler(tools) {
     kernelKeeper.incStat('syscalls');
     kernelKeeper.incStat('syscallVatstoreSet');
     kvStore.set(actualKey, value);
+    clearVatStoreIteration();
     return OKNULL;
   }
 
+  function vatstoreGetAfter(vatID, keyPrefix, priorKey) {
+    const actualKeyPrefix = vatstoreKeyKey(vatID, keyPrefix);
+    const actualPriorKey = vatstoreKeyKey(vatID, priorKey);
+    kernelKeeper.incStat('syscalls');
+    kernelKeeper.incStat('syscallVatstoreGetAfter');
+    let nextIter;
+    if (
+      workingKeyPrefix === actualKeyPrefix &&
+      workingPriorKey === actualPriorKey &&
+      workingKeyIterator
+    ) {
+      nextIter = workingKeyIterator.next();
+    } else {
+      let startKey;
+      if (priorKey === '') {
+        startKey = actualKeyPrefix;
+      } else {
+        startKey = actualPriorKey;
+      }
+      assert(startKey.startsWith(actualKeyPrefix));
+      const lastChar = String.fromCharCode(
+        actualKeyPrefix.slice(-1).charCodeAt(0) + 1,
+      );
+      const endKey = `${actualKeyPrefix.slice(0, -1)}${lastChar}`;
+      workingKeyPrefix = actualKeyPrefix;
+      workingKeyIterator = kvStore.getKeys(startKey, endKey);
+      nextIter = workingKeyIterator.next();
+      if (!nextIter.done && nextIter.value === actualPriorKey) {
+        nextIter = workingKeyIterator.next();
+      }
+    }
+    if (nextIter.done) {
+      clearVatStoreIteration();
+      return harden(['ok', undefined]);
+    } else {
+      const nextKey = nextIter.value;
+      const resultValue = kvStore.get(nextKey);
+      workingPriorKey = nextKey;
+      const resultKey = nextKey.slice(vatID.length + 4); // `${vatID}.vs.`.length
+      return harden(['ok', [resultKey, resultValue]]);
+    }
+  }
+
   function vatstoreDelete(vatID, key) {
     const actualKey = vatstoreKeyKey(vatID, key);
     kernelKeeper.incStat('syscalls');
     kernelKeeper.incStat('syscallVatstoreDelete');
     kvStore.delete(actualKey);
+    clearVatStoreIteration();
     return OKNULL;
   }
 
@@ -160,6 +215,8 @@ export function makeKernelSyscallHandler(tools) {
         return vatstoreGet(...args);
       case 'vatstoreSet':
         return vatstoreSet(...args);
+      case 'vatstoreGetAfter':
+        return vatstoreGetAfter(...args);
       case 'vatstoreDelete':
         return vatstoreDelete(...args);
       case 'dropImports':
diff --git a/packages/SwingSet/src/kernel/liveSlots.js b/packages/SwingSet/src/kernel/liveSlots.js
@@ -12,6 +12,7 @@ import { insistVatType, makeVatSlot, parseVatSlot } from '../parseVatSlots.js';
 import { insistCapData } from '../capdata.js';
 import { insistMessage } from '../message.js';
 import { makeVirtualObjectManager } from './virtualObjectManager.js';
+import { insistValidVatstoreKey } from './vatTranslator.js';
 
 const DEFAULT_VIRTUAL_OBJECT_CACHE_SIZE = 3; // XXX ridiculously small value to force churn for testing
 
@@ -1080,16 +1081,38 @@ function build(
     if (enableVatstore) {
       vpow.vatstore = harden({
         get: key => {
-          assert.typeof(key, 'string');
+          insistValidVatstoreKey(key);
           return syscall.vatstoreGet(`vvs.${key}`);
         },
         set: (key, value) => {
-          assert.typeof(key, 'string');
+          insistValidVatstoreKey(key);
           assert.typeof(value, 'string');
           syscall.vatstoreSet(`vvs.${key}`, value);
         },
+        getAfter: (keyPrefix, priorKey) => {
+          insistValidVatstoreKey(keyPrefix);
+          if (priorKey !== '') {
+            insistValidVatstoreKey(priorKey);
+            assert(
+              priorKey.startsWith(keyPrefix),
+              'priorKey must start with keyPrefix',
+            );
+            priorKey = `vvs.${priorKey}`;
+          }
+          const fetched = syscall.vatstoreGetAfter(
+            `vvs.${keyPrefix}`,
+            priorKey,
+          );
+          if (fetched) {
+            const [key, value] = fetched;
+            assert(key.startsWith('vvs.'));
+            return [key.slice(4), value];
+          } else {
+            return undefined;
+          }
+        },
         delete: key => {
-          assert.typeof(key, 'string');
+          insistValidVatstoreKey(key);
           syscall.vatstoreDelete(`vvs.${key}`);
         },
       });
diff --git a/packages/SwingSet/src/kernel/metrics.js b/packages/SwingSet/src/kernel/metrics.js
@@ -40,6 +40,11 @@ export const KERNEL_STATS_SUM_METRICS = [
     name: 'swingset_syscall_vatstore_set_total',
     description: 'Total number of SwingSet vatstore set kernel calls',
   },
+  {
+    key: 'syscallVatstoreGetAfter',
+    name: 'swingset_syscall_vatstore_getAfter_total',
+    description: 'Total number of SwingSet vatstore getAfter kernel calls',
+  },
   {
     key: 'syscallVatstoreDelete',
     name: 'swingset_syscall_vatstore_delete_total',
diff --git a/packages/SwingSet/src/kernel/vatManager/supervisor-helper.js b/packages/SwingSet/src/kernel/vatManager/supervisor-helper.js
@@ -117,10 +117,18 @@ function makeSupervisorSyscall(syscallToManager, workerCanBlock) {
       doSyscall(['callNow', target, method, args]),
     vatstoreGet: key => doSyscall(['vatstoreGet', key]),
     vatstoreSet: (key, value) => doSyscall(['vatstoreSet', key, value]),
+    vatstoreGetAfter: (keyPrefix, priorKey) =>
+      doSyscall(['vatstoreGetAfter', keyPrefix, priorKey]),
     vatstoreDelete: key => doSyscall(['vatstoreDelete', key]),
   };
 
-  const blocking = ['callNow', 'vatstoreGet', 'vatstoreSet', 'vatstoreDelete'];
+  const blocking = [
+    'callNow',
+    'vatstoreGet',
+    'vatstoreSet',
+    'vatstoreGetAfter',
+    'vatstoreDelete',
+  ];
 
   if (!workerCanBlock) {
     for (const name of blocking) {
diff --git a/packages/SwingSet/src/kernel/vatTranslator.js b/packages/SwingSet/src/kernel/vatTranslator.js
@@ -152,6 +152,11 @@ function makeTranslateKernelDeliveryToVatDelivery(vatID, kernelKeeper) {
   return kernelDeliveryToVatDelivery;
 }
 
+export function insistValidVatstoreKey(key) {
+  assert.typeof(key, 'string');
+  assert(key.match(/^[-\w.+/]+$/), 'invalid vatstore key');
+}
+
 /*
  * return a function that converts VatSyscall objects into KernelSyscall
  * objects
@@ -212,11 +217,6 @@ function makeTranslateVatSyscallToKernelSyscall(vatID, kernelKeeper) {
     return harden(['exit', vatID, !!isFailure, kernelInfo]);
   }
 
-  function insistValidVatstoreKey(key) {
-    assert.typeof(key, 'string');
-    assert(key.match(/^[-\w.+/]+$/));
-  }
-
   function translateVatstoreGet(key) {
     insistValidVatstoreKey(key);
     kdebug(`syscall[${vatID}].vatstoreGet(${key})`);
@@ -230,6 +230,15 @@ function makeTranslateVatSyscallToKernelSyscall(vatID, kernelKeeper) {
     return harden(['vatstoreSet', vatID, key, value]);
   }
 
+  function translateVatstoreGetAfter(keyPrefix, priorKey) {
+    insistValidVatstoreKey(keyPrefix);
+    if (priorKey !== '') {
+      insistValidVatstoreKey(priorKey);
+    }
+    kdebug(`syscall[${vatID}].vatstoreGetAfter(${keyPrefix}, ${priorKey})`);
+    return harden(['vatstoreGetAfter', vatID, keyPrefix, priorKey]);
+  }
+
   function translateVatstoreDelete(key) {
     insistValidVatstoreKey(key);
     kdebug(`syscall[${vatID}].vatstoreDelete(${key})`);
@@ -366,6 +375,8 @@ function makeTranslateVatSyscallToKernelSyscall(vatID, kernelKeeper) {
         return translateVatstoreGet(...args);
       case 'vatstoreSet':
         return translateVatstoreSet(...args);
+      case 'vatstoreGetAfter':
+        return translateVatstoreGetAfter(...args);
       case 'vatstoreDelete':
         return translateVatstoreDelete(...args);
       case 'dropImports':
@@ -418,6 +429,13 @@ function makeTranslateKernelSyscallResultToVatSyscallResult(
         } else {
           return harden(['ok', undefined]);
         }
+      case 'vatstoreGetAfter':
+        if (resultData) {
+          assert(Array.isArray(resultData));
+          return harden(['ok', resultData]);
+        } else {
+          return harden(['ok', undefined]);
+        }
       default:
         assert(resultData === null);
         return harden(['ok', null]);
diff --git a/packages/SwingSet/src/message.js b/packages/SwingSet/src/message.js
@@ -151,6 +151,12 @@ export function insistVatSyscallObject(vso) {
       assert.typeof(data, 'string');
       break;
     }
+    case 'vatstoreGetAfter': {
+      const [keyPrefix, priorKey] = rest;
+      assert.typeof(keyPrefix, 'string');
+      assert.typeof(priorKey, 'string');
+      break;
+    }
     case 'vatstoreDelete': {
       const [key] = rest;
       assert.typeof(key, 'string');
diff --git a/packages/SwingSet/test/test-vatstore.js b/packages/SwingSet/test/test-vatstore.js
@@ -40,6 +40,37 @@ test('vatstore', async t => {
   send('get', 'zot');
   await c.run();
 
+  for (const k of hostStorage.kvStore.getKeys('', '')) {
+    if (k.endsWith('.foo')) {
+      t.is(k, 'v1.vs.vvs.foo');
+    }
+  }
+
+  send('store', 'x.1', 'one');
+  send('store', 'x.2', 'two');
+  send('store', 'x.3', 'three');
+  send('store', 'x.a', 'four');
+  send('store', 'x.qrz', 'five');
+  send('store', 'xxx', 'not this');
+  send('store', 'y.1', 'oney');
+  send('store', 'y.2', 'twoy');
+  send('store', 'y.3', 'threey');
+  send('store', 'y.a', 'foury');
+  send('store', 'y.b', 'fivey');
+  send('store', 'y.c', 'sixy');
+  send('store', 'y.d', 'seveny');
+  send('store', 'yyy', 'not thisy');
+  // check that we hit all the 'x.' keys
+  send('scan', 'x.');
+  // check that this works even if the iteration is interrupted
+  send('scan', 'x.', 3);
+  // check that interleaved iterations don't interfere
+  send('scanInterleaved', 'x.', 'y.');
+  // check for a successful empty iteration if there's no match
+  send('scan', 'z.');
+  // exercise various calls with malformed parameters
+  send('apiAbuse', 'x.');
+  await c.run();
   t.deepEqual(c.dump().log, [
     'get zot -> <undefined>',
     'store zot <- "first zot"',
@@ -51,10 +82,52 @@ test('vatstore', async t => {
     'delete zot',
     'get foo -> "first foo"',
     'get zot -> <undefined>',
+    'store x.1 <- "one"',
+    'store x.2 <- "two"',
+    'store x.3 <- "three"',
+    'store x.a <- "four"',
+    'store x.qrz <- "five"',
+    'store xxx <- "not this"',
+    'store y.1 <- "oney"',
+    'store y.2 <- "twoy"',
+    'store y.3 <- "threey"',
+    'store y.a <- "foury"',
+    'store y.b <- "fivey"',
+    'store y.c <- "sixy"',
+    'store y.d <- "seveny"',
+    'store yyy <- "not thisy"',
+    'scan x.:',
+    '    x.1 -> one',
+    '    x.2 -> two',
+    '    x.3 -> three',
+    '    x.a -> four',
+    '    x.qrz -> five',
+    'scan x. 3:',
+    '    x.1 -> one',
+    '    x.2 -> two',
+    '    x.3 -> three',
+    '    interrupting...',
+    '    x.a -> four',
+    '    x.qrz -> five',
+    'scanInterleaved x. y.:',
+    '    1: x.1 -> one',
+    '    2: y.1 -> oney',
+    '    1: x.2 -> two',
+    '    2: y.2 -> twoy',
+    '    1: x.3 -> three',
+    '    2: y.3 -> threey',
+    '    1: x.a -> four',
+    '    2: y.a -> foury',
+    '    1: x.qrz -> five',
+    '    2: y.b -> fivey',
+    '    2: y.c -> sixy',
+    '    2: y.d -> seveny',
+    'scan z.:',
+    'apiAbuse x.: use prefix as prior key (should work)',
+    '  x.1 -> one',
+    'apiAbuse x.: use out of range prior key aaax.',
+    '  getAfter(x., aaax.) threw Error: priorKey must start with keyPrefix',
+    'apiAbuse x.: use invalid key prefix',
+    '  getAfter("ab@%%$#", "") threw Error: invalid vatstore key',
   ]);
-  for (const k of hostStorage.kvStore.getKeys('', '')) {
-    if (k.endsWith('.foo')) {
-      t.is(k, 'v1.vs.vvs.foo');
-    }
-  }
 });
diff --git a/packages/SwingSet/test/vat-vatstore.js b/packages/SwingSet/test/vat-vatstore.js
