Agoric
diff --git a/‎packages/SwingSet/src/controller.js
+19-35 b/‎packages/SwingSet/src/controller.js
+19-35
diff --git a/‎packages/SwingSet/src/initializeSwingset.js
-5 b/‎packages/SwingSet/src/initializeSwingset.js
-5
diff --git a/‎packages/SwingSet/src/kernel/kernel.js
+8-1 b/‎packages/SwingSet/src/kernel/kernel.js
+8-1
diff --git a/‎packages/SwingSet/src/kernel/state/kernelKeeper.js
+9-1 b/‎packages/SwingSet/src/kernel/state/kernelKeeper.js
+9-1
diff --git a/‎packages/SwingSet/src/kernel/state/vatKeeper.js
+57-1 b/‎packages/SwingSet/src/kernel/state/vatKeeper.js
+57-1
diff --git a/‎packages/SwingSet/src/kernel/vatManager/manager-helper.js
+22-5 b/‎packages/SwingSet/src/kernel/vatManager/manager-helper.js
+22-5
@@ -1,21 +1,19 @@
 /* global require */
 // @ts-check
 import fs from 'fs';
-import path from 'path';
 import process from 'process';
 import re2 from 're2';
 import { performance } from 'perf_hooks';
 import { spawn as ambientSpawn } from 'child_process';
 import { type as osType } from 'os';
 import { Worker } from 'worker_threads';
 import anylogger from 'anylogger';
-import { tmpName } from 'tmp';
 
 import { assert, details as X } from '@agoric/assert';
 import { isTamed, tameMetering } from '@agoric/tame-metering';
 import { importBundle } from '@agoric/import-bundle';
 import { makeMeteringTransformer } from '@agoric/transform-metering';
-import { xsnap, makeSnapstore, recordXSnap } from '@agoric/xsnap';
+import { xsnap, recordXSnap } from '@agoric/xsnap';
 
 import engineGC from './engine-gc.js';
 import { WeakRef, FinalizationRegistry } from './weakref.js';
@@ -49,12 +47,12 @@ function unhandledRejectionHandler(e) {
 /**
  * @param {{ moduleFormat: string, source: string }[]} bundles
  * @param {{
- *   snapstorePath?: string,
+ *   snapStore?: SnapStore,
  *   spawn: typeof import('child_process').spawn
  *   env: Record<string, string | undefined>,
  * }} opts
  */
-export function makeStartXSnap(bundles, { snapstorePath, env, spawn }) {
+export function makeStartXSnap(bundles, { snapStore, env, spawn }) {
   /** @type { import('@agoric/xsnap/src/xsnap').XSnapOptions } */
   const xsnapOpts = {
     os: osType(),
@@ -79,37 +77,27 @@ export function makeStartXSnap(bundles, { snapstorePath, env, spawn }) {
     };
   }
 
-  /** @type { ReturnType<typeof makeSnapstore> } */
-  let snapStore;
-
-  if (snapstorePath) {
-    fs.mkdirSync(snapstorePath, { recursive: true });
-
-    snapStore = makeSnapstore(snapstorePath, {
-      tmpName,
-      existsSync: fs.existsSync,
-      createReadStream: fs.createReadStream,
-      createWriteStream: fs.createWriteStream,
-      rename: fs.promises.rename,
-      unlink: fs.promises.unlink,
-      resolve: path.resolve,
-    });
-  }
-
-  let supervisorHash = '';
   /**
    * @param {string} name
    * @param {(request: Uint8Array) => Promise<Uint8Array>} handleCommand
    * @param { boolean } [metered]
+   * @param { string } [snapshotHash]
    */
-  async function startXSnap(name, handleCommand, metered) {
-    if (supervisorHash) {
-      return snapStore.load(supervisorHash, async snapshot => {
+  async function startXSnap(
+    name,
+    handleCommand,
+    metered,
+    snapshotHash = undefined,
+  ) {
+    if (snapStore && snapshotHash) {
+      // console.log('startXSnap from', { snapshotHash });
+      return snapStore.load(snapshotHash, async snapshot => {
         const xs = doXSnap({ snapshot, name, handleCommand, ...xsnapOpts });
         await xs.evaluate('null'); // ensure that spawn is done
         return xs;
       });
     }
+    // console.log('fresh xsnap', { snapStore: snapStore });
     const meterOpts = metered ? {} : { meteringLimit: 0 };
     const worker = doXSnap({ handleCommand, name, ...meterOpts, ...xsnapOpts });
 
@@ -121,9 +109,6 @@ export function makeStartXSnap(bundles, { snapstorePath, env, spawn }) {
       // eslint-disable-next-line no-await-in-loop
       await worker.evaluate(`(${bundle.source}\n)()`.trim());
     }
-    if (snapStore) {
-      supervisorHash = await snapStore.save(async fn => worker.snapshot(fn));
-    }
     return worker;
   }
   return startXSnap;
@@ -140,7 +125,6 @@ export function makeStartXSnap(bundles, { snapstorePath, env, spawn }) {
  *   slogFile?: string,
  *   testTrackDecref?: unknown,
  *   warehousePolicy?: { maxVatsOnline?: number },
- *   snapstorePath?: string,
  *   spawn?: typeof import('child_process').spawn,
  *   env?: Record<string, string | undefined>
  * }} runtimeOptions
@@ -162,7 +146,6 @@ export async function makeSwingsetController(
     debugPrefix = '',
     slogCallbacks,
     slogFile,
-    snapstorePath,
     spawn = ambientSpawn,
     warehousePolicy = {},
   } = runtimeOptions;
@@ -300,7 +283,11 @@ export async function makeSwingsetController(
     // @ts-ignore assume supervisorBundle is set
     JSON.parse(kvStore.get('supervisorBundle')),
   ];
-  const startXSnap = makeStartXSnap(bundles, { snapstorePath, env, spawn });
+  const startXSnap = makeStartXSnap(bundles, {
+    snapStore: hostStorage.snapStore,
+    env,
+    spawn,
+  });
 
   const kernelEndowments = {
     waitUntilQuiescent,
@@ -430,7 +417,6 @@ export async function makeSwingsetController(
  *   debugPrefix?: string,
  *   slogCallbacks?: unknown,
  *   testTrackDecref?: unknown,
- *   snapstorePath?: string,
  *   warehousePolicy?: { maxVatsOnline?: number },
  *   slogFile?: string,
  * }} runtimeOptions
@@ -447,15 +433,13 @@ export async function buildVatController(
     kernelBundles,
     debugPrefix,
     slogCallbacks,
-    snapstorePath,
     warehousePolicy,
     slogFile,
   } = runtimeOptions;
   const actualRuntimeOptions = {
     verbose,
     debugPrefix,
     slogCallbacks,
-    snapstorePath,
     warehousePolicy,
     slogFile,
   };
 
@@ -310,11 +310,6 @@ export async function initializeSwingset(
   // it to comms
   config.vats.vattp = {
     bundle: kernelBundles.vattp,
-    creationOptions: {
-      // we saw evidence of vattp dropping messages, and out of caution,
-      // we're keeping it on an in-kernel worker for now. See #3039.
-      managerType: 'local',
-    },
   };
 
   // timer wrapper vat is added automatically, but TODO: bootstraps must
 
@@ -125,7 +125,11 @@ export default function buildKernel(
   } = kernelOptions;
   const logStartup = verbose ? console.debug : () => 0;
 
-  const { kvStore, streamStore } = /** @type { HostStore } */ (hostStorage);
+  const {
+    kvStore,
+    streamStore,
+    snapStore,
+  } = /** @type { HostStore } */ (hostStorage);
   insistStorageAPI(kvStore);
   const { enhancedCrankBuffer, abortCrank, commitCrank } = wrapStorage(kvStore);
   const vatAdminRootKref = kvStore.get('vatAdminRootKref');
@@ -138,6 +142,7 @@ export default function buildKernel(
     enhancedCrankBuffer,
     streamStore,
     kernelSlog,
+    snapStore,
   );
 
   const meterManager = makeMeterManager(replaceGlobalMeter);
@@ -673,6 +678,8 @@ export default function buildKernel(
       if (!didAbort) {
         kernelKeeper.processRefcounts();
         kernelKeeper.saveStats();
+        // eslint-disable-next-line no-use-before-define
+        await vatWarehouse.maybeSaveSnapshot();
       }
       commitCrank();
       kernelKeeper.incrementCrankNumber();
 
@@ -52,6 +52,7 @@ const enableKernelGC = true;
 // v$NN.nextDeliveryNum = $NN
 // v$NN.t.endPosition = $NN
 // v$NN.vs.$key = string
+// v$NN.lastSnapshot = JSON({ snapshotID, startPos })
 
 // d$NN.o.nextID = $NN
 // d$NN.c.$kernelSlot = $deviceSlot = o-$NN/d+$NN/d-$NN
@@ -109,8 +110,14 @@ const FIRST_CRANK_NUMBER = 0n;
  * @param {KVStorePlus} kvStore
  * @param {StreamStore} streamStore
  * @param {KernelSlog} kernelSlog
+ * @param {SnapStore=} snapStore
  */
-export default function makeKernelKeeper(kvStore, streamStore, kernelSlog) {
+export default function makeKernelKeeper(
+  kvStore,
+  streamStore,
+  kernelSlog,
+  snapStore = undefined,
+) {
   insistEnhancedStorageAPI(kvStore);
 
   /**
@@ -939,6 +946,7 @@ export default function makeKernelKeeper(kvStore, streamStore, kernelSlog) {
       incStat,
       decStat,
       getCrankNumber,
+      snapStore,
     );
     ephemeral.vatKeepers.set(vatID, vk);
     return vk;
 
@@ -42,7 +42,7 @@ export function initializeVatState(kvStore, streamStore, vatID) {
 /**
  * Produce a vat keeper for a vat.
  *
- * @param {*} kvStore  The keyValue store in which the persistent state will be kept
+ * @param {KVStorePlus} kvStore  The keyValue store in which the persistent state will be kept
  * @param {StreamStore} streamStore  Accompanying stream store, for the transcripts
  * @param {*} kernelSlog
  * @param {string} vatID  The vat ID string of the vat in question
@@ -60,6 +60,7 @@ export function initializeVatState(kvStore, streamStore, vatID) {
  * @param {*} incStat
  * @param {*} decStat
  * @param {*} getCrankNumber
+ * @param { SnapStore= } snapStore
  * returns an object to hold and access the kernel's state for the given vat
  */
 export function makeVatKeeper(
@@ -79,6 +80,7 @@ export function makeVatKeeper(
   incStat,
   decStat,
   getCrankNumber,
+  snapStore = undefined,
 ) {
   insistVatID(vatID);
   const transcriptStream = `transcript-${vatID}`;
@@ -417,6 +419,57 @@ export function makeVatKeeper(
     kvStore.set(`${vatID}.t.endPosition`, `${JSON.stringify(newPos)}`);
   }
 
+  /** @returns { StreamPosition } */
+  function getTranscriptEndPosition() {
+    return JSON.parse(
+      kvStore.get(`${vatID}.t.endPosition`) ||
+        assert.fail('missing endPosition'),
+    );
+  }
+
+  /**
+   * @returns {{ snapshotID: string, startPos: StreamPosition } | undefined}
+   */
+  function getLastSnapshot() {
+    const notation = kvStore.get(`${vatID}.lastSnapshot`);
+    if (!notation) {
+      return undefined;
+    }
+    const { snapshotID, startPos } = JSON.parse(notation);
+    assert.typeof(snapshotID, 'string');
+    assert(startPos);
+    return { snapshotID, startPos };
+  }
+
+  function transcriptSnapshotStats() {
+    const totalEntries = getTranscriptEndPosition().itemCount;
+    const lastSnapshot = getLastSnapshot();
+    const snapshottedEntries = lastSnapshot
+      ? lastSnapshot.startPos.itemCount
+      : 0;
+    return { totalEntries, snapshottedEntries };
+  }
+
+  /**
+   * Store a snapshot, if given a snapStore.
+   *
+   * @param { VatManager } manager
+   * @returns { Promise<boolean> }
+   */
+  async function saveSnapshot(manager) {
+    if (!snapStore || !manager.makeSnapshot) {
+      return false;
+    }
+
+    const snapshotID = await manager.makeSnapshot(snapStore);
+    const endPosition = getTranscriptEndPosition();
+    kvStore.set(
+      `${vatID}.lastSnapshot`,
+      JSON.stringify({ snapshotID, startPos: endPosition }),
+    );
+    return true;
+  }
+
   function vatStats() {
     function getCount(key, first) {
       const id = Nat(BigInt(kvStore.get(key)));
@@ -477,8 +530,11 @@ export function makeVatKeeper(
     deleteCListEntry,
     deleteCListEntriesForKernelSlots,
     getTranscript,
+    transcriptSnapshotStats,
     addToTranscript,
     vatStats,
     dumpState,
+    saveSnapshot,
+    getLastSnapshot,
   });
 }
@@ -47,7 +47,8 @@ import { makeTranscriptManager } from './transcript.js';
 
 /**
  *
- * @typedef { { getManager: (shutdown: () => Promise<void>) => VatManager,
+ * @typedef { { getManager: (shutdown: () => Promise<void>,
+ *                           makeSnapshot?: (ss: SnapStore) => Promise<string>) => VatManager,
  *              syscallFromWorker: (vso: VatSyscallObject) => VatSyscallResult,
  *              setDeliverToWorker: (dtw: unknown) => void,
  *            } } ManagerKit
@@ -178,12 +179,18 @@ function makeManagerKit(
     kernelSlog.write({ type: 'finish-replay-delivery', vatID, deliveryNum });
   }
 
-  async function replayTranscript() {
+  /**
+   * @param {StreamPosition | undefined} startPos
+   * @returns { Promise<number?> } number of deliveries, or null if !useTranscript
+   */
+  async function replayTranscript(startPos) {
+    // console.log('replay from', { vatID, startPos });
+
     if (transcriptManager) {
       const total = vatKeeper.vatStats().transcriptCount;
       kernelSlog.write({ type: 'start-replay', vatID, deliveries: total });
       let deliveryNum = 0;
-      for (const t of vatKeeper.getTranscript()) {
+      for (const t of vatKeeper.getTranscript(startPos)) {
         // if (deliveryNum % 100 === 0) {
         //   console.debug(`replay vatID:${vatID} deliveryNum:${deliveryNum} / ${total}`);
         // }
@@ -194,7 +201,10 @@ function makeManagerKit(
       }
       transcriptManager.checkReplayError();
       kernelSlog.write({ type: 'finish-replay', vatID });
+      return deliveryNum;
     }
+
+    return null;
   }
 
   /**
@@ -235,10 +245,17 @@ function makeManagerKit(
   /**
    *
    * @param { () => Promise<void>} shutdown
+   * @param { (ss: SnapStore) => Promise<string> } makeSnapshot
    * @returns { VatManager }
    */
-  function getManager(shutdown) {
-    return harden({ replayTranscript, replayOneDelivery, deliver, shutdown });
+  function getManager(shutdown, makeSnapshot) {
+    return harden({
+      replayTranscript,
+      replayOneDelivery,
+      deliver,
+      shutdown,
+      makeSnapshot,
+    });
   }
 
   return harden({ getManager, syscallFromWorker, setDeliverToWorker });