feat(ava-xs): handle some zoe tests (#2573)

* feat(ava-xs): install as npm bin

* chore(avaHandler): SES Compartment already has harden

* feat(ava-xs): provide makeKind, weakStore to test compartment

* feat(ava-xs): handle ava.files, ava.require from package.json

 - support -v / --verbose
 - xsnap: add glob, @agoric/assert dependencies.
   hm... having ava-xs in the xsnap package is increasingly awkward

* chore(avaXS): log error before losing full message

* feat(ava-xs): test.todo

* feat(ava-xs): exclude takes a list of strings

* test(zoe): XS test exclusions with reasons

* test(zoe): add use --verbose on XS to know which script failed

  - add a separate test:xs-unit-debug script,
    since `ava-xs --verbose --debug` isn't yet supported

* test(zoe): postpone _ in underscores until #2503 is done

* test(zoe): be sure async tests wait for all promises

* test(zoe): handle __dirname for bundling zcfTesterContract

 - clean up some lint

* test(zoe): be flexible about error message from assertIssuerKeywords

* test(zoe): support test.failing on XS foro test-zoeHelpersWZcf.js

 - elaborate diagnostics for checkExpectation

* test(zoe): add test:xs-unit, test:xs-worker to test script

* fix(ava-xs): handle missing ava-xs config section

* feat(ava-xs): report count of tests, each in their own crank

* test(ERTP): use ava-xs bin, ava-xs.exclude

* fix(ava-xs): count where we can see crank boundaries

The code running inside xsnap can't see crank boundaries, so don't
bother to count planned assertions there. Count on the node parent
side.

* style(ava-xs): clarify type in avaHandler

* fix(ava-xs): notThrows() success case

* feat(ava-xs): summarize failed tests

* test(zoe): update list of XS exclusions

* test(zoe): postpone test:xs-worker

* test(zoe): postpone test-scriptedOracle

due to intermittent SEGV

* refactor(ava-xs): export powerless main

* test(zoe): try longer timeout

* fix(ava-xs): use cjs rather than -r esm in #!

* test(zoe): postpone zoeHelpers on XS

* style(ava-xs): jsdoc idiom

Co-authored-by: Kris Kowal <kris@agoric.com>

* style(ava-xs): jsdoc idiom

Co-authored-by: Kris Kowal <kris@agoric.com>

Author: dckc

.github/workflows/test-all-packages.yml

@@ -398,6 +398,7 @@ jobs:
     # END-RESTORE-BOILERPLATE
 
     - name: yarn test (zoe)
+      timeout-minutes: 20
       run: cd packages/zoe && yarn test
       env:
         ESM_DISABLE_CACHE: true

packages/ERTP/package.json

@@ -14,7 +14,7 @@
     "test": "yarn test:node && yarn test:xs",
     "test:node": "ava",
     "test:xs": "yarn test:xs-unit && yarn test:xs-worker",
-    "test:xs-unit": "node -r esm ../xsnap/src/avaXS.js test/unitTests/test-*.js",
+    "test:xs-unit": "ava-xs",
     "test:xs-worker": "WORKER_TYPE=xs-worker ava",
     "test:nyc": "nyc ava",
     "pretty-fix": "prettier --write '**/*.js'",
@@ -66,6 +66,9 @@
     "NEWS.md",
     "exported.js"
   ],
+  "ava-xs": {
+    "exclude": "swingsetTests"
+  },
   "ava": {
     "files": [
       "test/**/test-*.js"

packages/xsnap/package.json

@@ -9,6 +9,7 @@
   },
   "main": "./src/xsnap.js",
   "bin": {
+    "ava-xs": "./src/ava-xs.js",
     "xsrepl": "./src/xsrepl"
   },
   "scripts": {
@@ -25,10 +26,12 @@
     "test": "ava"
   },
   "dependencies": {
+    "@agoric/assert": "^0.2.2-dev.0",
     "@agoric/bundle-source": "^1.2.1",
     "@agoric/eventual-send": "^0.13.2",
     "@agoric/install-ses": "^0.5.1",
     "esm": "^3.2.5",
+    "glob": "^7.1.6",
     "ses": "^0.12.3"
   },
   "devDependencies": {

packages/xsnap/src/ava-xs.js

@@ -0,0 +1,35 @@
+#!/usr/bin/env node
+/* global require, module */
+// @ts-check
+const esmRequire = require('esm')(module);
+const process = require('process');
+const { spawn } = require('child_process');
+const { type: osType } = require('os');
+const { promises } = require('fs');
+const path = require('path');
+const glob = require('glob');
+
+esmRequire('@agoric/install-ses');
+const bundleSource = esmRequire('@agoric/bundle-source').default;
+
+const { main, makeBundleResolve } = esmRequire('./avaXS');
+
+Promise.resolve()
+  .then(_ =>
+    main(process.argv.slice(2), {
+      bundleSource,
+      spawn,
+      osType,
+      readFile: promises.readFile,
+      resolve: makeBundleResolve(path),
+      dirname: path.dirname,
+      glob,
+    }),
+  )
+  .then(status => {
+    process.exit(status);
+  })
+  .catch(err => {
+    console.error(err);
+    process.exit(1);
+  });

packages/xsnap/src/avaAssertXS.js

@@ -105,25 +105,12 @@ let theHarness = null; // ISSUE: ambient
  */
 function createHarness(send) {
   let testNum = 0;
-  let passCount = 0;
   /** @type {((ot: { context: Object }) => Promise<void>)[]} */
-  let beforeHooks = [];
-  /** @type {(() => Promise<void>)[]} */
-  let suitesToRun = [];
+  const beforeHooks = [];
+  /** @type {Record<string, () => Promise<void>>} */
+  const suitesToRun = {};
   const context = {};
 
-  /**
-   * @returns { Summary }
-   * @typedef {import('./avaXS').Summary} Summary
-   */
-  function summary() {
-    return {
-      pass: passCount,
-      fail: testNum - passCount,
-      total: testNum,
-    };
-  }
-
   const it = freeze({
     send,
     get context() {
@@ -134,29 +121,31 @@ function createHarness(send) {
       beforeHooks.push(hook);
     },
     /** @type { (ok: boolean) => number } */
-    finish(ok) {
+    finish(_ok) {
       testNum += 1;
-      if (ok) {
-        passCount += 1;
-      }
       return testNum;
     },
-    /** @type { (thunk: () => Promise<void>) => Promise<void> } */
-    async defer(thunk) {
-      suitesToRun.push(thunk);
+    /** @type { (name: string, thunk: () => Promise<void>) => void } */
+    queue(name, thunk) {
+      if (name in suitesToRun) {
+        throw Error(`duplicate name ${name}`);
+      }
+      suitesToRun[name] = thunk;
+    },
+    testNames() {
+      return keys(suitesToRun);
     },
-    summary,
-    async result() {
+    /**
+     * @param {string} name
+     * @returns { Promise<void> }
+     */
+    async run(name) {
       for await (const hook of beforeHooks) {
         await hook({ context });
       }
-      beforeHooks = [];
-      for await (const suite of suitesToRun) {
-        await suite();
-      }
-      suitesToRun = [];
 
-      return summary();
+      const suite = suitesToRun[name];
+      await suite();
     },
   });
 
@@ -169,17 +158,29 @@ function createHarness(send) {
 /**
  * @param {*} exc
  * @param {Expectation} expectation
- * @returns {boolean}
+ * @returns {null | { expected: unknown, actual: unknown }}
  * @typedef {{ instanceOf: Function } | { message: string | RegExp }=} Expectation
  */
 function checkExpectation(exc, expectation) {
-  if (!expectation) return true;
-  if ('instanceOf' in expectation) return exc instanceof expectation.instanceOf;
+  if (!expectation) return null;
+  if ('instanceOf' in expectation) {
+    if (exc instanceof expectation.instanceOf) {
+      return null;
+    } else {
+      return { expected: expectation.instanceOf, actual: exc };
+    }
+  }
   if ('message' in expectation) {
     const { message } = expectation;
-    return typeof message === 'string'
-      ? message === exc.message
-      : message.test(exc.message);
+    const ok =
+      typeof message === 'string'
+        ? message === exc.message
+        : message.test(exc.message);
+    if (ok) {
+      return null;
+    } else {
+      return { actual: exc.message, expected: message };
+    }
   }
   throw Error(`not implemented: ${JSON.stringify(expectation)}`);
 }
@@ -195,14 +196,8 @@ function checkExpectation(exc, expectation) {
  * @typedef {ReturnType<typeof makeTester>} Tester
  */
 function makeTester(htest, out) {
-  /** @type {number?} */
-  let pending;
-
   /** @type {(result: boolean, info?: string) => void} */
   function assert(result, info) {
-    if (typeof pending === 'number') {
-      pending -= 1;
-    }
     const testNum = htest.finish(result);
     if (result) {
       out.ok(testNum, info);
@@ -222,10 +217,7 @@ function makeTester(htest, out) {
   const t = freeze({
     /** @param {number} count */
     plan(count) {
-      pending = count;
-    },
-    get pending() {
-      return pending;
+      out.plan(count);
     },
     get context() {
       return htest.context;
@@ -289,7 +281,8 @@ function makeTester(htest, out) {
         fn();
         assert(false, message);
       } catch (ex) {
-        assert(checkExpectation(ex, expectation), message);
+        const delta = checkExpectation(ex, expectation);
+        assert(!delta, `${message}: ${JSON.stringify(delta)}`);
       }
     },
     /** @type {(fn: () => unknown, message?: string) => void } */
@@ -298,7 +291,9 @@ function makeTester(htest, out) {
         fn();
       } catch (ex) {
         assert(false, message);
+        return;
       }
+      assert(true, message);
     },
     /** @type {(thrower: () => Promise<unknown>, expectation?: Expectation, message?: string) => Promise<void> } */
     async throwsAsync(
@@ -310,7 +305,8 @@ function makeTester(htest, out) {
         await (typeof thrower === 'function' ? thrower() : thrower);
         assert(false, message);
       } catch (ex) {
-        assert(checkExpectation(ex, expectation), message);
+        const delta = checkExpectation(ex, expectation);
+        assert(!delta, `${message}: ${JSON.stringify(delta)}`);
       }
     },
     /** @type {(thrower: () => Promise<unknown>, message?: string) => Promise<void> } */
@@ -319,38 +315,44 @@ function makeTester(htest, out) {
         await (typeof nonThrower === 'function' ? nonThrower() : nonThrower);
       } catch (ex) {
         assert(false, message);
+        return;
       }
+      assert(true, message);
     },
   });
 
   return t;
 }
 
-/** @type {(label: string, run: (t: Tester) => Promise<void>, htestOpt: Harness?) => void } */
+/**
+ * @param {string} label
+ * @param {(t: Tester) => Promise<void>} run
+ * @param {Harness?} htestOpt
+ */
 function test(label, run, htestOpt) {
   const htest = htestOpt || theHarness;
   if (!htest) throw Error('no harness');
 
-  htest.defer(async () => {
+  htest.queue(label, async () => {
     const out = tapFormat(htest.send);
     const t = makeTester(htest, out);
     try {
       out.diagnostic('start', label);
       await run(t);
       out.diagnostic('end', label);
     } catch (ex) {
+      console.log('FAIL (todo route console)', ex);
       t.fail(`${label} threw: ${ex.message}`);
     }
-    const pending = t.pending;
-    if (typeof pending === 'number' && pending !== 0) {
-      t.fail(`bad plan: ${t.pending} still to go`);
-    }
   });
 }
 
+test.createHarness = createHarness;
+
 // TODO: test.skip, test.failing
 
-test.createHarness = createHarness;
+test.todo = _title => {};
+test.failing = (_title, _implementation) => {};
 
 /** @type {(label: string, fn: () => Promise<void>) => void } */
 test.before = (label, fn) => {