feat(xsnap): Add Node.js shell

Author: kriskowal

package.json

@@ -42,6 +42,7 @@
     "packages/deployment",
     "packages/notifier",
     "packages/xs-vat-worker",
+    "packages/xsnap",
     "packages/deploy-script-support"
   ],
   "devDependencies": {

packages/xsnap/.gitignore

@@ -0,0 +1 @@
+build

packages/xsnap/README.md

@@ -0,0 +1,40 @@
+# xsnap
+
+Xsnap is a utility for taking resumable snapshots of a running JavaScript
+worker, using Moddable’s XS JavaScript engine.
+
+Xsnap provides a Node.js API for controlling Xsnap workers.
+
+```js
+const worker = xsnap();
+await worker.evaluate(`
+  // Incrementer, running on XS.
+  function answerSysCall(message) {
+    const number = Number(String.fromArrayBuffer(message));
+    return ArrayBuffer.fromString(String(number + 1));
+  }
+`);
+await worker.snapshot('bootstrap.xss');
+await worker.close();
+```
+
+Some time later, possibly on a different computer…
+
+```js
+const decoder = new TextDecoder();
+const worker = xsnap({ snapshot: 'bootstrap.xss' });
+const answer = await worker.sysCall('1');
+console.log(decoder.decode(answer)); // 2
+await worker.close();
+```
+
+The parent and child communicate using "syscalls".
+
+- The XS child uses the synchronous `sysCall` function to send a request and
+  receive as response from the Node.js parent.
+- The XS child can implement a synchronous `answserSysCall` function to respond
+  to syscalls from the Node.js parent.
+- The Node.js parent uses an asynchronous `sysCall` method to send a request
+  and receive a response from the XS child.
+- The Node.js parent can implement an asynchronous `answerSysCall` function to
+  respond to syscalls from the XS child.

packages/xsnap/jsconfig.json

@@ -0,0 +1,18 @@
+// This file can contain .js-specific Typescript compiler config.
+{
+  "compilerOptions": {
+    "target": "esnext",
+
+    "noEmit": true,
+/*
+    // The following flags are for creating .d.ts files:
+    "noEmit": false,
+    "declaration": true,
+    "emitDeclarationOnly": true,
+*/
+    "downlevelIteration": true,
+    "strictNullChecks": true,
+    "moduleResolution": "node",
+  },
+  "include": ["src/**/*.js", "exported.js", "tools/**/*.js"],
+}

packages/xsnap/package.json

@@ -0,0 +1,58 @@
+{
+  "name": "@agoric/xsnap",
+  "version": "0.0.0+1-dev",
+  "description": "Description forthcoming.",
+  "author": "Agoric",
+  "license": "Apache-2.0",
+  "parsers": {
+    "js": "mjs"
+  },
+  "main": "./src/xsnap.js",
+  "scripts": {
+    "build": "node -r esm src/build.js",
+    "clean": "rm -rf build",
+    "lint": "yarn lint:js && yarn lint:types",
+    "lint:js": "eslint 'src/**/*.js'",
+    "lint:types": "tsc -p jsconfig.json",
+    "lint-fix": "eslint --fix 'src/**/*.js'",
+    "lint-check": "yarn lint",
+    "test": "ava",
+    "postinstall": "yarn build"
+  },
+  "dependencies": {},
+  "devDependencies": {
+    "@rollup/plugin-node-resolve": "^6.1.0",
+    "ava": "^3.12.1",
+    "esm": "^3.2.5",
+    "rollup-plugin-terser": "^5.1.3"
+  },
+  "files": [
+    "LICENSE*",
+    "makefiles",
+    "src"
+  ],
+  "publishConfig": {
+    "access": "public"
+  },
+  "eslintConfig": {
+    "extends": [
+      "@agoric"
+    ],
+    "ignorePatterns": [
+      "examples/**/*.js"
+    ]
+  },
+  "ava": {
+    "files": [
+      "test/**/test-*.js"
+    ],
+    "require": [
+      "esm"
+    ],
+    "timeout": "2m"
+  },
+  "prettier": {
+    "trailingComma": "all",
+    "singleQuote": true
+  }
+}

packages/xsnap/src/build.js

@@ -0,0 +1,35 @@
+import * as childProcess from 'child_process';
+import os from 'os';
+
+function exec(command, cwd) {
+  const child = childProcess.spawn(command, {
+    cwd,
+    stdio: ['inherit', 'inherit', 'inherit'],
+  });
+  return new Promise((resolve, reject) => {
+    child.on('close', () => {
+      resolve();
+    });
+    child.on('error', err => {
+      reject(new Error(`${command} error ${err}`));
+    });
+    child.on('exit', code => {
+      if (code !== 0) {
+        reject(new Error(`${command} exited with code ${code}`));
+      }
+    });
+  });
+}
+
+(async () => {
+  // Run command depending on the OS
+  if (os.type() === 'Linux') {
+    await exec('make', 'makefiles/lin');
+  } else if (os.type() === 'Darwin') {
+    await exec('make', 'makefiles/mac');
+  } else if (os.type() === 'Windows_NT') {
+    await exec('nmake', 'makefiles/win');
+  } else {
+    throw new Error(`Unsupported OS found: ${os.type()}`);
+  }
+})();

packages/xsnap/src/defer.js

@@ -0,0 +1,33 @@
+// @ts-check
+
+// eslint-disable-next-line jsdoc/require-returns-check
+/**
+ * @param {boolean} _flag
+ * @returns {asserts _flag}
+ */
+function assert(_flag) {}
+
+/**
+ * @template T
+ * @typedef {{
+ *   resolve(value?: T | Promise<T>): void,
+ *   reject(error: Error): void,
+ *   promise: Promise<T>
+ * }} Deferred
+ */
+
+/**
+ * @template T
+ * @returns {Deferred<T>}
+ */
+export function defer() {
+  let resolve;
+  let reject;
+  const promise = new Promise((res, rej) => {
+    resolve = res;
+    reject = rej;
+  });
+  assert(resolve !== undefined);
+  assert(reject !== undefined);
+  return { promise, resolve, reject };
+}

packages/xsnap/src/netstring.js

@@ -0,0 +1,117 @@
+// @ts-check
+
+/**
+ * @template T
+ * @template U
+ * @template V
+ * @typedef {import('./stream.js').Stream<T, U, V>} Stream
+ */
+
+const COLON = ':'.charCodeAt(0);
+const COMMA = ','.charCodeAt(0);
+
+const decoder = new TextDecoder();
+const encoder = new TextEncoder();
+
+/**
+ * @param {AsyncIterable<Uint8Array>} input
+ * @param {string=} name
+ * @param {number=} capacity
+ * @returns {AsyncIterableIterator<Uint8Array>} input
+ */
+export async function* reader(input, name = '<unknown>', capacity = 1024) {
+  let length = 0;
+  let buffer = new Uint8Array(capacity);
+  let offset = 0;
+
+  for await (const chunk of input) {
+    if (length + chunk.byteLength >= capacity) {
+      while (length + chunk.byteLength >= capacity) {
+        capacity *= 2;
+      }
+      const replacement = new Uint8Array(capacity);
+      replacement.set(buffer, 0);
+      buffer = replacement;
+    }
+    buffer.set(chunk, length);
+    length += chunk.byteLength;
+
+    let drained = false;
+    while (!drained && length > 0) {
+      const colon = buffer.indexOf(COLON);
+      if (colon === 0) {
+        throw new Error(
+          `Expected number before colon at offset ${offset} of ${name}`,
+        );
+      } else if (colon > 0) {
+        const prefixBytes = buffer.subarray(0, colon);
+        const prefixString = decoder.decode(prefixBytes);
+        const contentLength = +prefixString;
+        if (Number.isNaN(contentLength)) {
+          throw new Error(
+            `Invalid netstring prefix length ${prefixString} at offset ${offset} of ${name}`,
+          );
+        }
+        const messageLength = colon + contentLength + 2;
+        if (messageLength <= length) {
+          yield buffer.subarray(colon + 1, colon + 1 + contentLength);
+          buffer.copyWithin(0, messageLength);
+          length -= messageLength;
+          offset += messageLength;
+        } else {
+          drained = true;
+        }
+      } else {
+        drained = true;
+      }
+    }
+  }
+
+  if (length > 0) {
+    throw new Error(
+      `Unexpected dangling message at offset ${offset} of ${name}`,
+    );
+  }
+}
+
+/**
+ * @param {Stream<void, Uint8Array, void>} output
+ * @returns {Stream<void, Uint8Array, void>}
+ */
+export function writer(output) {
+  const scratch = new Uint8Array(8);
+
+  return {
+    async next(message) {
+      const { written: length = 0 } = encoder.encodeInto(
+        `${message.byteLength}`,
+        scratch,
+      );
+      scratch[length] = COLON;
+
+      const { done: done1 } = await output.next(
+        scratch.subarray(0, length + 1),
+      );
+      if (done1) {
+        return output.return();
+      }
+
+      const { done: done2 } = await output.next(message);
+      if (done2) {
+        return output.return();
+      }
+
+      scratch[0] = COMMA;
+      return output.next(scratch.subarray(0, 1));
+    },
+    async return() {
+      return output.return();
+    },
+    async throw(error) {
+      return output.throw(error);
+    },
+    [Symbol.asyncIterator]() {
+      return this;
+    },
+  };
+}

packages/xsnap/src/node-stream.js

@@ -0,0 +1,71 @@
+// @ts-check
+
+/**
+ * @template T
+ * @template U
+ * @template V
+ * @typedef {import('./stream.js').Stream<T, U, V>} Stream
+ */
+
+/**
+ * @template T
+ * @typedef {import('./defer.js').Deferred<T>} Deferred
+ */
+import { defer } from './defer';
+
+const continues = { value: undefined };
+
+/**
+ * Adapts a Node.js writable stream to a JavaScript
+ * async iterator of Uint8Array data chunks.
+ * Back pressure emerges from awaiting on the promise
+ * returned by `next` before calling `next` again.
+ *
+ * @param {NodeJS.WritableStream} output
+ * @returns {Stream<void, Uint8Array, void>}
+ */
+export function writer(output) {
+  /**
+   * @type {Deferred<IteratorResult<void>>}
+   */
+  let drained = defer();
+  drained.resolve(continues);
+
+  output.on('error', err => {
+    console.log('err', err);
+    drained.reject(err);
+  });
+
+  output.on('drain', () => {
+    drained.resolve(continues);
+    drained = defer();
+  });
+
+  return {
+    /**
+     * @param {Uint8Array} [chunk]
+     * @returns {Promise<IteratorResult<void>>}
+     */
+    async next(chunk) {
+      if (!chunk) {
+        return continues;
+      }
+      if (!output.write(chunk)) {
+        drained = defer();
+        return drained.promise;
+      }
+      return continues;
+    },
+    async return() {
+      output.end();
+      return drained.promise;
+    },
+    async throw() {
+      output.end();
+      return drained.promise;
+    },
+    [Symbol.asyncIterator]() {
+      return this;
+    },
+  };
+}