feat(xsnap): record / replay xsnap protcol

  - replay multiple folders across snapshots
    - downgrade snapshot errors to non-fatal
    - override stored os with running os

packages/xsnap/src/index.js

@@ -7,3 +7,4 @@ export {
   METER_TYPE,
 } from '../api.js';
 export { makeSnapstore } from './snapStore.js';
+export { recordXSnap, replayXSnap } from './replay.js';

packages/xsnap/src/replay.js

@@ -0,0 +1,300 @@
+/**
+ * Replay usage:
+ *   node -r esm replay.js <folder>...
+ *
+ * In case of more than one folder:
+ *  1. Spawn based on 00000-options.json in the first folder
+ *  2. For all folders but the last,
+ *     replay steps 00001 to the first snapshot step.
+ *  3. For the last folder, play steps 00001 to last.
+ */
+// @ts-check
+
+import { xsnap, DEFAULT_CRANK_METERING_LIMIT } from './xsnap.js';
+import { queue } from './stream.js';
+
+const { freeze } = Object;
+
+const encoder = new TextEncoder();
+
+/** @param { number } n */
+const pad5 = n => `00000${n}`.slice(-5);
+
+/**
+ * @param {string} path
+ * @param {{ writeFileSync: typeof import('fs').writeFileSync }} io
+ */
+function makeSyncStorage(path, { writeFileSync }) {
+  const base = new URL(path, 'file://');
+  return freeze({
+    /** @param {string} fn */
+    file: fn => {
+      /** @param { Uint8Array } data */
+      const put = data => writeFileSync(new URL(fn, base).pathname, data);
+
+      return freeze({
+        put,
+        /** @param { string } txt */
+        putText: txt => put(encoder.encode(txt)),
+      });
+    },
+  });
+}
+
+/**
+ * @param {string} path
+ * @param {{
+ *   readdirSync: typeof import('fs').readdirSync,
+ *   readFileSync: typeof import('fs').readFileSync,
+ * }} io
+ */
+function makeSyncAccess(path, { readdirSync, readFileSync }) {
+  const base = new URL(path, 'file://');
+  /** @param {string} fn */
+  const file = fn => {
+    const fullname = new URL(fn, base).pathname;
+
+    return freeze({
+      getData: () => readFileSync(fullname),
+      getText: () => readFileSync(fullname, 'utf-8'),
+    });
+  };
+  return freeze({ path, file, readdir: () => readdirSync(base.pathname) });
+}
+
+/**
+ * Start an xsnap subprocess controller that records data
+ * flowing to it for replay.
+ *
+ * @param { XSnapOptions } options used
+ *        to create the underlying xsnap subprocess. Note that
+ *        options.handleCommand is wrapped in order to capture
+ *        data sent to the process.
+ * @param { string } folderPath where to store files of the form
+ *        00000-options.json
+ *        00001-evaluate.dat
+ *        00002-issueCommand.dat
+ *        00003-reply.dat
+ * @param {{
+ *   writeFileSync: typeof import('fs').writeFileSync,
+ * }} io
+ * @returns {XSnap}
+ *
+ * @typedef {ReturnType <typeof import('./xsnap.js').xsnap>} XSnap
+ * @typedef { import('./xsnap.js').XSnapOptions } XSnapOptions
+ */
+export function recordXSnap(options, folderPath, { writeFileSync }) {
+  const folder = makeSyncStorage(folderPath, { writeFileSync });
+
+  let ix = 0;
+
+  /**
+   * @param { string } kind
+   * @param { string= } ext
+   */
+  const nextFile = (kind, ext = 'dat') => {
+    const fn = `${pad5(ix)}-${kind}.${ext}`;
+    ix += 1;
+
+    return folder.file(fn);
+  };
+
+  /** @param { Uint8Array } msg */
+  const echo = msg => msg;
+
+  /** @param { Uint8Array} msg */
+  async function handleCommand(msg) {
+    const { handleCommand: handle = echo } = options;
+    const result = await handle(msg);
+    nextFile('reply').put(result);
+    return result;
+  }
+
+  const {
+    os,
+    name = '_replay_',
+    debug = false,
+    parserBufferSize = undefined,
+    snapshot = undefined,
+    meteringLimit = DEFAULT_CRANK_METERING_LIMIT,
+  } = options;
+  nextFile('options', 'json').putText(
+    JSON.stringify({
+      os,
+      name,
+      debug,
+      parserBufferSize,
+      snapshot,
+      meteringLimit,
+    }),
+  );
+
+  const it = xsnap({ ...options, handleCommand });
+
+  return freeze({
+    /** @param { Uint8Array } msg */
+    issueCommand: async msg => {
+      nextFile('issueCommand').put(msg);
+      return it.issueCommand(msg);
+    },
+    issueStringCommand: async str => {
+      nextFile('issueCommand').putText(str);
+      return it.issueStringCommand(str);
+    },
+    close: it.close,
+    terminate: it.terminate,
+    evaluate: async code => {
+      nextFile('evaluate').putText(code);
+      return it.evaluate(code);
+    },
+    execute: async _fileName => {
+      throw Error('recording: execute not supported');
+    },
+    import: async _fileName => {
+      throw Error('recording: import not supported');
+    },
+    snapshot: async file => {
+      nextFile('snapshot').putText(file);
+      return it.snapshot(file);
+    },
+  });
+}
+
+/**
+ * Replay an xsnap subprocess from one or more folders of steps.
+ *
+ * @param {XSnapOptions} opts
+ * @param { string[] } folders
+ * @param {{
+ *   readdirSync: typeof import('fs').readdirSync,
+ *   readFileSync: typeof import('fs').readFileSync,
+ * }} io
+ */
+export async function replayXSnap(
+  opts,
+  folders,
+  { readdirSync, readFileSync },
+) {
+  const replies = queue();
+  async function handleCommand(_msg) {
+    const r = await replies.get();
+    // console.log('handleCommand', { r: decode(r), msg: decode(msg) });
+    return r;
+  }
+
+  /** @param { string } folder */
+  function start(folder) {
+    const rd = makeSyncAccess(folder, { readdirSync, readFileSync });
+    const [optionsFn] = rd.readdir();
+    const storedOpts = JSON.parse(rd.file(optionsFn).getText());
+    console.log(folder, optionsFn, ':', storedOpts);
+    const { os } = opts; // override stored os
+    return xsnap({ ...opts, ...storedOpts, os, handleCommand });
+  }
+
+  let running;
+  const done = [];
+  const it = start(folders[0]);
+
+  /**
+   * @param { ReturnType<typeof makeSyncAccess> } rd
+   * @param { string[] } steps
+   */
+  async function runSteps(rd, steps) {
+    const folder = rd.path;
+    for (const step of steps) {
+      const parts = step.match(/(\d+)-([a-zA-Z]+)\.(dat|json)$/);
+      if (!parts) {
+        throw Error(`expected 0001-abc.dat; got: ${step}`);
+      }
+      const [_match, digits, kind] = parts;
+      const seq = parseInt(digits, 10);
+      console.log(folder, seq, kind);
+      if (running && kind !== 'reply') {
+        // eslint-disable-next-line no-await-in-loop
+        await running;
+        running = undefined;
+      }
+      const file = rd.file(step);
+      switch (kind) {
+        case 'evaluate':
+          running = it.evaluate(file.getText());
+          break;
+        case 'issueCommand':
+          running = it.issueCommand(file.getData());
+          break;
+        case 'reply':
+          replies.put(file.getData());
+          break;
+        case 'snapshot':
+          if (folders.length > 1 && folder !== folders.slice(-1)[0]) {
+            console.log(folder, step, 'ignoring remaining steps from', folder);
+            return;
+          } else {
+            try {
+              // eslint-disable-next-line no-await-in-loop
+              await it.snapshot(file.getText());
+            } catch (err) {
+              console.warn(err, 'while taking snapshot:', err);
+            }
+          }
+          break;
+        default:
+          console.log(`bad kind: ${kind}`);
+          throw RangeError(`bad kind: ${kind}`);
+      }
+      done.push([folder, seq, kind]);
+    }
+  }
+
+  for await (const folder of folders) {
+    const rd = makeSyncAccess(folder, { readdirSync, readFileSync });
+
+    const [optionsFn, ...steps] = rd.readdir();
+    if (folder !== folders[0]) {
+      const storedOpts = JSON.parse(rd.file(optionsFn).getText());
+      console.log(folder, optionsFn, 'already spawned; ignoring:', storedOpts);
+    }
+    await runSteps(rd, steps);
+  }
+
+  await it.close();
+  return done;
+}
+
+/**
+ *
+ * @param {string[]} argv
+ * @param {{
+ *   spawn: typeof import('child_process').spawn,
+ *   osType: typeof import('os').type,
+ *   readdirSync: typeof import('fs').readdirSync,
+ *   readFileSync: typeof import('fs').readFileSync,
+ * }} io
+ */
+export async function main(argv, { spawn, osType, readdirSync, readFileSync }) {
+  const folders = argv;
+  if (!folders) {
+    throw Error(`usage: replay folder...`);
+  }
+  /** @type { import('./xsnap.js').XSnapOptions } */
+  const options = { spawn, os: osType(), stdout: 'inherit', stderr: 'inherit' };
+  await replayXSnap(options, folders, { readdirSync, readFileSync });
+}
+
+/* global require, module, process, require */
+if (typeof require !== 'undefined' && require.main === module) {
+  main([...process.argv.slice(2)], {
+    // eslint-disable-next-line global-require
+    spawn: require('child_process').spawn,
+    // eslint-disable-next-line global-require
+    osType: require('os').type,
+    // eslint-disable-next-line global-require
+    readdirSync: require('fs').readdirSync,
+    // eslint-disable-next-line global-require
+    readFileSync: require('fs').readFileSync,
+  }).catch(err => {
+    console.error(err);
+    process.exit(err.code || 1);
+  });
+}

packages/xsnap/src/xsnap.js

@@ -17,7 +17,7 @@ import * as netstring from './netstring.js';
 import * as node from './node-stream.js';
 
 // This will need adjustment, but seems to be fine for a start.
-const DEFAULT_CRANK_METERING_LIMIT = 1e7;
+export const DEFAULT_CRANK_METERING_LIMIT = 1e7;
 
 const OK = '.'.charCodeAt(0);
 const ERROR = '!'.charCodeAt(0);

packages/xsnap/test/test-replay.js

@@ -0,0 +1,77 @@
+// @ts-check
+/* global Buffer */
+// eslint-disable-next-line import/no-extraneous-dependencies
+import test from 'ava';
+
+import * as proc from 'child_process';
+import * as os from 'os';
+
+import { recordXSnap, replayXSnap } from '../src/replay.js';
+
+import { options, encode, decode } from './message-tools.js';
+
+const io = { spawn: proc.spawn, os: os.type() }; // WARNING: ambient
+
+const transcript1 = [
+  [
+    '/xsnap-tests/00000-options.json',
+    '{"os":"Linux","name":"xsnap test worker","debug":false,"meteringLimit":10000000}',
+  ],
+  [
+    '/xsnap-tests/00001-evaluate.dat',
+    'issueCommand(ArrayBuffer.fromString("Hello, World!"));',
+  ],
+  ['/xsnap-tests/00002-reply.dat', ''],
+];
+
+test('record: evaluate and issueCommand', async t => {
+  const opts = options(io);
+
+  /** @type { Map<string, Uint8Array> } */
+  const files = new Map();
+  const writeFileSync = (fn, bs) => files.set(fn, bs);
+
+  const vat = recordXSnap(opts, '/xsnap-tests/', { writeFileSync });
+
+  await vat.evaluate(`issueCommand(ArrayBuffer.fromString("Hello, World!"));`);
+  await vat.close();
+  t.deepEqual(['Hello, World!'], opts.messages);
+
+  t.deepEqual(
+    transcript1,
+    [...files].map(([k, v]) => [k, decode(v)]),
+  );
+});
+
+test('replay', async t => {
+  const opts = options(io);
+
+  /** @type { Map<string, Uint8Array> } */
+  const files = new Map(transcript1.map(([k, v]) => [k, encode(v)]));
+  const mockFS = {
+    readdirSync: (_folder, _opts) => [...files.keys()],
+    readFileSync: (n, encoding) => {
+      const bytes = files.get(n);
+      if (bytes === undefined) {
+        throw RangeError(n);
+      }
+      if (encoding) {
+        return decode(bytes);
+      } else {
+        return Buffer.from(bytes);
+      }
+    },
+  };
+
+  /** @typedef { any } FileMethods too much trouble to get exactly right. */
+  const done = await replayXSnap(
+    opts,
+    ['/xs-test/'],
+    /** @type { FileMethods } */ (mockFS),
+  );
+
+  t.deepEqual(done, [
+    ['/xs-test/', 1, 'evaluate'],
+    ['/xs-test/', 2, 'reply'],
+  ]);
+});