improvise/harness/tests/shell.spec.js

// The browser shell: the three assertions improvise-hwpi could not write.
//
// That ticket fixed four things and said of itself that the fixes were "as
// unverified as the bugs", because three of them are one-assertion tests that
// needed a browser. These are those three, plus the watchdog — which was found
// by reading, and only because someone went looking.
//
// `web/app.js` and `web/worker.js` have never been rendered or run (gotcha 48).
// Everything below is the first execution of either.

import { test, expect } from "@playwright/test";

const status = (page) => page.locator("#status");

/// A peer that accepts the TCP connection and then refuses the WebSocket
/// upgrade — the harness's own static file server. Deterministic where a
/// closed port is not: a refused port's `close` timing is a property of the
/// host's network stack, and on the GitHub runner it did not arrive inside
/// the window, which made two expected-failure tests report "expected to
/// fail, but passed". The page cannot tell the two apart and neither can
/// `web/app.js`: in both cases the socket never opened.
const NO_WEBSOCKET_HERE = "127.0.0.1:8099";
const json = (v) => JSON.stringify(v);

/// `web/app.js`'s own HANDSHAKE_TIMEOUT_MS. Named here because two tests have
/// to outlast it, and a number that silently disagrees with the page would
/// make them pass for the wrong reason.
const HANDSHAKE_TIMEOUT_MS = 8_000;

/// Record every distinct #status the page shows, not just the last one.
/// Several of these are about a message being REPLACED, which the final value
/// cannot see.
function logStatus() {
  window.__statusLog = [];
  new MutationObserver(() => {
    const text = document.getElementById("status")?.textContent ?? "";
    if (text && window.__statusLog.at(-1) !== text) window.__statusLog.push(text);
  }).observe(document, { childList: true, subtree: true, characterData: true });
}

test.describe("the shell reports what its peer is doing", () => {
  // improvise-hwpi: `on_frame` returns a Result, which wasm-bindgen throws.
  // Thrown in a bare listener it reached the devtools console and nothing
  // else: the grid simply stopped updating with nothing on screen saying why.
  //
  // The fault is injected at the *peer*, not in the page — a mocked socket
  // that answers the handshake with a frame the client cannot fold — so this
  // exercises the real `on_frame` and the real catch.
  test("a frame the client cannot fold says so on screen", async ({ page }) => {
    await page.routeWebSocket("ws://127.0.0.1:8787/", (ws) => {
      ws.onMessage(() => ws.send("{\"NotAFrame\":{}}"));
    });
    await page.goto("/index.html?ws=127.0.0.1:8787");
    await expect(status(page)).toContainText("bad frame", { timeout: 15_000 });
  });

  // improvise-hwpi: without an `error` listener, a connection that never
  // opened left #status blank, and "the server is not running" read exactly
  // like "still connecting". Port 9 (discard) refuses immediately.
  //
  // The watchdog half is asserted on the same page load rather than in its
  // own test, because it needs the same 9-second wait. It holds for a reason
  // its own comment does not lead you to expect: `armWatchdog` runs only
  // after `transport.ready` resolves, which happens on `open` — so on a
  // socket that never opened the watchdog is never armed at all.
  test("a peer that cannot be reached is named, and the watchdog stays out of it", async ({
    page,
  }) => {
    await page.addInitScript(logStatus);
    await page.goto(`/index.html?ws=${NO_WEBSOCKET_HERE}`);
    await expect(status(page)).not.toHaveText("", { timeout: 15_000 });
    await page.waitForTimeout(HANDSHAKE_TIMEOUT_MS + 1_000);

    const log = await page.evaluate(() => window.__statusLog);
    expect(log.some((t) => t.includes("cannot reach")), `#status said: ${json(log)}`).toBe(
      true,
    );
    expect(log.some((t) => t.includes("sent no document")), `#status said: ${json(log)}`).toBe(
      false,
    );
  });

  // The other arm, which must keep working when the first one is fixed: a
  // socket that really did open and then dropped IS a disconnect, and saying
  // so is the whole reason `close` cannot simply be suppressed.
  test("a socket that opened and then dropped reads as disconnected", async ({ page }) => {
    // Closes on the page's first frame, so "it opened first" is true by
    // construction rather than by timer.
    await page.routeWebSocket("ws://127.0.0.1:8790/", (ws) => {
      ws.onMessage(() => ws.close());
    });
    await page.goto("/index.html?ws=127.0.0.1:8790");
    await expect(status(page)).toContainText("connected", { timeout: 15_000 });
    await expect(status(page)).toContainText("disconnected", { timeout: 15_000 });
  });

  // TWO BUGS THIS FILE DELIBERATELY DOES NOT ASSERT, and the reason is worth
  // more than the assertions would have been.
  //
  // 1. improvise-m18f: the socket's `close` listener is unconditional, so on
  //    a connection that never opened it overwrites "cannot reach …" with
  //    "disconnected".
  // 2. On a connection that DID open and then dropped, nothing disarms the
  //    handshake watchdog, so 8s later "disconnected" is replaced by
  //    "accepted the connection but sent no document" — which is false. Same
  //    class as the bug `setShellFailure` fixed (a watchdog must be disarmed,
  //    not pre-empted, gotcha 48), and it SURVIVES the m18f fix, because that
  //    fix is about which message `close` writes rather than about the timer
  //    still running.
  //
  // Both are real and both were measured, on macOS/Chromium:
  //
  //    never-opened:    connecting… -> cannot reach -> disconnected
  //    open-then-close: connecting… -> connected -> disconnected
  //                     -> accepted the connection but sent no document
  //
  // They were asserted here as `test.fail` expected-failures and **had to be
  // removed**: neither reproduces on the GitHub runner, where `close` does
  // not follow `error` within the window and the watchdog does not land. An
  // expected-failure is green when the bug is present and red when it is
  // absent, so one whose trigger is platform-dependent does not merely flake
  // — it **inverts CI's meaning**, and it failed the build twice saying
  // "expected to fail, but passed".
  //
  // What that leaves is a genuine limit rather than an oversight: the trigger
  // for both is the ORDER AND TIMING of `error`/`close`/timer on the host's
  // network stack, which is not a property of `web/app.js` and not something
  // a harness can pin. The four assertions that remain in this file are the
  // portable half — every message the page must write, and the one case where
  // the watchdog must stay silent. Filed for the orchestrator with the
  // recipe above; fixing them is `web/app.js`, which is not this unit's file.

  // The first-frame watchdog (gotcha 46), found by reading. The condition is
  // one no per-transport check would catch: the peer accepted the connection
  // and then said nothing. The mock accepts and never answers.
  test("a peer that accepts and then says nothing trips the watchdog", async ({
    page,
  }) => {
    await page.routeWebSocket("ws://127.0.0.1:8788/", () => {});
    await page.goto("/index.html?ws=127.0.0.1:8788");
    await expect(status(page)).toContainText("sent no document", { timeout: 20_000 });
  });
});

test.describe("the standalone deployment", () => {
  // No `?ws=`, so `web/app.js` spawns the in-page worker over OPFS: the whole
  // application, with no server process. If this draws a grid, then
  // worker.js, bindings.rs, OpfsFs, vfs::open and the seed fetch all ran.
  test("the page draws the demo document out of the worker", async ({ page }) => {
    await page.goto("/index.html");
    await expect(page.locator("#grid table")).toBeVisible({ timeout: 20_000 });
    await expect(status(page)).toContainText("in-page worker");
    // web/demo.improv: Product on row, _Measure on column (gotcha 46).
    await expect(page.locator("#grid th", { hasText: "Revenue" })).toBeVisible();
    await expect(page.locator("#grid td.editable").first()).toBeVisible();
  });

  // A commit round-trips through the host, and what lands on screen is the
  // HOST's value rather than the string that was typed. `20000` is typed and
  // `20,000` asserted: the number format is applied by the host and relayed
  // back, so a page that echoed the input would fail.
  //
  // This is deliberately NOT named for improvise-hwpi's third fix (the
  // `closed` re-entry guard in `edit`), which Copilot asked for on PR #24.
  // That guard is **not observable from the DOM**, measured rather than
  // assumed: with it removed, `close` still runs exactly twice, the frame is
  // still sent exactly once, `#grid` still takes exactly two childList
  // mutations, and the final text is identical. The only difference is one
  // extra *entry* into `repaint()` that produces no output — because the
  // second close is `close(false)`, so it does not re-send, and everything
  // else it does is idempotent. That is precisely why hwpi called it
  // "harmless before, but only by accident".
  //
  // Counting repaint() entries would need a test hook inside `web/app.js`,
  // which would be a page that knows it is being tested. So the name says
  // what is checked and this comment says what is not.
  test("committing an edit sends one frame and shows the host's value", async ({
    page,
  }) => {
    await page.goto("/index.html?doc=commit-test.improv");
    await expect(page.locator("#grid table")).toBeVisible({ timeout: 20_000 });

    const cell = page.locator("#grid td.editable").first();
    const before = await cell.textContent();
    await cell.click();
    const input = cell.locator("input");
    await expect(input).toBeFocused();
    await input.fill("20000");
    await input.press("Enter");

    await expect(cell).toHaveText("20,000", { timeout: 10_000 });
    expect(before).not.toBe("20,000");
    // The editor is gone and did not come back. This is the half of the
    // re-entry guard that IS observable: whatever `close` ran, no stray
    // <input> is left behind and the host's value is not overwritten by the
    // `original` text the second close writes before repainting.
    await expect(page.locator("#grid input")).toHaveCount(0);
  });
});