From c5e8d7d92222bc322e182faa7269dd6af2afbe12 Mon Sep 17 00:00:00 2001 From: Ed L Date: Wed, 23 Sep 2026 20:05:56 -0700 Subject: [PATCH 1/5] chore: update Cargo.nix --- Cargo.nix | 188 ++++++++++++++++++++++++++++-------------------------- 1 file changed, 97 insertions(+), 91 deletions(-) diff --git a/Cargo.nix b/Cargo.nix index daf3b0be..4478da93 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -2640,7 +2640,6 @@ rec { { name = "indexmap"; packageId = "indexmap"; - features = [ "serde" ]; } { name = "ratatui"; @@ -2654,6 +2653,7 @@ rec { { name = "serde_json"; packageId = "serde_json"; + features = [ "float_roundtrip" ]; } { name = "thiserror"; @@ -2673,6 +2673,10 @@ rec { name = "improvise-wasm-client"; packageId = "improvise-wasm-client"; } + { + name = "libc"; + packageId = "libc"; + } { name = "pest"; packageId = "pest"; @@ -2735,6 +2739,7 @@ rec { { name = "serde_json"; packageId = "serde_json"; + features = [ "float_roundtrip" ]; } { name = "unicode-width"; @@ -2776,7 +2781,6 @@ rec { { name = "indexmap"; packageId = "indexmap"; - features = [ "serde" ]; } { name = "serde"; @@ -2898,7 +2902,6 @@ rec { { name = "indexmap"; packageId = "indexmap"; - features = [ "serde" ]; } { name = "js-sys"; @@ -2921,6 +2924,7 @@ rec { { name = "serde_json"; packageId = "serde_json"; + features = [ "float_roundtrip" ]; } { name = "wasm-bindgen"; @@ -2975,6 +2979,7 @@ rec { name = "serde_json"; packageId = "serde_json"; optional = true; + features = [ "float_roundtrip" ]; } { name = "unicode-width"; @@ -3011,6 +3016,7 @@ rec { { name = "serde_json"; packageId = "serde_json"; + features = [ "float_roundtrip" ]; } { name = "wasm-bindgen"; @@ -3068,6 +3074,7 @@ rec { { name = "serde_json"; packageId = "serde_json"; + features = [ "float_roundtrip" ]; } { name = "wasm-bindgen"; @@ -3135,6 +3142,7 @@ rec { { name = "serde_json"; packageId = "serde_json"; + features = [ "float_roundtrip" ]; } { name = "tokio"; @@ -3171,27 +3179,6 @@ rec { packageId = "hashbrown 0.17.1"; usesDefaultFeatures = false; } - { - name = "serde"; - packageId = "serde"; - optional = true; - usesDefaultFeatures = false; - target = { target, features }: false; - } - { - name = "serde_core"; - packageId = "serde_core"; - optional = true; - usesDefaultFeatures = false; - } - ]; - devDependencies = [ - { - name = "serde"; - packageId = "serde"; - usesDefaultFeatures = false; - features = [ "derive" ]; - } ]; features = { "arbitrary" = [ "dep:arbitrary" ]; @@ -3202,7 +3189,7 @@ rec { "serde" = [ "dep:serde_core" "dep:serde" ]; "sval" = [ "dep:sval" ]; }; - resolvedDefaultFeatures = [ "default" "serde" "std" ]; + resolvedDefaultFeatures = [ "default" "std" ]; }; "indoc" = rec { crateName = "indoc"; @@ -5655,7 +5642,7 @@ rec { "preserve_order" = [ "indexmap" "std" ]; "std" = [ "memchr/std" "serde_core/std" ]; }; - resolvedDefaultFeatures = [ "default" "std" ]; + resolvedDefaultFeatures = [ "default" "float_roundtrip" "std" ]; }; "sha1" = rec { crateName = "sha1"; @@ -8871,7 +8858,15 @@ rec { ) crateConfigs; target = makeTarget pkgs.stdenv.hostPlatform; - build = mkBuiltByPackageIdByPkgs pkgs.buildPackages; + # Build-time dependency graph (for proc-macros and build + # dependencies). When not cross-compiling it equals the host + # graph, so reuse `self`; otherwise build it for + # `pkgs.buildPackages`. + build = + if pkgs.stdenv.buildPlatform.config == pkgs.stdenv.hostPlatform.config then + self + else + mkBuiltByPackageIdByPkgs pkgs.buildPackages; }; in self; @@ -8887,38 +8882,35 @@ rec { devDependencies = lib.optionals (runTests && packageId == rootPackageId) ( crateConfig'.devDependencies or [ ] ); - dependencies = dependencyDerivations { + # Enabled (platform- and feature-filtered) dependency lists, reused + # for both derivation wiring and the crate renames below. + enabledDependencies = filterEnabledDependencies { inherit features; inherit (self) target; - buildByPackageId = - depPackageId: - # proc_macro crates must be compiled for the build architecture - if crateConfigs.${depPackageId}.procMacro or false then - self.build.crates.${depPackageId} - else - self.crates.${depPackageId}; dependencies = (crateConfig.dependencies or [ ]) ++ devDependencies; }; - buildDependencies = dependencyDerivations { + enabledBuildDependencies = filterEnabledDependencies { inherit features; inherit (self.build) target; - buildByPackageId = depPackageId: self.build.crates.${depPackageId}; dependencies = crateConfig.buildDependencies or [ ]; }; + dependencies = map + ( + dependency: + # proc_macro crates must be compiled for the build architecture + if crateConfigs.${dependency.packageId}.procMacro or false then + self.build.crates.${dependency.packageId} + else + self.crates.${dependency.packageId} + ) + enabledDependencies; + buildDependencies = map + (dependency: self.build.crates.${dependency.packageId}) + enabledBuildDependencies; + # Order (build dependencies, then normal dependencies) feeds the + # crateRenames grouping below. dependenciesWithRenames = - let - buildDeps = filterEnabledDependencies { - inherit features; - inherit (self) target; - dependencies = crateConfig.dependencies or [ ] ++ devDependencies; - }; - hostDeps = filterEnabledDependencies { - inherit features; - inherit (self.build) target; - dependencies = crateConfig.buildDependencies or [ ]; - }; - in - lib.filter (d: d ? "rename") (hostDeps ++ buildDeps); + lib.filter (d: d ? "rename") (enabledBuildDependencies ++ enabledDependencies); # Crate renames have the form: # # { @@ -9090,6 +9082,17 @@ rec { corresponding feature sets are merged. Features in rust are additive. */ mergePackageFeatures = + args: builtins.mapAttrs (_packageId: builtins.attrNames) (mergePackageFeaturesImpl args); + + /* + Core of the feature-resolution fixpoint. The cache (`featuresByPackageId`) + maps each packageId to a feature *set* (an attrset `feature -> 1`) rather + than a sorted list, so the fold merges with `//` and detects convergence + with attrset equality instead of re-concatenating and re-sorting the + accumulated feature list on every step. `mergePackageFeatures` projects the + result back to canonical sorted lists. + */ + mergePackageFeaturesImpl = { crateConfigs ? crates , packageId , rootPackageId ? packageId @@ -9138,14 +9141,15 @@ rec { cache: { packageId, features }: let - cacheFeatures = cache.${packageId} or [ ]; - combinedFeatures = sortedUnique (cacheFeatures ++ features); + cacheFeatures = cache.${packageId} or { }; + # `features` is the (small) incoming list; merge it into the set. + combinedFeatures = cacheFeatures // listToSet features; in - if cache ? ${packageId} && cache.${packageId} == combinedFeatures then + if cache ? ${packageId} && cacheFeatures == combinedFeatures then cache else - mergePackageFeatures { - features = combinedFeatures; + mergePackageFeaturesImpl { + features = builtins.attrNames combinedFeatures; featuresByPackageId = cache; inherit crateConfigs @@ -9158,8 +9162,8 @@ rec { ); cacheWithSelf = let - cacheFeatures = featuresByPackageId.${packageId} or [ ]; - combinedFeatures = sortedUnique (cacheFeatures ++ enabledFeatures); + cacheFeatures = featuresByPackageId.${packageId} or { }; + combinedFeatures = cacheFeatures // listToSet enabledFeatures; in featuresByPackageId // { @@ -9186,27 +9190,31 @@ rec { assert (builtins.isList features); assert (builtins.isAttrs target); + let + # Identical for every dep in this call; build the predicate arg once. + targetArgs = { inherit features target; }; + in lib.filter ( dep: - let - targetFunc = dep.target or (features: true); - in - targetFunc { inherit features target; } + (dep.target or (features: true)) targetArgs && (!(dep.optional or false) || builtins.any (doesFeatureEnableDependency dep) features) ) dependencies; # Returns whether the given feature should enable the given dependency. doesFeatureEnableDependency = - dependency: feature: + dependency: + # Callers partially apply this once per dependency, then test every feature, + # so hoist the dep-invariant strings out of the per-feature comparison. let name = dependency.rename or dependency.name; + depName = "dep:" + name; prefix = "${name}/"; len = builtins.stringLength prefix; - startsWithPrefix = builtins.substring 0 len feature == prefix; in - feature == name || feature == "dep:" + name || startsWithPrefix; + feature: + feature == name || feature == depName || builtins.substring 0 len feature == prefix; /* Returns the expanded features for the given inputFeatures by applying the @@ -9220,29 +9228,24 @@ rec { assert (builtins.isAttrs featureMap); assert (builtins.isList inputFeatures); let + # Transitive closure of `inputFeatures` under `featureMap`. `seen` + # tracks already-expanded features so each is visited at most once; + # this also breaks feature cycles (issue #209). expandFeaturesNoCycle = - oldSeen: inputFeatures: - if inputFeatures != [ ] then - let - # The feature we're currently expanding. - feature = builtins.head inputFeatures; - # All the features we've seen/expanded so far, including the one - # we're currently processing. - seen = oldSeen // { - ${feature} = 1; - }; - # Expand the feature but be careful to not re-introduce a feature - # that we've already seen: this can easily cause a cycle, see issue - # #209. - enables = builtins.filter (f: !(seen ? "${f}")) (featureMap."${feature}" or [ ]); - in - [ feature ] ++ (expandFeaturesNoCycle seen (builtins.tail inputFeatures ++ enables)) - # No more features left, nothing to expand to. - else - [ ]; - outFeatures = expandFeaturesNoCycle { } inputFeatures; + seen: features: + builtins.foldl' + ( + acc: feature: + if acc ? ${feature} then + acc + else + expandFeaturesNoCycle (acc // { ${feature} = 1; }) (featureMap.${feature} or [ ]) + ) + seen + features; + seen = expandFeaturesNoCycle { } inputFeatures; in - sortedUnique outFeatures; + sortedUnique (builtins.attrNames seen); /* This function adds optional dependencies as features if they are enabled @@ -9301,16 +9304,19 @@ rec { in defaultOrNil ++ explicitFeatures ++ additionalDependencyFeatures; + # Builds a feature set (attrset `feature -> 1`) from a list of feature names. + listToSet = + features: + builtins.listToAttrs (map (feature: { name = feature; value = 1; }) features); + # Sorts and removes duplicates from a list of strings. sortedUnique = features: assert (builtins.isList features); - assert (builtins.all builtins.isString features); - let - outFeaturesSet = lib.foldl (set: feature: set // { "${feature}" = 1; }) { } features; - outFeaturesUnique = builtins.attrNames outFeaturesSet; - in - builtins.sort (a: b: a < b) outFeaturesUnique; + # attrNames returns keys in ascending string order, so the result is + # sorted and deduplicated. The feature-merge fixpoint relies on this + # canonical order to detect convergence. + builtins.attrNames (listToSet features); deprecationWarning = message: value: From af44653f928fc53b21019abe348b803c2f9f5fe4 Mon Sep 17 00:00:00 2001 From: Ed L Date: Wed, 23 Sep 2026 20:27:52 -0700 Subject: [PATCH 2/5] chore: setup nixos runners --- .github/actionlint.yaml | 6 ++ .github/workflows/ci.yml | 132 ++++++++++++++++---------------------- flake.nix | 58 +++++++++++------ harness/package-lock.json | 48 +++++--------- harness/package.json | 2 +- scripts/checkers-test.sh | 12 +++- 6 files changed, 126 insertions(+), 132 deletions(-) create mode 100644 .github/actionlint.yaml diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml new file mode 100644 index 00000000..bce99740 --- /dev/null +++ b/.github/actionlint.yaml @@ -0,0 +1,6 @@ +# Labels the self-hosted NixOS microVM runners register with. +self-hosted-runner: + labels: + - nixos + - microvm + - audited diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4646ac2d..9b1cd222 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,6 +12,15 @@ on: env: CARGO_TERM_COLOR: always +# Every job runs on the self-hosted NixOS microVM runners, and every `run:` +# step inside the flake's `ci` dev shell. Generic-Linux downloads (rustup, +# setup-node's node, Playwright's browsers) cannot run on NixOS, so the +# toolchain comes from flake.nix, the same one `nix develop` gives locally. +# Each step re-enters the shell; after the first, the eval cache makes it cheap. +defaults: + run: + shell: nix develop .#ci --command bash --noprofile --norc -eo pipefail {0} + jobs: # --------------------------------------------------------------------------- # The one place wasm-pack runs (improvise CI dedup). @@ -35,7 +44,7 @@ jobs: # would serialise a 109s job behind a 126s one for no artifact at all. build: name: Build all bundles - runs-on: ubuntu-latest + runs-on: [self-hosted, nixos, microvm] steps: - uses: actions/checkout@v7 # First, because it needs nothing and takes a second: the gate that stops @@ -44,9 +53,10 @@ jobs: # below, so it fails under this job's name rather than another's. - name: The stamp gate refuses what it claims to run: ./scripts/checkers.sh assemble-stamp - - uses: dtolnay/rust-toolchain@stable - with: - targets: wasm32-unknown-unknown + # rust-cache is a JS action: it runs outside the dev shell, and needs + # cargo and rustc on the job's PATH to compute its key. + - name: Put the dev shell's toolchain on PATH for actions + run: dirname "$(command -v cargo)" >> "$GITHUB_PATH" # Two workspaces, because `harness/opfs-contract` carries an empty # `[workspace]` table and so has its own target dir, which the default # single-workspace configuration never cached (it recompiled its whole @@ -56,34 +66,21 @@ jobs: workspaces: | . -> target harness/opfs-contract -> target - # One home for the two pins, so the release capture - # (scripts/build-web-release.sh, improvise-oujb) provisions the same - # versions rather than a second copy free to go stale. The grep keeps a - # comment or a blank line out of $GITHUB_ENV, which takes NAME=VALUE only. - - name: Pinned web toolchain versions - run: grep -E '^[A-Z_]+=' scripts/web-toolchain-versions.env >> "$GITHUB_ENV" - # Pinned release binary rather than `cargo install` (a three-minute - # source build) or a third-party action. wasm-pack runs its own bundled - # `wasm-opt`, which is why improvise-wasm-client's Cargo.toml names - # `--enable-bulk-memory`: without it the optimizer rejects the - # `memory.copy` LLVM emits regardless. - # v0.15.0 is what `nix develop` supplies, and the bundle was built and - # measured against it locally (346 KB, 120 KB gzipped). Pinning the same - # version is what makes that local run evidence about this one. - - name: Install wasm-pack + # scripts/web-toolchain-versions.env stays the one home for the pins, so + # the release capture (scripts/build-web-release.sh, improvise-oujb) + # provisions the same versions. CI no longer installs them — the dev + # shell supplies wasm-pack and node — so this checks the shell against + # the pins instead: a nixpkgs bump that moves either fails here, by name. + # wasm-pack runs its own bundled `wasm-opt`, which is why + # improvise-wasm-client's Cargo.toml names `--enable-bulk-memory`. + - name: The dev shell supplies the pinned web toolchain run: | - set -eu - url="https://github.com/rustwasm/wasm-pack/releases/download/${WASM_PACK_VERSION}/wasm-pack-${WASM_PACK_VERSION}-x86_64-unknown-linux-musl.tar.gz" - curl -sSfL "$url" | tar -xz --strip-components=1 -C /usr/local/bin \ - "wasm-pack-${WASM_PACK_VERSION}-x86_64-unknown-linux-musl/wasm-pack" + . scripts/web-toolchain-versions.env + test "$(wasm-pack --version)" = "wasm-pack ${WASM_PACK_VERSION#v}" + test "$(node -p 'process.versions.node.split(".")[0]')" = "$NODE_VERSION" wasm-pack --version - - # Both assemblies below generate the prose site with node. The runner - # image ships one, but pinning it is what makes a local run evidence. - - uses: actions/setup-node@v4 - with: - node-version: ${{ env.NODE_VERSION }} + node --version # The same script a developer runs locally, so there is one answer to # "how is the bundle built" rather than two that can drift. It builds all @@ -133,12 +130,11 @@ jobs: test: name: Test and lint (host) - runs-on: ubuntu-latest + runs-on: [self-hosted, nixos, microvm] steps: - uses: actions/checkout@v7 - - uses: dtolnay/rust-toolchain@stable - with: - components: clippy, rustfmt + - name: Put the dev shell's toolchain on PATH for actions + run: dirname "$(command -v cargo)" >> "$GITHUB_PATH" - uses: Swatinem/rust-cache@v2 # NOT `cargo fmt --all`, which reaches only ONE of the two workspace # roots. Exit 2 here is a usage error, never a verdict (improvise-b4ks). @@ -187,12 +183,11 @@ jobs: wasm: name: wasm32 check (browser bundle crates) - runs-on: ubuntu-latest + runs-on: [self-hosted, nixos, microvm] steps: - uses: actions/checkout@v7 - - uses: dtolnay/rust-toolchain@stable - with: - targets: wasm32-unknown-unknown + - name: Put the dev shell's toolchain on PATH for actions + run: dirname "$(command -v cargo)" >> "$GITHUB_PATH" - uses: Swatinem/rust-cache@v2 # Only the crates that go into the browser bundle. The root `improvise` # crate is the terminal binary and depends on crossterm, which does not @@ -229,15 +224,12 @@ jobs: # here is a decision `web/shell.js` or `web/serial.js` got wrong, and # reporting that as "Browser harness" would put it behind an 80 MB browser # download and two wasm-pack builds. It needs neither — `node --test` is in - # the runtime, and this job installs nothing at all. + # the runtime, which the dev shell supplies; this job installs nothing. js: name: Browser JS unit tests (node --test) - runs-on: ubuntu-latest + runs-on: [self-hosted, nixos, microvm] steps: - uses: actions/checkout@v7 - - uses: actions/setup-node@v4 - with: - node-version: 22 # The glob is the SCRIPT's, not the shell's and not node's. Under bash an # unmatched glob survives as a literal, node globs it, matches nothing and # exits 0 having run no test -- so this step gated nothing the moment @@ -252,23 +244,23 @@ jobs: kb: name: Knowledge base consistency - runs-on: ubuntu-latest + runs-on: [self-hosted, nixos, microvm] steps: - uses: actions/checkout@v7 - - uses: cachix/install-nix-action@v31 # kb.pl is executable knowledge, so it gets checked like code: rules # must reject what they should, domains must close, and functional keys # must not carry conflicting values. See kb_report/0 in the file. + # trealla comes from the dev shell, as it does locally. - name: kb_consistent run: | - nix shell nixpkgs#trealla --command tpl prolog-sessions/kb.pl \ + tpl prolog-sessions/kb.pl \ -g "(kb_consistent -> halt(0) ; (kb_report, halt(1)))" # scripts/kb is the only thing that writes kb.pl, so its refusals are as # load-bearing as kb_consistent itself. Without this the replacement # writer could regress green -- the review that found the directive # bypass also found that nothing ran this suite. - name: The kb tool refuses what it claims to - run: nix shell nixpkgs#trealla --command ./scripts/checkers.sh kb + run: ./scripts/checkers.sh kb # Every suite goes through scripts/checkers.sh, which pins a bash >= 5 # and scrubs the environment, so the shell and the variables a suite # passes under here are the ones it passes under locally (improvise-f9qf). @@ -301,13 +293,11 @@ jobs: # failed the *Browser harness* job. `-p opfs-contract` scopes it to 1 file. harness-lint: name: Lint the harness crate - runs-on: ubuntu-latest + runs-on: [self-hosted, nixos, microvm] steps: - uses: actions/checkout@v7 - - uses: dtolnay/rust-toolchain@stable - with: - targets: wasm32-unknown-unknown - components: clippy, rustfmt + - name: Put the dev shell's toolchain on PATH for actions + run: dirname "$(command -v cargo)" >> "$GITHUB_PATH" - uses: Swatinem/rust-cache@v2 with: workspaces: harness/opfs-contract -> target @@ -335,37 +325,25 @@ jobs: browser: name: Browser harness (OPFS contract + shell) needs: build - runs-on: ubuntu-latest + runs-on: [self-hosted, nixos, microvm] steps: - uses: actions/checkout@v7 - - uses: actions/setup-node@v4 - with: - node-version: 22 - name: Install the harness working-directory: harness run: npm ci - # Keyed on the resolved @playwright/test version, so a version bump is a - # miss and anything else is a hit. The download was 50s on run - # 35535225637 and is pure I/O — nothing about it depends on this commit. - - name: Playwright version - id: pw + # No browser install step: the dev shell points PLAYWRIGHT_BROWSERS_PATH + # at nixpkgs' Chromium, Firefox and WebKit, since Playwright's own + # downloads cannot run on NixOS. All three, because improvise-xf1's + # acceptance criterion names Chrome, Firefox and Safari and + # `createSyncAccessHandle` is implemented separately by each engine. + # WebKit's OPFS is absent today; the spec skips on a runtime probe. + # @playwright/test must equal nixpkgs' playwright-driver version, or it + # looks for browser builds that are not there; this says which. + - name: Playwright matches the dev shell's browsers working-directory: harness - run: echo "v=$(node -p "require('@playwright/test/package.json').version")" >> "$GITHUB_OUTPUT" - - uses: actions/cache@v4 - id: pw-cache - with: - path: ~/.cache/ms-playwright - key: playwright-${{ runner.os }}-${{ steps.pw.outputs.v }} - # All three, because improvise-xf1's acceptance criterion names Chrome, - # Firefox and Safari and `createSyncAccessHandle` is implemented - # separately by each engine. WebKit is installed even though its OPFS is - # absent today: the spec skips on a runtime probe, so the day that build - # gains OPFS the check starts running without a CI change. - # `--with-deps` installs apt packages OUTSIDE the cached directory, so it - # runs on a cache hit too; only the browser download is skipped. - - name: Install browsers - working-directory: harness - run: npx playwright install --with-deps chromium firefox webkit + run: | + npx playwright --version + ls "$PLAYWRIGHT_BROWSERS_PATH" # The tree `build` assembled, not one this job rebuilds. Same bytes as # the Pages artifact, modulo the cache-bust token `web/assemble.sh` # stamps — the harness tree is stamped `harness` and the site the commit @@ -434,7 +412,7 @@ jobs: name: Deploy to GitHub Pages needs: [test, wasm, js, browser, harness-lint, kb, build] if: github.event_name == 'push' && github.ref == 'refs/heads/main' - runs-on: ubuntu-latest + runs-on: [self-hosted, nixos, microvm] permissions: contents: read pages: write diff --git a/flake.nix b/flake.nix index 0db23e81..5f323551 100644 --- a/flake.nix +++ b/flake.nix @@ -36,12 +36,47 @@ cargoNix = import ./Cargo.nix { inherit nixpkgs pkgs; }; + # Everything CI runs, and nothing else: `devShells.ci` is exactly this, + # and the default shell is this plus editor and release tooling, so a + # local `nix develop` runs the toolchain the checks run. + ciInputs = [ + rustToolchain + pkgs.pkg-config + + # `scripts/kb` and the CI knowledge-base check both run trealla. It + # was reached through `nix shell nixpkgs#trealla` per invocation, + # which cost more than the work: the tool's first real use took past + # two minutes for a write that takes half a second on PATH. + pkgs.trealla + + # `web/build.sh` — and so the Pages job's local dry run — needs this. + # The `wasm` CI job only runs `cargo check`, which does not, so the + # shell was missing the one tool that turns a passing check into a + # loadable page (improvise-d31). Must match WASM_PACK_VERSION in + # scripts/web-toolchain-versions.env; CI checks that it does. + pkgs.wasm-pack + + # NODE_VERSION's major, same check. The js and browser jobs ran 22 + # while `build` ran the pinned 24; one shell means one node. + pkgs.nodejs_24 + pkgs.python3 + pkgs.jq + ]; + + # Playwright's downloaded browsers are generic-Linux builds that cannot + # start on NixOS, so nixpkgs' build is used. harness/package.json pins + # @playwright/test to this playwright-driver's version: they must match. + playwrightEnv = { + PLAYWRIGHT_BROWSERS_PATH = "${pkgs.playwright-driver.browsers}"; + PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD = "1"; + PLAYWRIGHT_SKIP_VALIDATE_HOST_REQUIREMENTS = "true"; + RUST_BACKTRACE = "1"; + }; in { inherit cargoNix; - devShells.default = pkgs.mkShell { - nativeBuildInputs = [ - rustToolchain - pkgs.pkg-config + devShells.ci = pkgs.mkShell (playwrightEnv // {nativeBuildInputs = ciInputs;}); + devShells.default = pkgs.mkShell (playwrightEnv // { + nativeBuildInputs = ciInputs ++ [ pkgs.rust-analyzer crate2nix.packages.${system}.default pkgs.cargo-expand @@ -50,18 +85,6 @@ pkgs.universal-ctags beads.packages.${system}.default - # `scripts/kb` and the CI knowledge-base check both run trealla. It - # was reached through `nix shell nixpkgs#trealla` per invocation, - # which cost more than the work: the tool's first real use took past - # two minutes for a write that takes half a second on PATH. - pkgs.trealla - - # `web/build.sh` — and so the Pages job's local dry run — needs this. - # The `wasm` CI job only runs `cargo check`, which does not, so the - # shell was missing the one tool that turns a passing check into a - # loadable page (improvise-d31). - pkgs.wasm-pack - # Demo recording and release tooling pkgs.asciinema pkgs.vhs @@ -69,8 +92,7 @@ # nixpkgs cargo-dist installs as "dist"; alias so `cargo dist` works (pkgs.writeShellScriptBin "cargo-dist" ''exec ${pkgs.cargo-dist}/bin/dist "$@"'') ]; - RUST_BACKTRACE = "1"; - }; + }); packages = { improvise = cargoNix.workspaceMembers.improvise.build; diff --git a/harness/package-lock.json b/harness/package-lock.json index 2904809a..2862707d 100644 --- a/harness/package-lock.json +++ b/harness/package-lock.json @@ -6,70 +6,52 @@ "": { "name": "improvise-harness", "devDependencies": { - "@playwright/test": "1.56.1" + "@playwright/test": "1.63.0" } }, "node_modules/@playwright/test": { - "version": "1.56.1", - "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.56.1.tgz", - "integrity": "sha512-vSMYtL/zOcFpvJCW71Q/OEGQb7KYBPAdKh35WNSkaZA75JlAO8ED8UN6GUNTm3drWomcbcqRPFqQbLae8yBTdg==", + "version": "1.63.0", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.63.0.tgz", + "integrity": "sha512-oxMK4vllB9RK5NQ2l1pq1IfOf2AvnEuj/vYGDj0H2nMtmtZpKtCwt/l00GEO6xjGfpBNAvjovvYdCm50dRQkpQ==", "dev": true, "license": "Apache-2.0", "dependencies": { - "playwright": "1.56.1" + "playwright": "1.63.0" }, "bin": { "playwright": "cli.js" }, "engines": { - "node": ">=18" - } - }, - "node_modules/fsevents": { - "version": "2.3.2", - "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", - "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", - "dev": true, - "hasInstallScript": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + "node": ">=20" } }, "node_modules/playwright": { - "version": "1.56.1", - "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.56.1.tgz", - "integrity": "sha512-aFi5B0WovBHTEvpM3DzXTUaeN6eN0qWnTkKx4NQaH4Wvcmc153PdaY2UBdSYKaGYw+UyWXSVyxDUg5DoPEttjw==", + "version": "1.63.0", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.63.0.tgz", + "integrity": "sha512-+7ziBLidS4NaNCdt57SUDT+wYmmd5fmiQejUic/kb+YsYSCPyOOE9sebzMjNmQrsnNpDJqd4WHvV/8lfKfUDUg==", "dev": true, "license": "Apache-2.0", "dependencies": { - "playwright-core": "1.56.1" + "playwright-core": "1.63.0" }, "bin": { "playwright": "cli.js" }, "engines": { - "node": ">=18" - }, - "optionalDependencies": { - "fsevents": "2.3.2" + "node": ">=20" } }, "node_modules/playwright-core": { - "version": "1.56.1", - "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.56.1.tgz", - "integrity": "sha512-hutraynyn31F+Bifme+Ps9Vq59hKuUCz7H1kDOcBs+2oGguKkWTU50bBWrtz34OUWmIwpBTWDxaRPXrIXkgvmQ==", + "version": "1.63.0", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.63.0.tgz", + "integrity": "sha512-rYCsBF/M5HjUch52bbtVONEFjv6Xu8sm8h72dNlR5bzIE1fvC/bxgspzkjSfU+MweEMmPM8KJebG6nnyxo5mCg==", "dev": true, "license": "Apache-2.0", "bin": { "playwright-core": "cli.js" }, "engines": { - "node": ">=18" + "node": ">=20" } } } diff --git a/harness/package.json b/harness/package.json index 2923ee0f..bfc4fc72 100644 --- a/harness/package.json +++ b/harness/package.json @@ -8,6 +8,6 @@ "test": "playwright test" }, "devDependencies": { - "@playwright/test": "1.56.1" + "@playwright/test": "1.63.0" } } diff --git a/scripts/checkers-test.sh b/scripts/checkers-test.sh index 01995e07..ba7ab935 100755 --- a/scripts/checkers-test.sh +++ b/scripts/checkers-test.sh @@ -133,9 +133,15 @@ EOF chmod +x "$work/fakebin/bash" # The runner is invoked through a REAL bash, so the fake one below is only # what its own `command -v bash` probe finds -- not its interpreter. -# /usr/bin:/bin carries the utilities the runner itself needs and carries no -# nix, so the fake bash is the only candidate and the nix fallback is absent. -env PATH="$work/fakebin:/usr/bin:/bin" HOME="$HOME" \ +# utilbin carries only the utilities the runner itself needs, and no nix, so +# the fake bash is the only candidate and the nix fallback is absent. Linked +# one by one rather than taking /usr/bin:/bin, which on NixOS hold only `env` +# and `sh` -- the runner died on `dirname` there, exit 1 instead of 3. +mkdir -p "$work/utilbin" +for tool in dirname; do + ln -sf "$(command -v "$tool")" "$work/utilbin/$tool" +done +env PATH="$work/fakebin:$work/utilbin" HOME="$HOME" \ "${BASH:-bash}" "$d/checkers.sh" any > "$work/out" 2>&1 got=$? [ "$got" = 3 ] && ok "an old bash with no nix is exit 3" \ From 171f982de3ab84cde43accae6924ce54c4089164 Mon Sep 17 00:00:00 2001 From: Ed L Date: Wed, 23 Sep 2026 21:25:11 -0700 Subject: [PATCH 3/5] chore: fix wasm builds --- .github/workflows/ci.yml | 48 ++++++++++++++++++++-------------------- flake.nix | 21 ++++++++++++++++++ 2 files changed, 45 insertions(+), 24 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9b1cd222..7b2d49df 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -81,6 +81,14 @@ jobs: test "$(node -p 'process.versions.node.split(".")[0]')" = "$NODE_VERSION" wasm-pack --version node --version + # wasm-pack uses the shell's wasm-bindgen only at Cargo.lock's exact + # version; otherwise it cargo-installs one (71s) — so a lock bump + # must move flake.nix's wasmBindgenCli too, and this says so. + lock="$(awk '/^name = "wasm-bindgen"$/ {getline; gsub(/[^0-9.]/, ""); print}' Cargo.lock)" + test "$(wasm-bindgen --version)" = "wasm-bindgen $lock" || { + echo "flake.nix wasmBindgenCli is $(wasm-bindgen --version); Cargo.lock wants $lock" >&2 + exit 1 + } # The same script a developer runs locally, so there is one answer to # "how is the bundle built" rather than two that can drift. It builds all @@ -157,29 +165,21 @@ jobs: - name: Every check job gates the deploy run: python3 scripts/check-ci-graph.py - # Too slow on GitHub Actions - # - # nix-test: - # name: Nix Test - # runs-on: ubuntu-latest - # steps: - # - name: Free up disk space - # run: | - # sudo rm -rf /usr/share/dotnet \ - # /usr/local/lib/android - # sudo df -h - - # - uses: actions/checkout@v7 - # - name: Install Nix - # uses: DeterminateSystems/nix-installer-action@v14 - - # - name: Magic Nix Cache - # uses: DeterminateSystems/magic-nix-cache-action@v8 - - # - name: Check Nix health - # run: nix doctor - - # - run: nix build + # --------------------------------------------------------------------------- + # The crate2nix build (Cargo.nix), which `cargo` never exercises. It was off + # because GitHub-hosted runners start with an empty store and rebuilt every + # dependency crate each run. The NixOS runners substitute from titan's + # harmonia first, and titan builds gh/main's `#improvise` hourly, so a PR that + # leaves Cargo.lock alone rebuilds only the workspace crates it changed. + nix-build: + name: Nix build (crate2nix) + runs-on: [self-hosted, nixos, microvm] + steps: + - uses: actions/checkout@v7 + # Plain bash, not the dev shell: this step needs nix and nothing else. + - name: nix build .#improvise + shell: bash + run: nix build --print-build-logs .#improvise wasm: name: wasm32 check (browser bundle crates) @@ -410,7 +410,7 @@ jobs: # exists to stop. pages-deploy: name: Deploy to GitHub Pages - needs: [test, wasm, js, browser, harness-lint, kb, build] + needs: [test, wasm, js, browser, harness-lint, kb, build, nix-build] if: github.event_name == 'push' && github.ref == 'refs/heads/main' runs-on: [self-hosted, nixos, microvm] permissions: diff --git a/flake.nix b/flake.nix index 5f323551..8e686b9b 100644 --- a/flake.nix +++ b/flake.nix @@ -36,6 +36,25 @@ cargoNix = import ./Cargo.nix { inherit nixpkgs pkgs; }; + # wasm-pack downloads wasm-bindgen and binaryen from GitHub releases + # unless they are on PATH, and its downloader ignores SSL_CERT_FILE, so + # behind the runners' TLS-auditing proxy those downloads fail (binaryen + # with no fallback). On PATH, it downloads nothing. wasm-pack only takes + # a PATH wasm-bindgen whose version equals Cargo.lock's `wasm-bindgen`, + # so this pin moves with the lock; CI checks that it does. + wasmBindgenCli = pkgs.buildWasmBindgenCli rec { + src = pkgs.fetchCrate { + pname = "wasm-bindgen-cli"; + version = "0.2.128"; + hash = "sha256-a7lcXJnnZkYReja+iUO7NqqrWyv3toxnUgQb8s4IS5s="; + }; + cargoDeps = pkgs.rustPlatform.fetchCargoVendor { + inherit src; + inherit (src) pname version; + hash = "sha256-R1Tas33Ursy8kqsxguAkG0ZhNed2n5uFTAhw1l2qlLY="; + }; + }; + # Everything CI runs, and nothing else: `devShells.ci` is exactly this, # and the default shell is this plus editor and release tooling, so a # local `nix develop` runs the toolchain the checks run. @@ -55,6 +74,8 @@ # loadable page (improvise-d31). Must match WASM_PACK_VERSION in # scripts/web-toolchain-versions.env; CI checks that it does. pkgs.wasm-pack + wasmBindgenCli + pkgs.binaryen # NODE_VERSION's major, same check. The js and browser jobs ran 22 # while `build` ran the pinned 24; one shell means one node. From 91fee245b8e55dd2e4d72c160be62bdeca3d52b7 Mon Sep 17 00:00:00 2001 From: Ed L Date: Wed, 23 Sep 2026 21:44:36 -0700 Subject: [PATCH 4/5] ci: build with Nix (crane) instead of cargo in the dev shell Every CI build is now a derivation in nix/ci.nix, so the self-hosted runners substitute it from titan's harmonia cache instead of rebuilding: - checks.clippy, checks.test (with the --check-evidence gate, which needs the compiled tests), checks.wasm-check (per crate, no feature unification), checks.harness-lint (opfs-contract fmt + clippy) - packages.web: harness/build.sh unchanged -> the three wasm bundles and harness/serve; packages.site re-assembles with the commit as the cache-bust token - packages.ci-deps: the crane dependency builds, for titan's cache timer to keep rooted Dependencies are separate buildDepsOnly derivations keyed on the lock, so a PR that leaves Cargo.lock alone rebuilds only the workspace. The script-only checks (node, kb, checkers, comment/CI-graph gates, fmt) stay dev-shell steps: they compile nothing, and their enforced `#!/usr/bin/env bash` cannot run in the build sandbox. ci.yml's build, test, wasm and harness-lint jobs now run `nix build` on these; rust-cache and the PATH workaround for it are gone. Job names and the deploy graph are unchanged. Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/workflows/ci.yml | 122 ++++++++++---------------- flake.lock | 16 ++++ flake.nix | 28 +++++- nix/ci.nix | 184 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 270 insertions(+), 80 deletions(-) create mode 100644 nix/ci.nix diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7b2d49df..f8f7f9ea 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -53,20 +53,6 @@ jobs: # below, so it fails under this job's name rather than another's. - name: The stamp gate refuses what it claims to run: ./scripts/checkers.sh assemble-stamp - # rust-cache is a JS action: it runs outside the dev shell, and needs - # cargo and rustc on the job's PATH to compute its key. - - name: Put the dev shell's toolchain on PATH for actions - run: dirname "$(command -v cargo)" >> "$GITHUB_PATH" - # Two workspaces, because `harness/opfs-contract` carries an empty - # `[workspace]` table and so has its own target dir, which the default - # single-workspace configuration never cached (it recompiled its whole - # dependency tree every run). - - uses: Swatinem/rust-cache@v2 - with: - workspaces: | - . -> target - harness/opfs-contract -> target - # scripts/web-toolchain-versions.env stays the one home for the pins, so # the release capture (scripts/build-web-release.sh, improvise-oujb) # provisions the same versions. CI no longer installs them — the dev @@ -90,20 +76,29 @@ jobs: exit 1 } - # The same script a developer runs locally, so there is one answer to - # "how is the bundle built" rather than two that can drift. It builds all - # three wasm bundles (client, worker, opfs-contract) and assembles - # `harness/serve` — the deployed arrangement plus /opfs/. - - name: Build the bundles and the harness tree - run: ./harness/build.sh - - # Re-assembles from the pkg dirs the step above already produced — no - # wasm-pack runs here. A second assembly rather than a copy of - # `harness/serve` because the cache-bust token differs: the harness tree - # is stamped `harness` and the published site the commit SHA, which is - # what makes a returning visitor pick up a fresh bundle (improvise-9x6). - - name: Assemble the site - run: ./web/assemble.sh site "$GITHUB_SHA" + # Built by Nix (packages.web and packages.site in nix/ci.nix), which runs + # the same harness/build.sh and web/assemble.sh a developer runs, so there + # is still one answer to "how is the bundle built". `web` is the three + # wasm bundles plus `harness/serve`; `site` re-assembles from web's pkg + # dirs with the commit SHA as the cache-bust token (improvise-9x6), where + # the harness tree is stamped `harness`. Dependencies are their own + # derivations, substituted from titan's cache when Cargo.lock is unchanged. + - name: Build the bundles, the harness tree and the site + shell: bash + run: | + nix build --print-build-logs --out-link result-web .#web + nix build --print-build-logs --out-link result-site .#site + # The upload steps below read these paths. Copied rather than linked: + # an artifact of a symlink into /nix/store is a dangling link elsewhere. + - name: Lay the outputs out where the uploads read them + shell: bash + run: | + rm -rf harness/serve web/pkg web/worker-pkg site + cp -rL result-web/harness-serve harness/serve + cp -rL result-web/pkg web/pkg + cp -rL result-web/worker-pkg web/worker-pkg + cp -rL result-site site + chmod -R u+w harness/serve web/pkg web/worker-pkg site - run: find site -type f | sort - run: du -sh site @@ -141,23 +136,26 @@ jobs: runs-on: [self-hosted, nixos, microvm] steps: - uses: actions/checkout@v7 - - name: Put the dev shell's toolchain on PATH for actions - run: dirname "$(command -v cargo)" >> "$GITHUB_PATH" - - uses: Swatinem/rust-cache@v2 # NOT `cargo fmt --all`, which reaches only ONE of the two workspace # roots. Exit 2 here is a usage error, never a verdict (improvise-b4ks). - name: Every workspace root is formatted run: ./scripts/fmt-workspaces.sh - - run: cargo clippy --workspace --all-targets -- -D warnings - - run: cargo test --workspace + # checks.clippy and checks.test in nix/ci.nix: the same cargo commands, + # on the dev/test profiles, with dependencies as a cached derivation. + - name: cargo clippy --workspace --all-targets -- -D warnings + shell: bash + run: nix build --print-build-logs .#checks.x86_64-linux.clippy + # Also runs the test-evidence gate (check-comment-runs.py + # --check-evidence), which lists the tests this derivation compiled. + - name: cargo test --workspace, and test-evidence headings name real tests + shell: bash + run: nix build --print-build-logs .#checks.x86_64-linux.test # The comment cap and the test-evidence keys (improvise-9c8c); without # this step the rule is advisory. See context/design-principles.md §6. - name: The comment checker catches what it has been fixed for run: python3 scripts/check-comment-runs.py --self-test - name: Comment runs capped at two lines run: python3 scripts/check-comment-runs.py - - name: Test-evidence headings name real tests - run: python3 scripts/check-comment-runs.py --check-evidence # `pages-deploy` needs every other job, and saying so was not enough: a # job added during review of PR #83 was ungated and nothing noticed. - name: The deploy gate refuses what it claims to @@ -186,38 +184,14 @@ jobs: runs-on: [self-hosted, nixos, microvm] steps: - uses: actions/checkout@v7 - - name: Put the dev shell's toolchain on PATH for actions - run: dirname "$(command -v cargo)" >> "$GITHUB_PATH" - - uses: Swatinem/rust-cache@v2 - # Only the crates that go into the browser bundle. The root `improvise` - # crate is the terminal binary and depends on crossterm, which does not - # build for wasm — that is correct, not a gap (improvise-ywd). - - name: improvise-core - run: cargo check -p improvise-core --target wasm32-unknown-unknown - - name: improvise-formula - run: cargo check -p improvise-formula --target wasm32-unknown-unknown - - name: improvise-protocol - run: cargo check -p improvise-protocol --target wasm32-unknown-unknown - - name: improvise-io - run: cargo check -p improvise-io --target wasm32-unknown-unknown - - name: improvise-command - run: cargo check -p improvise-command --target wasm32-unknown-unknown - # The browser client itself. This is the one crate for which a wasm - # build is the *point* rather than a property worth keeping, so a - # failure here is not a portability regression but a broken deliverable - # (improvise-gsw). - - name: improvise-wasm-client - run: cargo check -p improvise-wasm-client --target wasm32-unknown-unknown - # The worker half of the standalone deployment (improvise-9x6). Same - # terms as the client: a wasm build is the point, so a failure here is a - # broken deliverable rather than a portability regression. - - name: improvise-worker-server - run: cargo check -p improvise-worker-server --target wasm32-unknown-unknown - # Not in the browser bundle yet — nothing depends on improvise-gp. Gated - # anyway because it is a candidate (fitting a tree in the browser) and - # staying wasm-clean is free now, expensive to retrofit (improvise-z5eg). - - name: improvise-gp - run: cargo check -p improvise-gp --target wasm32-unknown-unknown + # checks.wasm-check in nix/ci.nix: `cargo check --target wasm32` for each + # crate in the browser bundle, one at a time, so features are not unified + # across them. The root `improvise` crate is the terminal binary and + # depends on crossterm, which does not build for wasm — that is correct, + # not a gap (improvise-ywd). The log names each crate as it is checked. + - name: cargo check --target wasm32-unknown-unknown, per bundle crate + shell: bash + run: nix build --print-build-logs .#checks.x86_64-linux.wasm-check # --------------------------------------------------------------------------- # The browser JavaScript's unit tests. Its own job, deliberately: a failure @@ -296,15 +270,11 @@ jobs: runs-on: [self-hosted, nixos, microvm] steps: - uses: actions/checkout@v7 - - name: Put the dev shell's toolchain on PATH for actions - run: dirname "$(command -v cargo)" >> "$GITHUB_PATH" - - uses: Swatinem/rust-cache@v2 - with: - workspaces: harness/opfs-contract -> target - - working-directory: harness/opfs-contract - run: | - cargo fmt -p opfs-contract -- --check - cargo clippy --target wasm32-unknown-unknown --all-targets -- -D warnings + # checks.harness-lint in nix/ci.nix, run from inside harness/opfs-contract: + # `cargo fmt -p opfs-contract -- --check` and clippy for wasm32. + - name: cargo fmt -p opfs-contract and clippy --target wasm32 + shell: bash + run: nix build --print-build-logs .#checks.x86_64-linux.harness-lint # --------------------------------------------------------------------------- # The browser harness (improvise-vvl3, absorbing improvise-xf1). diff --git a/flake.lock b/flake.lock index 5d755b13..32d9649a 100644 --- a/flake.lock +++ b/flake.lock @@ -45,6 +45,21 @@ "type": "github" } }, + "crane": { + "locked": { + "lastModified": 1789760033, + "narHash": "sha256-jtT4yxZpR8seYnlCMMWSSPlFN92zO6ICuZ8pJrmi86k=", + "owner": "ipetkov", + "repo": "crane", + "rev": "73b980519cefc727a5f6cc8e5c0947a2f9be6edd", + "type": "github" + }, + "original": { + "owner": "ipetkov", + "repo": "crane", + "type": "github" + } + }, "crate2nix": { "inputs": { "cachix": "cachix", @@ -341,6 +356,7 @@ "root": { "inputs": { "beads": "beads", + "crane": "crane", "crate2nix": "crate2nix", "flake-utils": "flake-utils", "nixpkgs": "nixpkgs_4", diff --git a/flake.nix b/flake.nix index 8e686b9b..7f06ffe0 100644 --- a/flake.nix +++ b/flake.nix @@ -9,6 +9,7 @@ }; flake-utils.url = "github:numtide/flake-utils"; crate2nix.url = "github:nix-community/crate2nix"; + crane.url = "github:ipetkov/crane"; }; outputs = { @@ -18,6 +19,7 @@ rust-overlay, flake-utils, crate2nix, + crane, }: flake-utils.lib.eachDefaultSystem (system: let pkgs = import nixpkgs { @@ -87,6 +89,14 @@ # Playwright's downloaded browsers are generic-Linux builds that cannot # start on NixOS, so nixpkgs' build is used. harness/package.json pins # @playwright/test to this playwright-driver's version: they must match. + craneLib = (crane.mkLib pkgs).overrideToolchain (_: rustToolchain); + ci = import ./nix/ci.nix { + inherit pkgs craneLib wasmBindgenCli; + src = self; + # What the site's cache-bust token names; CI checks out a clean tree. + rev = self.rev or "dirty"; + }; + playwrightEnv = { PLAYWRIGHT_BROWSERS_PATH = "${pkgs.playwright-driver.browsers}"; PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD = "1"; @@ -115,9 +125,19 @@ ]; }); - packages = { - improvise = cargoNix.workspaceMembers.improvise.build; - default = self.packages.${system}.improvise; - }; + packages = + ci.packages + // { + improvise = cargoNix.workspaceMembers.improvise.build; + default = self.packages.${system}.improvise; + }; + + # `nix flake check` builds every CI build: crane's checks plus the + # crate2nix package and the bundles. + checks = + ci.checks + // { + inherit (self.packages.${system}) improvise web; + }; }); } diff --git a/nix/ci.nix b/nix/ci.nix new file mode 100644 index 00000000..ceb4f384 --- /dev/null +++ b/nix/ci.nix @@ -0,0 +1,184 @@ +# Everything CI *builds*, as Nix derivations (crane). The script-only checks +# (node tests, kb, the checkers, the comment and CI-graph gates, fmt) stay dev +# shell steps in ci.yml: they compile nothing, and their `#!/usr/bin/env bash` +# shebangs -- which scripts/checkers.sh itself enforces -- cannot run in the +# build sandbox, which has no /usr/bin/env. +# +# Why Nix: the self-hosted runners substitute from titan's harmonia first, and +# titan builds gh/main's outputs hourly. Dependencies are separate derivations +# (buildDepsOnly, keyed on the Cargo.lock), so a PR that leaves the lock alone +# rebuilds only the workspace. +{ + pkgs, + craneLib, + wasmBindgenCli, + src, + rev, +}: let + inherit (pkgs) lib; + + # Two lockfiles: harness/opfs-contract is its own workspace root. + cargoVendorDir = craneLib.vendorMultipleCargoDeps { + cargoLockList = [../Cargo.lock ../harness/opfs-contract/Cargo.lock]; + }; + + common = { + inherit src cargoVendorDir; + pname = "improvise"; + version = "0"; + strictDeps = true; + # The `test` job ran the dev/test profiles, not crane's default release: + # debug assertions and overflow checks are part of what it tested. + CARGO_PROFILE = ""; + }; + + wasmTarget = "wasm32-unknown-unknown"; + + # The crates in the browser bundle, checked for wasm32 one at a time: a + # single `cargo check -p a -p b` unifies features across them and can hide + # a crate that only builds with a sibling's features (improvise-ywd). + wasmCrates = [ + "improvise-core" + "improvise-formula" + "improvise-protocol" + "improvise-io" + "improvise-command" + "improvise-wasm-client" + "improvise-worker-server" + "improvise-gp" + ]; + pFlags = crates: lib.concatMapStringsSep " " (c: "-p ${c}") crates; + + nativeDeps = craneLib.buildDepsOnly (common + // { + pname = "improvise-deps"; + cargoExtraArgs = "--locked --workspace"; + }); + + wasmCheckDeps = craneLib.buildDepsOnly (common + // { + pname = "improvise-wasm-check-deps"; + cargoExtraArgs = "--locked --target ${wasmTarget} ${pFlags wasmCrates}"; + cargoCheckExtraArgs = ""; + doCheck = false; + }); + + # What wasm-pack's `cargo build --release --target wasm32` needs. + bundleDeps = craneLib.buildDepsOnly (common + // { + pname = "improvise-bundle-deps"; + CARGO_PROFILE = "release"; + cargoExtraArgs = "--locked --target ${wasmTarget} ${pFlags ["improvise-wasm-client" "improvise-worker-server"]}"; + cargoCheckCommand = "true"; + doCheck = false; + }); + + # A derivation that runs cargo commands and only has to succeed. + cargoCheck = name: args: + craneLib.mkCargoDerivation (common + // { + pname = "improvise-${name}"; + doInstallCargoArtifacts = false; + installPhaseCommand = "mkdir -p $out"; + } + // args); +in rec { + checks = { + clippy = craneLib.cargoClippy (common + // { + pname = "improvise-clippy"; + cargoArtifacts = nativeDeps; + cargoClippyExtraArgs = "--workspace --all-targets -- -D warnings"; + }); + + # The evidence gate runs `cargo test --workspace -- --list`, so it runs + # here, where the tests are already compiled, not as a separate step. + test = craneLib.cargoTest (common + // { + pname = "improvise-test"; + cargoArtifacts = nativeDeps; + cargoTestExtraArgs = "--workspace"; + nativeBuildInputs = [pkgs.python3 pkgs.git]; + # tests/shipped_documents_load.rs walks the git index (`git ls-files`) + # and makes scratch repos. The flake source is exactly the tracked + # files, so force-adding all of it (but crane's target/) rebuilds the + # checkout's index, ignored-but-tracked documents included. + preBuild = '' + export HOME=$TMPDIR + export GIT_AUTHOR_NAME=nix GIT_AUTHOR_EMAIL=nix@localhost + export GIT_COMMITTER_NAME=nix GIT_COMMITTER_EMAIL=nix@localhost + git init -q . + git add --all --force -- . ':!target' + ''; + postBuild = '' + python3 scripts/check-comment-runs.py --check-evidence + ''; + }); + + wasm-check = cargoCheck "wasm-check" { + cargoArtifacts = wasmCheckDeps; + buildPhaseCargoCommand = lib.concatMapStringsSep "\n" (c: '' + echo "== ${c}" + cargo check --locked -p ${c} --target ${wasmTarget} + '') + wasmCrates; + }; + + # harness/opfs-contract is its own workspace, so the root clippy never + # reaches it; `-p` keeps cargo fmt to this one crate (see ci.yml). + harness-lint = cargoCheck "harness-lint" { + cargoArtifacts = null; + buildPhaseCargoCommand = '' + cd harness/opfs-contract + cargo fmt -p opfs-contract -- --check + cargo clippy --locked --target ${wasmTarget} --all-targets -- -D warnings + ''; + }; + }; + + packages = { + # The dependency builds on their own, for titan's cache timer to keep as a + # GC root: the checks' outputs do not reference them, so rooting only the + # checks would let the part a PR actually reuses be collected. + ci-deps = pkgs.linkFarmFromDrvs "improvise-ci-deps" [nativeDeps wasmCheckDeps bundleDeps]; + + # harness/build.sh, unchanged: the three wasm bundles and harness/serve. + # No network: wasm-bindgen and wasm-opt come from PATH, so wasm-pack + # downloads nothing (see wasmBindgenCli in flake.nix). + web = craneLib.mkCargoDerivation (common + // { + pname = "improvise-web"; + cargoArtifacts = bundleDeps; + doInstallCargoArtifacts = false; + nativeBuildInputs = [pkgs.wasm-pack wasmBindgenCli pkgs.binaryen pkgs.nodejs_24]; + # web/assemble.sh calls scripts/assemble-stamp.sh, whose + # `#!/usr/bin/env bash` has no interpreter in the sandbox. Patching + # this private copy is harmless: no checker runs here. + postPatch = "patchShebangs scripts web harness"; + buildPhaseCargoCommand = '' + export HOME=$TMPDIR + ./harness/build.sh + ''; + installPhaseCommand = '' + mkdir -p $out + cp -r harness/serve $out/harness-serve + cp -r web/pkg web/worker-pkg $out/ + ''; + }); + + # The Pages tree, stamped with the commit it was built from. Separate from + # `web` so the stamp changing every commit does not rebuild the bundles. + site = + pkgs.runCommand "improvise-site" { + nativeBuildInputs = [pkgs.nodejs_24]; + } '' + cp -r ${src} tree + chmod -R u+w tree + patchShebangs tree/scripts tree/web + cp -r ${packages.web}/pkg ${packages.web}/worker-pkg tree/web/ + chmod -R u+w tree/web + cd tree + ./web/assemble.sh $out ${rev} + ''; + }; +} From 1790764f766959a437e9c5e3ace4f5c9970f113b Mon Sep 17 00:00:00 2001 From: Ed L Date: Wed, 23 Sep 2026 21:58:51 -0700 Subject: [PATCH 5/5] ci: run the browser harness on Chromium and Firefox only nixpkgs' WebKit (WPEWebProcess) aborts on its first page inside the self-hosted runner VMs, though not on the host with the same sandbox; its OPFS specs skip on a runtime probe anyway. Re-add --project=webkit once the abort is understood. Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/workflows/ci.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f8f7f9ea..2fdaca1d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -337,9 +337,13 @@ jobs: # renamed export is cheaper to read here than as a blank page. - name: The page's calls match the bundle it calls run: ./scripts/node-test.sh 'web/tests/bundle/*.test.js' + # Chromium and Firefox only, for now. nixpkgs' WebKit (WPEWebProcess) + # aborts on its first page inside the runner VMs, though not on the + # host; its four specs skip on the OPFS probe anyway, so nothing gated + # is lost. Put `--project=webkit` back once that abort is understood. - name: Run the harness working-directory: harness - run: npx playwright test + run: npx playwright test --project=chromium --project=firefox - if: failure() uses: actions/upload-artifact@v7 with: