diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7b2d49df..f8f7f9ea 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -53,20 +53,6 @@ jobs: # below, so it fails under this job's name rather than another's. - name: The stamp gate refuses what it claims to run: ./scripts/checkers.sh assemble-stamp - # rust-cache is a JS action: it runs outside the dev shell, and needs - # cargo and rustc on the job's PATH to compute its key. - - name: Put the dev shell's toolchain on PATH for actions - run: dirname "$(command -v cargo)" >> "$GITHUB_PATH" - # Two workspaces, because `harness/opfs-contract` carries an empty - # `[workspace]` table and so has its own target dir, which the default - # single-workspace configuration never cached (it recompiled its whole - # dependency tree every run). - - uses: Swatinem/rust-cache@v2 - with: - workspaces: | - . -> target - harness/opfs-contract -> target - # scripts/web-toolchain-versions.env stays the one home for the pins, so # the release capture (scripts/build-web-release.sh, improvise-oujb) # provisions the same versions. CI no longer installs them — the dev @@ -90,20 +76,29 @@ jobs: exit 1 } - # The same script a developer runs locally, so there is one answer to - # "how is the bundle built" rather than two that can drift. It builds all - # three wasm bundles (client, worker, opfs-contract) and assembles - # `harness/serve` — the deployed arrangement plus /opfs/. - - name: Build the bundles and the harness tree - run: ./harness/build.sh - - # Re-assembles from the pkg dirs the step above already produced — no - # wasm-pack runs here. A second assembly rather than a copy of - # `harness/serve` because the cache-bust token differs: the harness tree - # is stamped `harness` and the published site the commit SHA, which is - # what makes a returning visitor pick up a fresh bundle (improvise-9x6). - - name: Assemble the site - run: ./web/assemble.sh site "$GITHUB_SHA" + # Built by Nix (packages.web and packages.site in nix/ci.nix), which runs + # the same harness/build.sh and web/assemble.sh a developer runs, so there + # is still one answer to "how is the bundle built". `web` is the three + # wasm bundles plus `harness/serve`; `site` re-assembles from web's pkg + # dirs with the commit SHA as the cache-bust token (improvise-9x6), where + # the harness tree is stamped `harness`. Dependencies are their own + # derivations, substituted from titan's cache when Cargo.lock is unchanged. + - name: Build the bundles, the harness tree and the site + shell: bash + run: | + nix build --print-build-logs --out-link result-web .#web + nix build --print-build-logs --out-link result-site .#site + # The upload steps below read these paths. Copied rather than linked: + # an artifact of a symlink into /nix/store is a dangling link elsewhere. + - name: Lay the outputs out where the uploads read them + shell: bash + run: | + rm -rf harness/serve web/pkg web/worker-pkg site + cp -rL result-web/harness-serve harness/serve + cp -rL result-web/pkg web/pkg + cp -rL result-web/worker-pkg web/worker-pkg + cp -rL result-site site + chmod -R u+w harness/serve web/pkg web/worker-pkg site - run: find site -type f | sort - run: du -sh site @@ -141,23 +136,26 @@ jobs: runs-on: [self-hosted, nixos, microvm] steps: - uses: actions/checkout@v7 - - name: Put the dev shell's toolchain on PATH for actions - run: dirname "$(command -v cargo)" >> "$GITHUB_PATH" - - uses: Swatinem/rust-cache@v2 # NOT `cargo fmt --all`, which reaches only ONE of the two workspace # roots. Exit 2 here is a usage error, never a verdict (improvise-b4ks). - name: Every workspace root is formatted run: ./scripts/fmt-workspaces.sh - - run: cargo clippy --workspace --all-targets -- -D warnings - - run: cargo test --workspace + # checks.clippy and checks.test in nix/ci.nix: the same cargo commands, + # on the dev/test profiles, with dependencies as a cached derivation. + - name: cargo clippy --workspace --all-targets -- -D warnings + shell: bash + run: nix build --print-build-logs .#checks.x86_64-linux.clippy + # Also runs the test-evidence gate (check-comment-runs.py + # --check-evidence), which lists the tests this derivation compiled. + - name: cargo test --workspace, and test-evidence headings name real tests + shell: bash + run: nix build --print-build-logs .#checks.x86_64-linux.test # The comment cap and the test-evidence keys (improvise-9c8c); without # this step the rule is advisory. See context/design-principles.md §6. - name: The comment checker catches what it has been fixed for run: python3 scripts/check-comment-runs.py --self-test - name: Comment runs capped at two lines run: python3 scripts/check-comment-runs.py - - name: Test-evidence headings name real tests - run: python3 scripts/check-comment-runs.py --check-evidence # `pages-deploy` needs every other job, and saying so was not enough: a # job added during review of PR #83 was ungated and nothing noticed. - name: The deploy gate refuses what it claims to @@ -186,38 +184,14 @@ jobs: runs-on: [self-hosted, nixos, microvm] steps: - uses: actions/checkout@v7 - - name: Put the dev shell's toolchain on PATH for actions - run: dirname "$(command -v cargo)" >> "$GITHUB_PATH" - - uses: Swatinem/rust-cache@v2 - # Only the crates that go into the browser bundle. The root `improvise` - # crate is the terminal binary and depends on crossterm, which does not - # build for wasm — that is correct, not a gap (improvise-ywd). - - name: improvise-core - run: cargo check -p improvise-core --target wasm32-unknown-unknown - - name: improvise-formula - run: cargo check -p improvise-formula --target wasm32-unknown-unknown - - name: improvise-protocol - run: cargo check -p improvise-protocol --target wasm32-unknown-unknown - - name: improvise-io - run: cargo check -p improvise-io --target wasm32-unknown-unknown - - name: improvise-command - run: cargo check -p improvise-command --target wasm32-unknown-unknown - # The browser client itself. This is the one crate for which a wasm - # build is the *point* rather than a property worth keeping, so a - # failure here is not a portability regression but a broken deliverable - # (improvise-gsw). - - name: improvise-wasm-client - run: cargo check -p improvise-wasm-client --target wasm32-unknown-unknown - # The worker half of the standalone deployment (improvise-9x6). Same - # terms as the client: a wasm build is the point, so a failure here is a - # broken deliverable rather than a portability regression. - - name: improvise-worker-server - run: cargo check -p improvise-worker-server --target wasm32-unknown-unknown - # Not in the browser bundle yet — nothing depends on improvise-gp. Gated - # anyway because it is a candidate (fitting a tree in the browser) and - # staying wasm-clean is free now, expensive to retrofit (improvise-z5eg). - - name: improvise-gp - run: cargo check -p improvise-gp --target wasm32-unknown-unknown + # checks.wasm-check in nix/ci.nix: `cargo check --target wasm32` for each + # crate in the browser bundle, one at a time, so features are not unified + # across them. The root `improvise` crate is the terminal binary and + # depends on crossterm, which does not build for wasm — that is correct, + # not a gap (improvise-ywd). The log names each crate as it is checked. + - name: cargo check --target wasm32-unknown-unknown, per bundle crate + shell: bash + run: nix build --print-build-logs .#checks.x86_64-linux.wasm-check # --------------------------------------------------------------------------- # The browser JavaScript's unit tests. Its own job, deliberately: a failure @@ -296,15 +270,11 @@ jobs: runs-on: [self-hosted, nixos, microvm] steps: - uses: actions/checkout@v7 - - name: Put the dev shell's toolchain on PATH for actions - run: dirname "$(command -v cargo)" >> "$GITHUB_PATH" - - uses: Swatinem/rust-cache@v2 - with: - workspaces: harness/opfs-contract -> target - - working-directory: harness/opfs-contract - run: | - cargo fmt -p opfs-contract -- --check - cargo clippy --target wasm32-unknown-unknown --all-targets -- -D warnings + # checks.harness-lint in nix/ci.nix, run from inside harness/opfs-contract: + # `cargo fmt -p opfs-contract -- --check` and clippy for wasm32. + - name: cargo fmt -p opfs-contract and clippy --target wasm32 + shell: bash + run: nix build --print-build-logs .#checks.x86_64-linux.harness-lint # --------------------------------------------------------------------------- # The browser harness (improvise-vvl3, absorbing improvise-xf1). diff --git a/flake.lock b/flake.lock index 5d755b13..32d9649a 100644 --- a/flake.lock +++ b/flake.lock @@ -45,6 +45,21 @@ "type": "github" } }, + "crane": { + "locked": { + "lastModified": 1789760033, + "narHash": "sha256-jtT4yxZpR8seYnlCMMWSSPlFN92zO6ICuZ8pJrmi86k=", + "owner": "ipetkov", + "repo": "crane", + "rev": "73b980519cefc727a5f6cc8e5c0947a2f9be6edd", + "type": "github" + }, + "original": { + "owner": "ipetkov", + "repo": "crane", + "type": "github" + } + }, "crate2nix": { "inputs": { "cachix": "cachix", @@ -341,6 +356,7 @@ "root": { "inputs": { "beads": "beads", + "crane": "crane", "crate2nix": "crate2nix", "flake-utils": "flake-utils", "nixpkgs": "nixpkgs_4", diff --git a/flake.nix b/flake.nix index 8e686b9b..7f06ffe0 100644 --- a/flake.nix +++ b/flake.nix @@ -9,6 +9,7 @@ }; flake-utils.url = "github:numtide/flake-utils"; crate2nix.url = "github:nix-community/crate2nix"; + crane.url = "github:ipetkov/crane"; }; outputs = { @@ -18,6 +19,7 @@ rust-overlay, flake-utils, crate2nix, + crane, }: flake-utils.lib.eachDefaultSystem (system: let pkgs = import nixpkgs { @@ -87,6 +89,14 @@ # Playwright's downloaded browsers are generic-Linux builds that cannot # start on NixOS, so nixpkgs' build is used. harness/package.json pins # @playwright/test to this playwright-driver's version: they must match. + craneLib = (crane.mkLib pkgs).overrideToolchain (_: rustToolchain); + ci = import ./nix/ci.nix { + inherit pkgs craneLib wasmBindgenCli; + src = self; + # What the site's cache-bust token names; CI checks out a clean tree. + rev = self.rev or "dirty"; + }; + playwrightEnv = { PLAYWRIGHT_BROWSERS_PATH = "${pkgs.playwright-driver.browsers}"; PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD = "1"; @@ -115,9 +125,19 @@ ]; }); - packages = { - improvise = cargoNix.workspaceMembers.improvise.build; - default = self.packages.${system}.improvise; - }; + packages = + ci.packages + // { + improvise = cargoNix.workspaceMembers.improvise.build; + default = self.packages.${system}.improvise; + }; + + # `nix flake check` builds every CI build: crane's checks plus the + # crate2nix package and the bundles. + checks = + ci.checks + // { + inherit (self.packages.${system}) improvise web; + }; }); } diff --git a/nix/ci.nix b/nix/ci.nix new file mode 100644 index 00000000..ceb4f384 --- /dev/null +++ b/nix/ci.nix @@ -0,0 +1,184 @@ +# Everything CI *builds*, as Nix derivations (crane). The script-only checks +# (node tests, kb, the checkers, the comment and CI-graph gates, fmt) stay dev +# shell steps in ci.yml: they compile nothing, and their `#!/usr/bin/env bash` +# shebangs -- which scripts/checkers.sh itself enforces -- cannot run in the +# build sandbox, which has no /usr/bin/env. +# +# Why Nix: the self-hosted runners substitute from titan's harmonia first, and +# titan builds gh/main's outputs hourly. Dependencies are separate derivations +# (buildDepsOnly, keyed on the Cargo.lock), so a PR that leaves the lock alone +# rebuilds only the workspace. +{ + pkgs, + craneLib, + wasmBindgenCli, + src, + rev, +}: let + inherit (pkgs) lib; + + # Two lockfiles: harness/opfs-contract is its own workspace root. + cargoVendorDir = craneLib.vendorMultipleCargoDeps { + cargoLockList = [../Cargo.lock ../harness/opfs-contract/Cargo.lock]; + }; + + common = { + inherit src cargoVendorDir; + pname = "improvise"; + version = "0"; + strictDeps = true; + # The `test` job ran the dev/test profiles, not crane's default release: + # debug assertions and overflow checks are part of what it tested. + CARGO_PROFILE = ""; + }; + + wasmTarget = "wasm32-unknown-unknown"; + + # The crates in the browser bundle, checked for wasm32 one at a time: a + # single `cargo check -p a -p b` unifies features across them and can hide + # a crate that only builds with a sibling's features (improvise-ywd). + wasmCrates = [ + "improvise-core" + "improvise-formula" + "improvise-protocol" + "improvise-io" + "improvise-command" + "improvise-wasm-client" + "improvise-worker-server" + "improvise-gp" + ]; + pFlags = crates: lib.concatMapStringsSep " " (c: "-p ${c}") crates; + + nativeDeps = craneLib.buildDepsOnly (common + // { + pname = "improvise-deps"; + cargoExtraArgs = "--locked --workspace"; + }); + + wasmCheckDeps = craneLib.buildDepsOnly (common + // { + pname = "improvise-wasm-check-deps"; + cargoExtraArgs = "--locked --target ${wasmTarget} ${pFlags wasmCrates}"; + cargoCheckExtraArgs = ""; + doCheck = false; + }); + + # What wasm-pack's `cargo build --release --target wasm32` needs. + bundleDeps = craneLib.buildDepsOnly (common + // { + pname = "improvise-bundle-deps"; + CARGO_PROFILE = "release"; + cargoExtraArgs = "--locked --target ${wasmTarget} ${pFlags ["improvise-wasm-client" "improvise-worker-server"]}"; + cargoCheckCommand = "true"; + doCheck = false; + }); + + # A derivation that runs cargo commands and only has to succeed. + cargoCheck = name: args: + craneLib.mkCargoDerivation (common + // { + pname = "improvise-${name}"; + doInstallCargoArtifacts = false; + installPhaseCommand = "mkdir -p $out"; + } + // args); +in rec { + checks = { + clippy = craneLib.cargoClippy (common + // { + pname = "improvise-clippy"; + cargoArtifacts = nativeDeps; + cargoClippyExtraArgs = "--workspace --all-targets -- -D warnings"; + }); + + # The evidence gate runs `cargo test --workspace -- --list`, so it runs + # here, where the tests are already compiled, not as a separate step. + test = craneLib.cargoTest (common + // { + pname = "improvise-test"; + cargoArtifacts = nativeDeps; + cargoTestExtraArgs = "--workspace"; + nativeBuildInputs = [pkgs.python3 pkgs.git]; + # tests/shipped_documents_load.rs walks the git index (`git ls-files`) + # and makes scratch repos. The flake source is exactly the tracked + # files, so force-adding all of it (but crane's target/) rebuilds the + # checkout's index, ignored-but-tracked documents included. + preBuild = '' + export HOME=$TMPDIR + export GIT_AUTHOR_NAME=nix GIT_AUTHOR_EMAIL=nix@localhost + export GIT_COMMITTER_NAME=nix GIT_COMMITTER_EMAIL=nix@localhost + git init -q . + git add --all --force -- . ':!target' + ''; + postBuild = '' + python3 scripts/check-comment-runs.py --check-evidence + ''; + }); + + wasm-check = cargoCheck "wasm-check" { + cargoArtifacts = wasmCheckDeps; + buildPhaseCargoCommand = lib.concatMapStringsSep "\n" (c: '' + echo "== ${c}" + cargo check --locked -p ${c} --target ${wasmTarget} + '') + wasmCrates; + }; + + # harness/opfs-contract is its own workspace, so the root clippy never + # reaches it; `-p` keeps cargo fmt to this one crate (see ci.yml). + harness-lint = cargoCheck "harness-lint" { + cargoArtifacts = null; + buildPhaseCargoCommand = '' + cd harness/opfs-contract + cargo fmt -p opfs-contract -- --check + cargo clippy --locked --target ${wasmTarget} --all-targets -- -D warnings + ''; + }; + }; + + packages = { + # The dependency builds on their own, for titan's cache timer to keep as a + # GC root: the checks' outputs do not reference them, so rooting only the + # checks would let the part a PR actually reuses be collected. + ci-deps = pkgs.linkFarmFromDrvs "improvise-ci-deps" [nativeDeps wasmCheckDeps bundleDeps]; + + # harness/build.sh, unchanged: the three wasm bundles and harness/serve. + # No network: wasm-bindgen and wasm-opt come from PATH, so wasm-pack + # downloads nothing (see wasmBindgenCli in flake.nix). + web = craneLib.mkCargoDerivation (common + // { + pname = "improvise-web"; + cargoArtifacts = bundleDeps; + doInstallCargoArtifacts = false; + nativeBuildInputs = [pkgs.wasm-pack wasmBindgenCli pkgs.binaryen pkgs.nodejs_24]; + # web/assemble.sh calls scripts/assemble-stamp.sh, whose + # `#!/usr/bin/env bash` has no interpreter in the sandbox. Patching + # this private copy is harmless: no checker runs here. + postPatch = "patchShebangs scripts web harness"; + buildPhaseCargoCommand = '' + export HOME=$TMPDIR + ./harness/build.sh + ''; + installPhaseCommand = '' + mkdir -p $out + cp -r harness/serve $out/harness-serve + cp -r web/pkg web/worker-pkg $out/ + ''; + }); + + # The Pages tree, stamped with the commit it was built from. Separate from + # `web` so the stamp changing every commit does not rebuild the bundles. + site = + pkgs.runCommand "improvise-site" { + nativeBuildInputs = [pkgs.nodejs_24]; + } '' + cp -r ${src} tree + chmod -R u+w tree + patchShebangs tree/scripts tree/web + cp -r ${packages.web}/pkg ${packages.web}/worker-pkg tree/web/ + chmod -R u+w tree/web + cd tree + ./web/assemble.sh $out ${rev} + ''; + }; +}