voidzero-dev
diff --git a/.github/workflows/config-performance.yml b/.github/workflows/config-performance.yml
@@ -0,0 +1,182 @@
+name: Config Performance
+
+permissions:
+  contents: read
+  actions: read
+  packages: read
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
+    paths:
+      - 'bench/config-performance/**'
+      - 'packages/**'
+      - 'crates/**'
+      - '.github/actions/**'
+      - '.github/workflows/config-performance.yml'
+      - '.node-version'
+      - '.cargo/**'
+      - 'Cargo.*'
+      - 'rust-toolchain.toml'
+      - 'pnpm-*.yaml'
+      - 'package.json'
+
+concurrency:
+  group: config-performance-${{ github.event.pull_request.number }}
+  cancel-in-progress: true
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  benchmark:
+    name: Config loading benchmark
+    runs-on: namespace-profile-linux-x64-default
+    timeout-minutes: 30
+    outputs:
+      report-ready: ${{ steps.measure.outputs.report-ready }}
+    steps:
+      - uses: taiki-e/checkout-action@7d1e50e93dc4fb3bba58f85018fadf77898aee8b # v1.4.2
+      - uses: ./.github/actions/clone
+      - uses: oxc-project/setup-rust@68c3199c5339f965e6e163924c3c450773eba42b # main
+        with:
+          save-cache: false
+          cache-key: config-performance
+      - uses: oxc-project/setup-node@1f1a5b4450c8905bd1830c3a908d22b960c4559a # main
+      - uses: ./.github/actions/build-upstream
+        with:
+          target: x86_64-unknown-linux-gnu
+
+      # Prefer the base branch of a stacked PR, then an earlier run of this PR.
+      - name: Find previous measurements
+        id: baseline
+        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9
+        with:
+          script: |
+            const repo = context.repo;
+            const current = await github.rest.actions.getWorkflowRun({ ...repo, run_id: context.runId });
+            const { data } = await github.rest.actions.listWorkflowRuns({
+              ...repo, workflow_id: current.data.workflow_id, status: 'success', per_page: 100,
+            });
+            const runs = data.workflow_runs.filter(run => run.id !== context.runId);
+            const pr = context.payload.pull_request;
+            const baseBranch = pr.base.ref !== 'main' ? runs.filter(run =>
+              run.event === 'pull_request' && run.head_branch === pr.base.ref &&
+              run.head_repository?.full_name === pr.base.repo.full_name
+            ) : [];
+            const previousPR = pr ? runs.filter(run =>
+              run.event === 'pull_request' && run.head_branch === pr.head.ref &&
+              run.head_repository?.full_name === pr.head.repo.full_name
+            ) : [];
+            // A rerun can compare with the retained artifact of its own last
+            // successful attempt. Upload replaces that artifact only at the end.
+            const previousAttempt = [];
+            const attemptNumber = Number(process.env.GITHUB_RUN_ATTEMPT);
+            if (attemptNumber > 1) {
+              const attempt = await github.request(
+                'GET /repos/{owner}/{repo}/actions/runs/{run_id}/attempts/{attempt_number}',
+                { ...repo, run_id: context.runId, attempt_number: attemptNumber - 1 },
+              );
+              if (attempt.data.conclusion === 'success') previousAttempt.push(current.data);
+            }
+            core.info(`Attempt ${attemptNumber}: ${baseBranch.length} base branch, ${previousAttempt.length} previous attempt, ${previousPR.length} PR baseline candidates.`);
+            for (const run of [...baseBranch, ...previousAttempt, ...previousPR]) {
+              const artifacts = await github.rest.actions.listWorkflowRunArtifacts({ ...repo, run_id: run.id });
+              if (artifacts.data.artifacts.some(artifact => artifact.name === 'config-performance' && !artifact.expired)) {
+                core.setOutput('run-id', String(run.id));
+                core.info(`Using baseline from ${run.html_url}`);
+                return;
+              }
+            }
+            core.info('No previous measurements; this run establishes a baseline.');
+
+      - name: Download previous measurements
+        if: steps.baseline.outputs.run-id != ''
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: config-performance
+          path: ${{ runner.temp }}/config-performance-baseline
+          run-id: ${{ steps.baseline.outputs.run-id }}
+          github-token: ${{ github.token }}
+
+      - name: Run benchmark
+        id: measure
+        env:
+          BASELINE_RUN: ${{ steps.baseline.outputs.run-id }}
+        run: |
+          args=()
+          if [[ -n "$BASELINE_RUN" ]]; then
+            args+=(--baseline "$RUNNER_TEMP/config-performance-baseline/results.json")
+          fi
+          node bench/config-performance/run.ts --output "$RUNNER_TEMP/config-performance" "${args[@]}"
+
+      - name: Upload measurements
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: config-performance
+          path: ${{ runner.temp }}/config-performance
+          if-no-files-found: warn
+          retention-days: 90
+          overwrite: true
+
+  comment:
+    name: Report config performance changes
+    needs: benchmark
+    # Report timing regressions even when the benchmark fails its timing gate.
+    # Fork tokens cannot write comments; their reports stay in the job summary.
+    if: >-
+      always() && !cancelled() &&
+      needs.benchmark.outputs.report-ready == 'true' &&
+      github.event.pull_request.head.repo.full_name == github.repository
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      issues: write
+      pull-requests: write
+    steps:
+      - name: Download measurements
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: config-performance
+          path: ${{ runner.temp }}/config-performance
+
+      - name: Update PR comment for changes beyond five percent
+        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9
+        env:
+          REPORT_DIR: ${{ runner.temp }}/config-performance
+        with:
+          script: |
+            const fs = require('node:fs');
+            const path = require('node:path');
+            const pr = context.payload.pull_request;
+            const { data: latest } = await github.rest.pulls.get({
+              ...context.repo, pull_number: pr.number,
+            });
+            if (latest.state !== 'open' || latest.head.sha !== pr.head.sha) {
+              core.info('The PR closed or its head changed; skip this older report.');
+              return;
+            }
+            const marker = '<!-- vp-config-performance -->';
+            const comments = await github.paginate(github.rest.issues.listComments, {
+              ...context.repo, issue_number: pr.number,
+            });
+            const existing = comments.find(comment =>
+              comment.user?.login === 'github-actions[bot]' && comment.body?.includes(marker)
+            );
+            const report = fs.readFileSync(path.join(process.env.REPORT_DIR, 'comment.md'), 'utf8');
+            if (!report) {
+              if (existing) {
+                await github.rest.issues.deleteComment({ ...context.repo, comment_id: existing.id });
+              }
+              core.info('No comparable median change beyond ±5%; no PR comment needed.');
+              return;
+            }
+            const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}/attempts/${process.env.GITHUB_RUN_ATTEMPT}`;
+            const body = `${marker}\n\n[Benchmark run](${runUrl}) for PR head \`${pr.head.sha}\`.\n\n${report}`;
+            if (existing) {
+              await github.rest.issues.updateComment({ ...context.repo, comment_id: existing.id, body });
+            } else {
+              await github.rest.issues.createComment({ ...context.repo, issue_number: pr.number, body });
+            }
diff --git a/bench/config-performance/README.md b/bench/config-performance/README.md
@@ -0,0 +1,39 @@
+This benchmark tracks the config-loading costs described in [#2698](https://github.com/voidzero-dev/vite-plus/issues/2698). It runs the checkout's built CLI without modifying its code or dependencies.
+
+Build the checkout as described in [CONTRIBUTING.md](../../CONTRIBUTING.md), then run on Linux or macOS:
+
+```sh
+node bench/config-performance/run.ts
+```
+
+Results go to `tmp/config-performance/results.json` and `summary.md`. The summary shows baseline → current values for median and p95, plus the absolute and percentage median change. To compare with an earlier result on the same machine and Node version:
+
+```sh
+node bench/config-performance/run.ts --baseline previous-results.json
+```
+
+Use `--samples 2 --warmup 1` for a smoke run. A timing comparison requires at least seven samples. Defaults are 15 samples and three warmup rounds.
+
+The cases cover root and package-directory `vp check --fix`, missing and minimal configs, `defineConfig`, root `lint`/`fmt` blocks, standalone tools, and `vp staged` with a check or a no-op task. Each sample starts a fresh process. Cases rotate between rounds. File and Git preparation happen outside the timed interval, and each sample receives the same three unformatted TypeScript files. Staged samples verify the formatted Git index. Command errors and timeouts fail the benchmark.
+
+Config evaluations are measured in separate runs. Synchronous log writes do not affect the timing samples. The current ceilings are four evaluations for a root check, seven for a package check, and five for staged checks. Each Oxc child may evaluate its config at most once. These ceilings retain the current package-directory overhead until a separate optimization reduces it. Reduce the relevant ceiling with that optimization.
+
+The benchmark uses temporary projects outside the checkout so ancestor config discovery cannot find the repository's config. It runs the same Node executable in staged tasks and tool children. It uses a separate Node compile-cache directory and removes fixtures after completion. Warmups make this a measurement of fresh-process startup with warm filesystem and compile caches, not a cold-disk benchmark.
+
+The [Config Performance workflow](../../.github/workflows/config-performance.yml) runs only when a relevant PR opens, updates, or reopens. Draft PRs run normally.
+
+CI retains JSON samples and Markdown reports for 90 days. Stacked PRs prefer measurements from their base branch. Otherwise, reruns can use their previous successful attempt, and PR updates can use an earlier successful run of the same branch. The first run records a baseline. Workload, Node version, operating system, architecture, CPU model, and CPU count must match for a timing comparison; a mismatch is reported explicitly.
+
+If any comparable case's median changes by more than +5% or −5%, CI posts or updates one PR comment with the full comparison and a link to the run. Faster and slower results both trigger a comment, without an absolute-time threshold. Exactly ±5% does not trigger a comment. CI removes its previous comment when no case exceeds this threshold or comparison is unavailable. Missing or incompatible baselines do not trigger comments. Fork PRs retain the report in the job summary because their GitHub token cannot write comments.
+
+The notification threshold is separate from the failure threshold below. A timing regression still produces a comment if the benchmark fails. `comment.md` contains the comment text, or is empty when no notification is needed.
+
+A timing regression fails CI when all three conditions hold:
+
+- The median grows by more than 20%.
+- The median grows by more than 40 ms.
+- The current 25th percentile exceeds the baseline 75th percentile.
+
+This catches substantial, sustained regressions while tolerating isolated slow samples. It does not prove that smaller changes are harmless. Review the medians, p95 values, raw samples, and config counts when changing config resolution. CPU load and runner image changes can still affect results. Config-count ceilings apply even when timing results are not comparable.
+
+The workload hash changes with the fixture or command definitions, so those changes establish a new timing baseline. Evaluation ceilings are excluded from the hash so lowering a ceiling after an optimization preserves the timing comparison. Tool versions are recorded but are not part of the compatibility check: dependency updates must remain visible in the comparison.
diff --git a/bench/config-performance/results.spec.ts b/bench/config-performance/results.spec.ts
@@ -0,0 +1,86 @@
+import { expect, test } from 'vite-plus/test';
+
+import { compareReports, type BenchmarkReport } from './results.ts';
+
+function report(samples: number[]): BenchmarkReport {
+  return {
+    schemaVersion: 1,
+    workload: 'fixture-v1',
+    revision: 'test-revision',
+    environment: { node: 'v22.18.0', platform: 'linux', arch: 'x64', cpu: 'test', cpus: 4 },
+    versions: {},
+    results: [{ id: 'root/check/minimal', samples, configLoads: [] }],
+  };
+}
+
+test('a single slow sample does not cause a timing regression', () => {
+  const baseline = report([490, 495, 499, 500, 501, 505, 510]);
+  const current = report([490, 495, 499, 500, 501, 505, 5000]);
+  expect(compareReports(current, baseline).regressions).toEqual([]);
+});
+
+test('a sustained slowdown fails the timing comparison', () => {
+  const baseline = report([490, 495, 499, 500, 501, 505, 510]);
+  const current = report([640, 645, 649, 650, 651, 655, 660]);
+  expect(compareReports(current, baseline).regressions).toHaveLength(1);
+});
+
+test('small absolute changes and overlapping distributions do not fail', () => {
+  expect(
+    compareReports(
+      report([130, 130, 130, 130, 130, 130, 130]),
+      report([100, 100, 100, 100, 100, 100, 100]),
+    ).regressions,
+  ).toEqual([]);
+  expect(
+    compareReports(
+      report([490, 500, 550, 650, 750, 800, 900]),
+      report([400, 450, 500, 500, 550, 700, 800]),
+    ).regressions,
+  ).toEqual([]);
+});
+
+test('PR notifications include improvements and small absolute changes beyond five percent', () => {
+  const baseline = report(Array(7).fill(100));
+  for (const median of [94, 106]) {
+    const comparison = compareReports(report(Array(7).fill(median)), baseline);
+    expect(comparison.notableChanges).toEqual(['root/check/minimal']);
+    expect(comparison.regressions).toEqual([]);
+  }
+});
+
+test('PR notifications exclude changes within or exactly at five percent', () => {
+  const baseline = report(Array(7).fill(100));
+  for (const median of [95, 96, 100, 104, 105]) {
+    expect(compareReports(report(Array(7).fill(median)), baseline).notableChanges).toEqual([]);
+  }
+});
+
+test('PR notifications require a compatible baseline', () => {
+  const baseline = report(Array(7).fill(100));
+  const current = report(Array(7).fill(500));
+  expect(compareReports(current).notableChanges).toEqual([]);
+  current.environment.node = 'v24.0.0';
+  expect(compareReports(current, baseline).notableChanges).toEqual([]);
+});
+
+test('different environments and workloads are explicitly not compared', () => {
+  const baseline = report([100, 100, 100, 100, 100, 100, 100]);
+  const current = report([500, 500, 500, 500, 500, 500, 500]);
+  current.environment.node = 'v24.0.0';
+  expect(compareReports(current, baseline).markdown).toContain('comparison skipped');
+  current.environment = baseline.environment;
+  current.workload = 'fixture-v2';
+  expect(compareReports(current, baseline).markdown).toContain('comparison skipped');
+});
+
+test('invalid or incomplete baseline data cannot silently pass', () => {
+  const current = report([500, 500, 500, 500, 500, 500, 500]);
+  expect(() => compareReports(current, report([Number.NaN, 1, 1, 1, 1, 1, 1]))).toThrow(
+    'positive, finite',
+  );
+  expect(() => compareReports(current, report([500]))).toThrow('seven samples');
+  const baseline = report([500, 500, 500, 500, 500, 500, 500]);
+  baseline.results = [];
+  expect(() => compareReports(current, baseline)).toThrow('missing case');
+});