diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 8dc29b56..35e5fccd 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -1,8 +1,13 @@ name: Benchmark on: + pull_request: + branches: [dev] + types: [opened, synchronize, reopened, labeled] push: branches: [dev] + schedule: + - cron: "17 3 * * *" workflow_dispatch: inputs: duration: @@ -18,32 +23,75 @@ env: GIT_CONFIG_COUNT: 1 GIT_CONFIG_KEY_0: init.defaultBranch GIT_CONFIG_VALUE_0: main + BENCHER_API_TOKEN: ${{ secrets.BENCHER_API_TOKEN }} + BENCHER_PROJECT: ${{ secrets.BENCHER_PROJECT }} + BENCHER_DASHBOARD_URL: ${{ vars.BENCHER_DASHBOARD_URL }} jobs: benchmark: permissions: contents: read + pull-requests: write + issues: write statuses: write runs-on: blacksmith-2vcpu-ubuntu-2404 timeout-minutes: 30 steps: - uses: actions/checkout@v4 + - name: Check benchmark trigger gate + id: gate + uses: actions/github-script@v9 + with: + script: | + if (context.eventName !== 'pull_request') { + core.setOutput('run', 'true'); + return; + } + + const labels = context.payload.pull_request.labels.map((label) => label.name); + if (labels.includes('run-benchmark')) { + core.setOutput('run', 'true'); + return; + } + + const files = await github.paginate(github.rest.pulls.listFiles, { + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.payload.pull_request.number, + per_page: 100, + }); + const shouldRun = files.some((file) => { + const name = file.filename; + return name === 'build.zig' || name.startsWith('src/') || name.startsWith('modules/') || name.startsWith('assets/shaders/'); + }); + core.setOutput('run', shouldRun ? 'true' : 'false'); + + - name: Explain skipped benchmark + if: steps.gate.outputs.run != 'true' + run: | + printf 'Skipping benchmark: no benchmark-relevant paths changed and run-benchmark label is absent.\n' + - name: Setup Nix + if: steps.gate.outputs.run == 'true' uses: ./.github/actions/setup-nix - name: Setup Zig cache + if: steps.gate.outputs.run == 'true' uses: ./.github/actions/setup-zig-cache with: cache-key-prefix: zig-ci-graphics - name: Start headless Wayland compositor + if: steps.gate.outputs.run == 'true' uses: ./.github/actions/start-weston - name: Setup Lavapipe Vulkan + if: steps.gate.outputs.run == 'true' uses: ./.github/actions/setup-lavapipe - name: Run benchmark suite + if: steps.gate.outputs.run == 'true' id: run_benchmark uses: ./.github/actions/run-with-log with: @@ -60,18 +108,51 @@ jobs: BENCHMARK_DURATION: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.duration || '5' }} - name: Stop headless Wayland compositor - if: always() + if: always() && steps.gate.outputs.run == 'true' uses: ./.github/actions/stop-weston - name: Compare against baseline + if: steps.gate.outputs.run == 'true' id: compare run: | for preset in low medium high; do bash scripts/compare_benchmarks.sh docs/benchmarks/baseline.json "benchmark-results/${preset}.json" --preset "$preset" done + - name: Install Bencher CLI + if: steps.gate.outputs.run == 'true' && env.BENCHER_API_TOKEN != '' && env.BENCHER_PROJECT != '' + uses: bencherdev/bencher@30a740a2e4246560b1a5fd424057d84aa2b188d6 + + - name: Publish Bencher trends + if: steps.gate.outputs.run == 'true' && env.BENCHER_API_TOKEN != '' && env.BENCHER_PROJECT != '' + env: + BENCHER_BRANCH: ${{ github.head_ref || github.ref_name }} + run: | + for preset in low medium high; do + bencher run \ + --project "$BENCHER_PROJECT" \ + --branch "$BENCHER_BRANCH" \ + --testbed "github-actions-lavapipe" \ + --token "$BENCHER_API_TOKEN" \ + --adapter json \ + --err \ + "cat benchmark-results/${preset}.json" + done + + - name: Comment Bencher dashboard + if: steps.gate.outputs.run == 'true' && github.event_name == 'pull_request' && env.BENCHER_DASHBOARD_URL != '' + uses: actions/github-script@v9 + with: + script: | + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.payload.pull_request.number, + body: `Benchmark trends: ${process.env.BENCHER_DASHBOARD_URL}`, + }); + - name: Upload benchmark artifacts - if: always() + if: always() && steps.gate.outputs.run == 'true' uses: actions/upload-artifact@v7 with: name: benchmark-results @@ -82,7 +163,7 @@ jobs: retention-days: 30 - name: Publish commit status - if: always() + if: always() && steps.gate.outputs.run == 'true' uses: actions/github-script@v9 env: BENCHMARK_RUN_OUTCOME: ${{ steps.run_benchmark.outcome }} diff --git a/.github/workflows/opencode-audit.yml b/.github/workflows/opencode-audit.yml index 17dc0fe9..ac4a2b0b 100644 --- a/.github/workflows/opencode-audit.yml +++ b/.github/workflows/opencode-audit.yml @@ -73,7 +73,7 @@ jobs: MODULE_PATH: ${{ steps.select-module.outputs.module_path }} - name: Run opencode audit - uses: anomalyco/opencode/github@latest + uses: anomalyco/opencode/github@77fc88c8ade8e5a620ebbe1197f3a572d29ae91a env: GITHUB_TOKEN: ${{ github.token }} GH_TOKEN: ${{ secrets.OPENCODE_PAT }} diff --git a/.github/workflows/opencode-pr.yml b/.github/workflows/opencode-pr.yml index 9fedf19a..7f9b5944 100644 --- a/.github/workflows/opencode-pr.yml +++ b/.github/workflows/opencode-pr.yml @@ -75,7 +75,7 @@ jobs: - name: Run opencode id: ai_review timeout-minutes: 40 - uses: anomalyco/opencode/github@latest + uses: anomalyco/opencode/github@77fc88c8ade8e5a620ebbe1197f3a572d29ae91a env: GITHUB_TOKEN: ${{ github.token }} GH_TOKEN: ${{ github.token }} diff --git a/.github/workflows/opencode-test-writer.yml b/.github/workflows/opencode-test-writer.yml index 782d58c8..ac151548 100644 --- a/.github/workflows/opencode-test-writer.yml +++ b/.github/workflows/opencode-test-writer.yml @@ -92,7 +92,7 @@ jobs: - name: Run opencode test writer if: steps.check-existing.outputs.skip != 'true' - uses: anomalyco/opencode/github@latest + uses: anomalyco/opencode/github@77fc88c8ade8e5a620ebbe1197f3a572d29ae91a env: GITHUB_TOKEN: ${{ secrets.OPENCODE_PAT }} GH_TOKEN: ${{ secrets.OPENCODE_PAT }} diff --git a/.github/workflows/opencode-triage.yml b/.github/workflows/opencode-triage.yml index 74ff1f24..e262b3bc 100644 --- a/.github/workflows/opencode-triage.yml +++ b/.github/workflows/opencode-triage.yml @@ -58,7 +58,7 @@ jobs: with: prompt-file: .github/prompts/triage.md - - uses: anomalyco/opencode/github@latest + - uses: anomalyco/opencode/github@77fc88c8ade8e5a620ebbe1197f3a572d29ae91a if: steps.check.outputs.result == 'true' env: GITHUB_TOKEN: ${{ github.token }} diff --git a/.github/workflows/opencode.yml b/.github/workflows/opencode.yml index 40338183..bb08aa73 100644 --- a/.github/workflows/opencode.yml +++ b/.github/workflows/opencode.yml @@ -42,7 +42,7 @@ jobs: run: mkdir -p /tmp/opencode-cache && echo "XDG_CACHE_HOME=/tmp/opencode-cache" >> "$GITHUB_ENV" - name: Run opencode - uses: anomalyco/opencode/github@latest + uses: anomalyco/opencode/github@77fc88c8ade8e5a620ebbe1197f3a572d29ae91a env: GITHUB_TOKEN: ${{ secrets.OPENCODE_PAT }} GH_TOKEN: ${{ secrets.OPENCODE_PAT }} diff --git a/.github/workflows/visual-test.yml b/.github/workflows/visual-test.yml index a43cc8e4..b9a5dfe5 100644 --- a/.github/workflows/visual-test.yml +++ b/.github/workflows/visual-test.yml @@ -1,6 +1,11 @@ name: visual-test on: + schedule: + - cron: "43 4 * * *" + pull_request: + branches: [dev] + types: [labeled] workflow_dispatch: concurrency: @@ -9,12 +14,14 @@ concurrency: jobs: visual-test: + if: github.event_name != 'pull_request' || github.event.label.name == 'run-visual-test' runs-on: ubuntu-latest timeout-minutes: 30 permissions: id-token: write contents: read issues: write + pull-requests: read steps: - uses: actions/checkout@v4 @@ -52,6 +59,11 @@ jobs: --description "Issues from automated visual regression tests" \ --color "E06C75" fi + if ! gh label list --json name --jq '.[].name' | grep -q '^run-visual-test$'; then + gh label create "run-visual-test" \ + --description "Run deterministic visual regression workflow on a PR" \ + --color "E06C75" + fi env: GH_TOKEN: ${{ secrets.OPENCODE_PAT }} @@ -66,7 +78,7 @@ jobs: name: Visual Test timeout: 20m log-file: build-output.log - command: nix develop .#ci-graphics --command zig build run -Dscreenshot-path=screenshot.ppm -Dskip-present=true + command: nix develop .#ci-graphics --command zig build run -Dscreenshot-path=screenshot.png -Dskip-present=true env: ZIG_GLOBAL_CACHE_DIR: /tmp/zig-cache-global XDG_RUNTIME_DIR: /tmp/runtime-runner @@ -74,25 +86,32 @@ jobs: ZIGCRAFT_SMOKE_FRAMES: "5" ZIGCRAFT_SAFE_RENDER: "1" - - name: Convert PPM to PNG - id: convert + - name: Check screenshot exists + id: check_screenshot if: always() run: | - if [ -f screenshot.ppm ]; then - nix run nixpkgs#imagemagick -- screenshot.ppm screenshot.png + if [ -f screenshot.png ]; then echo "screenshot_exists=true" >> "$GITHUB_OUTPUT" else echo "screenshot_exists=false" >> "$GITHUB_OUTPUT" fi + - name: Compare against golden image + id: golden_diff + if: steps.check_screenshot.outputs.screenshot_exists == 'true' + run: | + nix shell nixpkgs#imagemagick -c bash scripts/compare_visual_golden.sh screenshot.png docs/visual-test/golden/menu.png visual-diff.png + env: + VISUAL_DIFF_RMSE_TOLERANCE: "0.015" + - name: Upload screenshot artifact - if: steps.convert.outputs.screenshot_exists == 'true' + if: always() && steps.check_screenshot.outputs.screenshot_exists == 'true' uses: actions/upload-artifact@v7 with: name: menu-screenshot path: | screenshot.png - screenshot.ppm + visual-diff.png retention-days: 30 - name: Check build log exists @@ -115,8 +134,9 @@ jobs: uses: ./.github/actions/stop-weston - name: Run opencode visual verification - if: steps.convert.outputs.screenshot_exists == 'true' - uses: anomalyco/opencode/github@latest + if: always() && steps.check_screenshot.outputs.screenshot_exists == 'true' + continue-on-error: true + uses: anomalyco/opencode/github@77fc88c8ade8e5a620ebbe1197f3a572d29ae91a env: GITHUB_TOKEN: ${{ secrets.OPENCODE_PAT }} MINIMAX_API_KEY: ${{ secrets.MINIMAX_API_KEY }} @@ -125,8 +145,9 @@ jobs: prompt: ${{ env.VISUAL_VERIFY_PROMPT }} - name: Run opencode failure diagnosis - if: failure() || steps.screenshot.outcome == 'failure' || steps.convert.outputs.screenshot_exists != 'true' - uses: anomalyco/opencode/github@latest + if: failure() || steps.screenshot.outcome == 'failure' || steps.check_screenshot.outputs.screenshot_exists != 'true' + continue-on-error: true + uses: anomalyco/opencode/github@77fc88c8ade8e5a620ebbe1197f3a572d29ae91a env: GITHUB_TOKEN: ${{ secrets.OPENCODE_PAT }} MINIMAX_API_KEY: ${{ secrets.MINIMAX_API_KEY }} diff --git a/build.zig b/build.zig index bc287c4d..316a94f8 100644 --- a/build.zig +++ b/build.zig @@ -866,38 +866,8 @@ fn applySanitizeC(sanitize_c: ?std.zig.SanitizeC, modules: []const *std.Build.Mo } fn defineShaderValidation(b: *std.Build, test_step: *std.Build.Step) void { - const shader_paths = [_][]const u8{ - "assets/shaders/vulkan/terrain.vert", - "assets/shaders/vulkan/terrain.frag", - "assets/shaders/vulkan/shadow.vert", - "assets/shaders/vulkan/shadow.frag", - "assets/shaders/vulkan/sky.vert", - "assets/shaders/vulkan/sky.frag", - "assets/shaders/vulkan/ui.vert", - "assets/shaders/vulkan/ui.frag", - "assets/shaders/vulkan/ui_tex.vert", - "assets/shaders/vulkan/ui_tex.frag", - "assets/shaders/vulkan/debug_shadow.vert", - "assets/shaders/vulkan/debug_shadow.frag", - "assets/shaders/vulkan/ssao.vert", - "assets/shaders/vulkan/ssao.frag", - "assets/shaders/vulkan/ssao_blur.frag", - "assets/shaders/vulkan/g_pass.frag", - "assets/shaders/vulkan/taa.vert", - "assets/shaders/vulkan/taa.frag", - "assets/shaders/vulkan/lpv_inject.comp", - "assets/shaders/vulkan/lpv_propagate.comp", - "assets/shaders/vulkan/culling.comp", - "assets/shaders/vulkan/depth_pyramid.comp", - "assets/shaders/vulkan/water.vert", - "assets/shaders/vulkan/water.frag", - "assets/shaders/vulkan/mesh.comp", - }; - - for (shader_paths) |path| { - const validate = b.addSystemCommand(&.{ "glslangValidator", "-V", path }); - test_step.dependOn(&validate.step); - } + const validate = b.addSystemCommand(&.{ "bash", "scripts/check_spirv_sizes.sh", "docs/shaders/spirv-sizes.json" }); + test_step.dependOn(&validate.step); } fn addCimgui(_: *std.Build, compile: *std.Build.Step.Compile) void { diff --git a/docs/benchmarks/README.md b/docs/benchmarks/README.md index a4c09d79..0a732991 100644 --- a/docs/benchmarks/README.md +++ b/docs/benchmarks/README.md @@ -39,3 +39,19 @@ This gate is intentionally a short canary so every `dev` push gets a bounded per - Higher `draw_calls_avg` is a regression. The p1 FPS metric is the primary user-visible smoothness guard. GPU time and draw calls catch rendering-cost regressions even when FPS is noisy on virtualized runners. + +## Absolute SLOs + +The benchmark harness also enforces absolute service-level objectives before regression comparison. These thresholds are intentionally separate from `baseline.json`: a run can fail even if there is no baseline drift when the absolute floor or ceiling is breached. + +| Preset | p1 FPS min | Max frame ms | Draw calls avg max | Vertices avg max | GPU memory max | +| --- | ---: | ---: | ---: | ---: | ---: | +| low | 12 | 260 | 700 | 3,500,000 | 1,800 MB | +| medium | 8 | 260 | 2,600 | 6,000,000 | 2,400 MB | +| high | 6 | 260 | 3,600 | 8,500,000 | 2,800 MB | +| ultra | 4 | 260 | 4,500 | 12,000,000 | 3,400 MB | +| extreme | 3 | 260 | 5,500 | 16,000,000 | 4,096 MB | + +The FPS floors are Lavapipe CI canary values, not player-facing hardware targets. The 260 ms spike guard allows the current Lavapipe startup spike in 5-second canary runs while still failing large stalls. Draw-call, vertex, and measured GPU resource memory ceilings are deliberately loose enough for runner variance but strict enough to fail large accidental rendering explosions. + +Benchmark history is pushed to Bencher when the `BENCHER_API_TOKEN` and `BENCHER_PROJECT` secrets are configured. Pull-request benchmark runs are gated by either relevant file changes (`src/**`, `modules/**`, `assets/shaders/**`, or `build.zig`) or the `run-benchmark` label. diff --git a/docs/benchmarks/baseline.json b/docs/benchmarks/baseline.json index 9218f654..3dcb5341 100644 --- a/docs/benchmarks/baseline.json +++ b/docs/benchmarks/baseline.json @@ -1,81 +1,90 @@ { - "generated": false, + "generated": true, "presets": { "low": { "preset": "low", - "render_distance": 0, - "frames": 0, - "duration_s": 0, + "render_distance": 6, + "gpu_memory_mb_avg": 897.0490755179424, + "gpu_memory_mb_max": 1687.2586669921875, + "frames": 1689, + "duration_s": 5, "fps": { - "min": 0, - "avg": 0, - "max": 0, - "p1": 0, - "p5": 0, - "p50": 0, - "p95": 0, - "p99": 0 + "min": 4.251558780670166, + "avg": 337.6105611060426, + "max": 22987.44921875, + "p1": 16.746460189819334, + "p5": 255.20651245117188, + "p50": 568.6878051757812, + "p95": 2452.8090820312477, + "p99": 4087.7167382812445 }, - "cpu_ms_avg": 0, + "max_frame_ms": 235.20785522460938, + "cpu_ms_avg": 2.9619926483458037, "gpu_ms": { - "shadow_avg": 0, - "opaque_avg": 0, - "total_avg": 0 + "shadow_avg": 0.0003647246829023427, + "opaque_avg": 1.2093640969721011, + "total_avg": 1.4246198751469066 }, - "draw_calls_avg": 0, - "vertices_avg": 0, - "chunks_rendered_avg": 0 + "draw_calls_avg": 530.0740082889283, + "vertices_avg": 802823.8223801065, + "chunks_rendered_avg": 116.20367081113085 }, "medium": { "preset": "medium", - "render_distance": 0, - "frames": 0, - "duration_s": 0, + "render_distance": 10, + "gpu_memory_mb_avg": 908.5787859218208, + "gpu_memory_mb_max": 1687.2586669921875, + "frames": 1009, + "duration_s": 5, "fps": { - "min": 0, - "avg": 0, - "max": 0, - "p1": 0, - "p5": 0, - "p50": 0, - "p95": 0, - "p99": 0 + "min": 4.1065673828125, + "avg": 201.66515096869702, + "max": 22180.326171875, + "p1": 13.274825859069825, + "p5": 136.4259033203125, + "p50": 352.11181640625, + "p95": 1130.5526123046873, + "p99": 3102.763417968748 }, - "cpu_ms_avg": 0, + "max_frame_ms": 243.5123748779297, + "cpu_ms_avg": 4.958714954946394, "gpu_ms": { - "shadow_avg": 0, - "opaque_avg": 0, - "total_avg": 0 + "shadow_avg": 0.19116780963035004, + "opaque_avg": 1.3985375998425413, + "total_avg": 2.3772802531305697 }, - "draw_calls_avg": 0, - "vertices_avg": 0, - "chunks_rendered_avg": 0 + "draw_calls_avg": 2238.9692765113973, + "vertices_avg": 1249039.9623389496, + "chunks_rendered_avg": 190.07829534192268 }, "high": { "preset": "high", - "render_distance": 0, - "frames": 0, - "duration_s": 0, + "render_distance": 12, + "gpu_memory_mb_avg": 884.0339247241462, + "gpu_memory_mb_max": 1639.2586669921875, + "frames": 970, + "duration_s": 5, "fps": { - "min": 0, - "avg": 0, - "max": 0, - "p1": 0, - "p5": 0, - "p50": 0, - "p95": 0, - "p99": 0 + "min": 4.183460712432861, + "avg": 193.99032472602713, + "max": 26217.818359375, + "p1": 12.47351658821106, + "p5": 125.32577514648438, + "p50": 325.8839569091797, + "p95": 1058.7565307617183, + "p99": 2874.141457519523 }, - "cpu_ms_avg": 0, + "max_frame_ms": 239.0365447998047, + "cpu_ms_avg": 5.154896263060035, "gpu_ms": { - "shadow_avg": 0, - "opaque_avg": 0, - "total_avg": 0 + "shadow_avg": 0.2523342988425002, + "opaque_avg": 1.3704036711263903, + "total_avg": 2.593851779969697 }, - "draw_calls_avg": 0, - "vertices_avg": 0, - "chunks_rendered_avg": 0 + "draw_calls_avg": 2228.2072164948454, + "vertices_avg": 1245972.1917525774, + "chunks_rendered_avg": 189.34329896907215 } }, - "note": "Populate this file with real benchmark baselines before enabling regression gating." + "note": "Generated from 5-second Lavapipe ReleaseFast benchmark runs for CI canary gating." } diff --git a/docs/shaders/spirv-sizes.json b/docs/shaders/spirv-sizes.json new file mode 100644 index 00000000..52009cc4 --- /dev/null +++ b/docs/shaders/spirv-sizes.json @@ -0,0 +1,39 @@ +{ + "threshold_percent": 10, + "shaders": { + "assets/shaders/vulkan/bloom_downsample.frag": 9764, + "assets/shaders/vulkan/bloom_downsample.vert": 1160, + "assets/shaders/vulkan/bloom_upsample.frag": 3956, + "assets/shaders/vulkan/bloom_upsample.vert": 1160, + "assets/shaders/vulkan/culling.comp": 11348, + "assets/shaders/vulkan/debug_shadow.frag": 836, + "assets/shaders/vulkan/debug_shadow.vert": 1344, + "assets/shaders/vulkan/depth_pyramid.comp": 2936, + "assets/shaders/vulkan/fxaa.frag": 5916, + "assets/shaders/vulkan/fxaa.vert": 1160, + "assets/shaders/vulkan/g_pass.frag": 6912, + "assets/shaders/vulkan/lpv_inject.comp": 4844, + "assets/shaders/vulkan/lpv_propagate.comp": 8316, + "assets/shaders/vulkan/mesh.comp": 45244, + "assets/shaders/vulkan/post_process.frag": 12308, + "assets/shaders/vulkan/post_process.vert": 1160, + "assets/shaders/vulkan/shadow.frag": 552, + "assets/shaders/vulkan/shadow.vert": 4128, + "assets/shaders/vulkan/sky.frag": 18756, + "assets/shaders/vulkan/sky.vert": 2696, + "assets/shaders/vulkan/ssao_blur.frag": 1760, + "assets/shaders/vulkan/ssao.frag": 6588, + "assets/shaders/vulkan/ssao.vert": 1160, + "assets/shaders/vulkan/taa.frag": 9760, + "assets/shaders/vulkan/taa.vert": 1160, + "assets/shaders/vulkan/terrain_debug.frag": 1336, + "assets/shaders/vulkan/terrain.frag": 84752, + "assets/shaders/vulkan/terrain.vert": 11144, + "assets/shaders/vulkan/ui.frag": 744, + "assets/shaders/vulkan/ui_tex.frag": 1408, + "assets/shaders/vulkan/ui_tex.vert": 1344, + "assets/shaders/vulkan/ui.vert": 1336, + "assets/shaders/vulkan/water.frag": 22236, + "assets/shaders/vulkan/water.vert": 8796 + } +} diff --git a/docs/visual-test/README.md b/docs/visual-test/README.md new file mode 100644 index 00000000..20b0557f --- /dev/null +++ b/docs/visual-test/README.md @@ -0,0 +1,13 @@ +# Visual Regression Tests + +`visual-test.yml` captures the menu in Lavapipe headless mode and compares it against `docs/visual-test/golden/menu.png` with ImageMagick RMSE. The golden diff is the primary gate; LLM verification remains a non-blocking diagnostic to explain failures. + +## Regenerating The Golden + +Run the same capture path used by CI, then replace the golden image after reviewing the result: + +```bash +nix develop .#ci-graphics --command zig build run -Dskip-present=true -Dscreenshot-path=docs/visual-test/golden/menu.png +``` + +CI uses `VISUAL_DIFF_RMSE_TOLERANCE=0.015` to allow small Lavapipe version differences while still catching deterministic layout or rendering regressions. diff --git a/docs/visual-test/golden/menu.png b/docs/visual-test/golden/menu.png new file mode 100644 index 00000000..b196e76d Binary files /dev/null and b/docs/visual-test/golden/menu.png differ diff --git a/modules/engine-graphics/src/render_system.zig b/modules/engine-graphics/src/render_system.zig index 8fdf1720..fc0479bb 100644 --- a/modules/engine-graphics/src/render_system.zig +++ b/modules/engine-graphics/src/render_system.zig @@ -3,6 +3,7 @@ const Allocator = std.mem.Allocator; const c = @import("c").c; const log = @import("engine-core").log; +const RenderDevice = @import("engine-rhi").RenderDevice; const rhi_pkg = @import("engine-rhi").rhi; const RHI = rhi_pkg.RHI; const backend_dispatcher = @import("backend_dispatcher.zig"); @@ -50,6 +51,7 @@ pub const RenderSystem = struct { allocator: Allocator, rhi: RHI, + render_device: *RenderDevice, shader: rhi_pkg.ShaderHandle, resource_pack_manager: ResourcePackManager, atlas: TextureAtlas, @@ -125,15 +127,21 @@ pub const RenderSystem = struct { } log.log.info("Initializing {s} backend...", .{@tagName(backend_dispatcher.BackendChoice.vulkan)}); - const rhi = try backend_dispatcher.createRHI(allocator, window, .vulkan, .{ + var rhi = try backend_dispatcher.createRHI(allocator, window, .vulkan, .{ .shadow_resolution = config.shadow_resolution, .msaa_samples = config.msaa_samples, .anisotropic_filtering = config.anisotropic_filtering, }); errdefer rhi.deinit(); + const render_device = try allocator.create(RenderDevice); + errdefer allocator.destroy(render_device); + render_device.* = try RenderDevice.init(allocator); + errdefer render_device.deinit(); + log.log.info("RenderSystem.init: initializing RHI device", .{}); - try rhi.init(allocator, null); + try rhi.init(allocator, render_device); + rhi.device = render_device; log.log.info("RenderSystem.init: scanning resource packs", .{}); var resource_pack_manager = ResourcePackManager.init(allocator); @@ -197,6 +205,7 @@ pub const RenderSystem = struct { self.* = .{ .allocator = allocator, .rhi = rhi, + .render_device = render_device, .shader = rhi_pkg.InvalidShaderHandle, .resource_pack_manager = resource_pack_manager, .atlas = atlas, @@ -299,6 +308,8 @@ pub const RenderSystem = struct { self.resource_pack_manager.deinit(); if (self.shader != rhi_pkg.InvalidShaderHandle) self.rhi.resourceManager().destroyShader(self.shader); self.rhi.deinit(); + self.render_device.deinit(); + self.allocator.destroy(self.render_device); self.allocator.destroy(self); } diff --git a/modules/engine-graphics/src/vulkan/resource_manager.zig b/modules/engine-graphics/src/vulkan/resource_manager.zig index b67fe080..faf2da26 100644 --- a/modules/engine-graphics/src/vulkan/resource_manager.zig +++ b/modules/engine-graphics/src/vulkan/resource_manager.zig @@ -1,6 +1,7 @@ const std = @import("std"); const c = @import("c").c; const rhi = @import("engine-rhi").rhi; +const RenderDeviceStats = @import("engine-rhi").Stats; const log = @import("engine-core").log; const VulkanDevice = @import("../vulkan_device.zig").VulkanDevice; const Utils = @import("utils.zig"); @@ -24,6 +25,7 @@ pub const TextureResource = struct { depth: u32, format: rhi.TextureFormat, config: rhi.TextureConfig, + allocation_size: usize = 0, is_3d: bool = false, is_owned: bool = true, }; @@ -177,6 +179,28 @@ pub const ResourceManager = struct { self.image_deletion_queue[frame_index].clearRetainingCapacity(); } + pub fn stats(self: *const ResourceManager) RenderDeviceStats { + var result = RenderDeviceStats{ + .buffer_count = @intCast(self.buffers.count()), + .texture_count = @intCast(self.textures.count()), + .shader_count = 0, + .total_buffer_memory = 0, + .total_texture_memory = 0, + }; + + var buf_it = self.buffers.valueIterator(); + while (buf_it.next()) |buf| { + result.total_buffer_memory += @intCast(buf.size); + } + + var tex_it = self.textures.valueIterator(); + while (tex_it.next()) |tex| { + result.total_texture_memory += tex.allocation_size; + } + + return result; + } + pub fn resetTransferState(self: *ResourceManager) void { self.transfer.resetTransferState(); } @@ -342,6 +366,7 @@ pub const ResourceManager = struct { .depth = 1, .format = format, .config = .{}, // Default config + .allocation_size = 0, .is_3d = false, .is_owned = false, }); @@ -365,6 +390,7 @@ pub const ResourceManager = struct { .depth = 1, .format = format, .config = .{}, + .allocation_size = 0, .is_3d = false, .is_owned = false, }); diff --git a/modules/engine-graphics/src/vulkan/resource_texture_ops.zig b/modules/engine-graphics/src/vulkan/resource_texture_ops.zig index dcd47066..f59af46e 100644 --- a/modules/engine-graphics/src/vulkan/resource_texture_ops.zig +++ b/modules/engine-graphics/src/vulkan/resource_texture_ops.zig @@ -216,6 +216,7 @@ pub fn createTexture(self: anytype, width: u32, height: u32, format: rhi.Texture .depth = 1, .format = format, .config = config, + .allocation_size = @intCast(mem_reqs.size), .is_3d = false, .is_owned = true, }); @@ -367,6 +368,7 @@ pub fn createTexture3D(self: anytype, width: u32, height: u32, depth: u32, forma .depth = depth, .format = format, .config = texture_config, + .allocation_size = @intCast(mem_reqs.size), .is_3d = true, .is_owned = true, }); diff --git a/modules/engine-graphics/src/vulkan/rhi_pass_orchestration.zig b/modules/engine-graphics/src/vulkan/rhi_pass_orchestration.zig index 63c5108c..27084c47 100644 --- a/modules/engine-graphics/src/vulkan/rhi_pass_orchestration.zig +++ b/modules/engine-graphics/src/vulkan/rhi_pass_orchestration.zig @@ -481,5 +481,9 @@ pub fn endFrame(ctx: anytype) void { ctx.resources.resetTransferState(); } + if (ctx.render_device) |device| { + device.setStats(ctx.resources.stats()); + } + ctx.runtime.frame_index += 1; } diff --git a/modules/game-core/src/benchmark.zig b/modules/game-core/src/benchmark.zig index 5d95e4e8..c2886261 100644 --- a/modules/game-core/src/benchmark.zig +++ b/modules/game-core/src/benchmark.zig @@ -30,6 +30,15 @@ pub const FrameSample = struct { draw_calls: u32, vertices: u64, chunks_rendered: u32, + gpu_memory_mb: f32, +}; + +pub const SloThresholds = struct { + fps_p1_min: f64, + max_frame_ms: f64, + draw_calls_max: f64, + vertices_max: f64, + gpu_memory_mb_max: f64, }; pub const Summary = struct { @@ -52,9 +61,12 @@ pub const GpuSummary = struct { pub const BenchmarkResults = struct { preset: []const u8, render_distance: i32, + gpu_memory_mb_avg: f64, + gpu_memory_mb_max: f64, frames: u32, duration_s: f32, fps: Summary, + max_frame_ms: f64, cpu_ms_avg: f64, gpu_ms: GpuSummary, draw_calls_avg: f64, @@ -104,7 +116,7 @@ pub const BenchmarkRunner = struct { player.target_block = null; } - pub fn recordFrame(self: *BenchmarkRunner, dt: f32, fps: f32, gpu: GpuTimingResults, world_stats: ?WorldStats, draw_calls: u32) !void { + pub fn recordFrame(self: *BenchmarkRunner, dt: f32, fps: f32, gpu: GpuTimingResults, world_stats: ?WorldStats, draw_calls: u32, gpu_memory_mb: f32) !void { const shadow_avg = averageArray(&gpu.shadow_pass_ms); const chunks_rendered = if (world_stats) |ws| ws.chunks_rendered else 0; const vertices = if (world_stats) |ws| ws.vertices_rendered else 0; @@ -119,6 +131,7 @@ pub const BenchmarkRunner = struct { .draw_calls = draw_calls, .vertices = vertices, .chunks_rendered = chunks_rendered, + .gpu_memory_mb = gpu_memory_mb, }); self.elapsed_s += dt; } @@ -139,6 +152,8 @@ pub const BenchmarkRunner = struct { var file = try fs.cwd().createFile(self.output_path, .{ .truncate = true }); defer file.close(); try file.writeAll(json); + + try enforceSlo(results); } pub fn makeResults(self: *const BenchmarkRunner) !BenchmarkResults { @@ -152,15 +167,21 @@ pub const BenchmarkRunner = struct { var draw_sum: f64 = 0; var vertices_sum: f64 = 0; var chunks_sum: f64 = 0; + var memory_sum: f64 = 0; + var memory_max: f64 = 0; + var max_frame_ms: f64 = 0; for (self.samples.items) |sample| { cpu_sum += sample.cpu_ms; + max_frame_ms = @max(max_frame_ms, sample.cpu_ms); shadow_sum += sample.gpu_shadow_ms; opaque_sum += sample.gpu_opaque_ms; total_sum += sample.gpu_total_ms; draw_sum += @floatFromInt(sample.draw_calls); vertices_sum += @floatFromInt(sample.vertices); chunks_sum += @floatFromInt(sample.chunks_rendered); + memory_sum += sample.gpu_memory_mb; + memory_max = @max(memory_max, sample.gpu_memory_mb); } const count = @as(f64, @floatFromInt(@max(self.samples.items.len, 1))); @@ -173,9 +194,12 @@ pub const BenchmarkRunner = struct { return .{ .preset = self.preset, .render_distance = self.render_distance, + .gpu_memory_mb_avg = memory_sum / count, + .gpu_memory_mb_max = memory_max, .frames = @intCast(self.samples.items.len), .duration_s = self.duration_s, .fps = fps_summary, + .max_frame_ms = max_frame_ms, .cpu_ms_avg = cpu_sum / count, .gpu_ms = .{ .shadow_avg = shadow_sum / count, @@ -201,6 +225,43 @@ pub const BenchmarkRunner = struct { } }; +pub fn thresholdsForPreset(preset: []const u8) SloThresholds { + if (std.ascii.eqlIgnoreCase(preset, "low")) return .{ .fps_p1_min = 12, .max_frame_ms = 260, .draw_calls_max = 700, .vertices_max = 3_500_000, .gpu_memory_mb_max = 1800 }; + if (std.ascii.eqlIgnoreCase(preset, "medium")) return .{ .fps_p1_min = 8, .max_frame_ms = 260, .draw_calls_max = 2600, .vertices_max = 6_000_000, .gpu_memory_mb_max = 2400 }; + if (std.ascii.eqlIgnoreCase(preset, "high")) return .{ .fps_p1_min = 6, .max_frame_ms = 260, .draw_calls_max = 3600, .vertices_max = 8_500_000, .gpu_memory_mb_max = 2800 }; + if (std.ascii.eqlIgnoreCase(preset, "ultra")) return .{ .fps_p1_min = 4, .max_frame_ms = 260, .draw_calls_max = 4500, .vertices_max = 12_000_000, .gpu_memory_mb_max = 3400 }; + if (std.ascii.eqlIgnoreCase(preset, "extreme")) return .{ .fps_p1_min = 3, .max_frame_ms = 260, .draw_calls_max = 5500, .vertices_max = 16_000_000, .gpu_memory_mb_max = 4096 }; + return .{ .fps_p1_min = 6, .max_frame_ms = 260, .draw_calls_max = 3600, .vertices_max = 8_500_000, .gpu_memory_mb_max = 2800 }; +} + +fn enforceSlo(results: BenchmarkResults) !void { + const thresholds = thresholdsForPreset(results.preset); + var failed = false; + + if (results.fps.p1 < thresholds.fps_p1_min) { + std.log.err("benchmark SLO breach: {s} p1 FPS {d:.2} < {d:.2}", .{ results.preset, results.fps.p1, thresholds.fps_p1_min }); + failed = true; + } + if (results.max_frame_ms > thresholds.max_frame_ms) { + std.log.err("benchmark SLO breach: {s} max frame {d:.2}ms > {d:.2}ms", .{ results.preset, results.max_frame_ms, thresholds.max_frame_ms }); + failed = true; + } + if (results.draw_calls_avg > thresholds.draw_calls_max) { + std.log.err("benchmark SLO breach: {s} draw calls avg {d:.2} > {d:.2}", .{ results.preset, results.draw_calls_avg, thresholds.draw_calls_max }); + failed = true; + } + if (results.vertices_avg > thresholds.vertices_max) { + std.log.err("benchmark SLO breach: {s} vertices avg {d:.2} > {d:.2}", .{ results.preset, results.vertices_avg, thresholds.vertices_max }); + failed = true; + } + if (results.gpu_memory_mb_max > thresholds.gpu_memory_mb_max) { + std.log.err("benchmark SLO breach: {s} GPU memory max {d:.2}MB > {d:.2}MB", .{ results.preset, results.gpu_memory_mb_max, thresholds.gpu_memory_mb_max }); + failed = true; + } + + if (failed) return error.BenchmarkSloBreach; +} + fn fpsField(sample: FrameSample) f32 { return sample.fps; } diff --git a/scripts/check_spirv_sizes.sh b/scripts/check_spirv_sizes.sh new file mode 100755 index 00000000..0dec074c --- /dev/null +++ b/scripts/check_spirv_sizes.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +set -euo pipefail + +baseline=${1:-docs/shaders/spirv-sizes.json} +threshold_percent=${SPIRV_SIZE_REGRESSION_THRESHOLD_PERCENT:-10} +update_baseline=${SPIRV_UPDATE_BASELINE:-0} + +mkdir -p "$(dirname "$baseline")" +if [[ ! -f "$baseline" ]]; then + printf '{\n "threshold_percent": %s,\n "shaders": {}\n}\n' "$threshold_percent" > "$baseline" +fi + +tmp_dir=$(mktemp -d) +current=$(mktemp) +updated=$(mktemp) +trap 'rm -rf "$tmp_dir" "$current" "$updated"' EXIT + +jq -n --argjson threshold "$threshold_percent" '{threshold_percent: $threshold, shaders: {}}' > "$current" + +failed=0 +updated_baseline=0 + +while IFS= read -r shader; do + output="$tmp_dir/$(basename "$shader").spv" + start_ns=$(date +%s%N) + glslangValidator -V "$shader" -o "$output" >/dev/null + end_ns=$(date +%s%N) + size=$(stat -c '%s' "$output") + compile_ms=$(( (end_ns - start_ns) / 1000000 )) + + printf 'SPIR-V size: %s %s bytes (%sms compile)\n' "$shader" "$size" "$compile_ms" + jq --arg shader "$shader" --argjson size "$size" '.shaders[$shader] = $size' "$current" > "$updated" + mv "$updated" "$current" + + baseline_size=$(jq -r --arg shader "$shader" '.shaders[$shader] // empty' "$baseline") + if [[ -z "$baseline_size" ]]; then + if [[ "$update_baseline" == "1" ]]; then + printf 'SPIR-V baseline updated: new shader %s = %s bytes\n' "$shader" "$size" + jq --arg shader "$shader" --argjson size "$size" '.shaders[$shader] = $size' "$baseline" > "$updated" + mv "$updated" "$baseline" + updated_baseline=1 + else + printf 'SPIR-V baseline missing for new shader %s (%s bytes). Run scripts/update_spirv_baseline.sh and commit the updated baseline.\n' "$shader" "$size" >&2 + failed=1 + fi + continue + fi + + max_size=$(awk -v base="$baseline_size" -v threshold="$threshold_percent" 'BEGIN { printf "%d", int(base * (1 + threshold / 100.0) + 0.999999) }') + if (( size > max_size )); then + increase=$(awk -v base="$baseline_size" -v size="$size" 'BEGIN { printf "%.2f", ((size - base) / base) * 100.0 }') + printf 'SPIR-V size regression: %s grew from %s to %s bytes (%s%% > %s%%)\n' "$shader" "$baseline_size" "$size" "$increase" "$threshold_percent" >&2 + failed=1 + fi +done < <(find assets/shaders/vulkan -maxdepth 1 \( -name '*.vert' -o -name '*.frag' -o -name '*.comp' \) | sort) + +if [[ "$updated_baseline" -eq 1 ]]; then + jq '.shaders |= (to_entries | sort_by(.key) | from_entries)' "$baseline" > "$updated" + mv "$updated" "$baseline" +fi + +if [[ "$failed" -ne 0 ]]; then + exit 1 +fi diff --git a/scripts/compare_visual_golden.sh b/scripts/compare_visual_golden.sh new file mode 100755 index 00000000..0ebe0958 --- /dev/null +++ b/scripts/compare_visual_golden.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $# -lt 2 ]]; then + printf 'Usage: %s actual.png golden.png [diff.png]\n' "$0" >&2 + exit 2 +fi + +actual=$1 +golden=$2 +diff=${3:-visual-diff.png} +tolerance=${VISUAL_DIFF_RMSE_TOLERANCE:-0.015} + +if [[ ! -f "$actual" ]]; then + printf 'Actual screenshot missing: %s\n' "$actual" >&2 + exit 2 +fi + +if [[ ! -f "$golden" ]]; then + printf 'Golden screenshot missing: %s\n' "$golden" >&2 + exit 2 +fi + +metric_output=$(magick compare -metric RMSE "$golden" "$actual" "$diff" 2>&1 || true) +normalized=$(printf '%s\n' "$metric_output" | sed -n 's/.*(\([0-9.]*\)).*/\1/p') +if [[ -z "$normalized" ]]; then + normalized=1 +fi + +printf 'Visual RMSE: %s (tolerance %s)\n' "$normalized" "$tolerance" + +if awk -v value="$normalized" -v tolerance="$tolerance" 'BEGIN { exit !(value > tolerance) }'; then + printf 'Visual golden diff exceeds tolerance: %s > %s\n' "$normalized" "$tolerance" >&2 + exit 1 +fi diff --git a/scripts/update_spirv_baseline.sh b/scripts/update_spirv_baseline.sh new file mode 100755 index 00000000..f1f1c0e0 --- /dev/null +++ b/scripts/update_spirv_baseline.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -euo pipefail + +SPIRV_UPDATE_BASELINE=1 bash scripts/check_spirv_sizes.sh "${1:-docs/shaders/spirv-sizes.json}" diff --git a/src/game/app.zig b/src/game/app.zig index 54e2e8a7..01ebd06d 100644 --- a/src/game/app.zig +++ b/src/game/app.zig @@ -442,8 +442,10 @@ pub const App = struct { if (build_options.benchmark) { if (self.benchmark_runner) |runner| { const gpu_timing = self.render_system.getRHI().timing().getTimingResults(); - const draw_calls = self.render_system.getRHI().query().getDrawCallCount(); - try runner.recordFrame(self.time.delta_time, self.time.fps, gpu_timing, world_stats, draw_calls); + const rhi = self.render_system.getRHI(); + const draw_calls = rhi.query().getDrawCallCount(); + const gpu_memory_mb = if (rhi.device) |device| gpuMemoryMb(device.getStats()) else 0; + try runner.recordFrame(self.time.delta_time, self.time.fps, gpu_timing, world_stats, draw_calls, gpu_memory_mb); if (runner.isComplete()) { try runner.writeResults(); @@ -534,6 +536,11 @@ fn applyBenchmarkPreset(settings: *Settings, preset_name: []const u8) void { } } +fn gpuMemoryMb(stats: @import("engine-rhi").Stats) f32 { + const bytes = stats.total_buffer_memory + stats.total_texture_memory; + return @as(f32, @floatFromInt(bytes)) / (1024.0 * 1024.0); +} + fn applyNamedPreset(settings: *Settings, preset_name: []const u8, label: []const u8) bool { if (json_presets.findAndApplyNamed(settings, preset_name)) |applied_name| { log.log.info("{s}: Applied graphics preset '{s}'", .{ label, applied_name });