fix(vllm-cpp): map the CUDA 12 capabilities at the CPU build

backend/go/vllm-cpp/README.md

@@ -35,6 +35,29 @@ the Makefile therefore means updating `abiVersion` plus the mirrors (and their
 offsets in `vllmcpp_test.go`) in the same change; `make abi-check` compares the
 pinned header against the bindings and the library build runs it first.
 
+## Hardware coverage
+
+The CUDA builds require the CUDA 13 toolchain and target Blackwell only:
+`sm_120a` + `sm_121a` on x86_64, `sm_121a` (GB10 / DGX Spark) on arm64. CUDA
+12.x nvcc cannot compile the Blackwell fp4 kernels, so no CUDA 12 variant is
+shipped and `backend/index.yaml` maps the `nvidia-cuda-12` /
+`nvidia-l4t-cuda-12` capabilities at the CPU build. Practically:
+
+| Host | Installed build |
+|---|---|
+| x86_64 + CUDA 13 | `cuda13-vllm-cpp` |
+| DGX Spark / GB10 (JetPack 7, CUDA 13) | `nvidia-l4t-arm64-vllm-cpp` |
+| Jetson AGX Orin (sm_87, JetPack 6, CUDA 12) | `cpu-vllm-cpp` |
+| Apple Silicon | `metal-vllm-cpp` |
+| Anything else | `vulkan-vllm-cpp` or `cpu-vllm-cpp` |
+
+The capability a host reports comes from `/run/localai/capability` inside the
+LocalAI container, which the image bakes in at build time (see `Dockerfile`).
+A DGX Spark running the CUDA 12 `-nvidia-l4t-arm64` image therefore reports
+`nvidia-l4t-cuda-12` and gets the CPU build; use the `-nvidia-l4t-arm64-cuda-13`
+image, or set `LOCALAI_FORCE_META_BACKEND_CAPABILITY=nvidia-l4t-cuda-13`, to
+get the GPU one.
+
 Model config example:
 
 ```yaml

backend/index.yaml

@@ -207,6 +207,11 @@
     decoding and KV offload, and runs on CPU, NVIDIA CUDA (Blackwell-family), Apple
     Metal and Vulkan.
 
+    The CUDA builds require the CUDA 13 toolchain and target Blackwell only: sm_120a
+    plus sm_121a on x86_64, and sm_121a (GB10 / DGX Spark) on arm64. Older NVIDIA
+    hardware and CUDA 12 hosts - including Jetson AGX Orin (sm_87, JetPack 6) - run
+    the CPU build instead.
+
     The project is expected to be renamed as it diverges further from vLLM; the new
     name is still to be decided.
   urls:
@@ -226,6 +231,12 @@
     nvidia-cuda-13: "cuda13-vllm-cpp"
     nvidia-l4t: "nvidia-l4t-arm64-vllm-cpp"
     nvidia-l4t-cuda-13: "nvidia-l4t-arm64-vllm-cpp"
+    # No CUDA 12 variant exists: 12.x nvcc cannot compile the Blackwell fp4
+    # kernels, so those hosts run the CPU build. Mapped explicitly rather than
+    # left to the "default" catch-all so the fallback is visible here instead
+    # of looking like an oversight.
+    nvidia-cuda-12: "cpu-vllm-cpp"
+    nvidia-l4t-cuda-12: "cpu-vllm-cpp"
 - !!merge <<: *vllm-cpp
   name: "vllm-cpp-development"
   capabilities:
@@ -236,6 +247,8 @@
     nvidia-cuda-13: "cuda13-vllm-cpp-development"
     nvidia-l4t: "nvidia-l4t-arm64-vllm-cpp-development"
     nvidia-l4t-cuda-13: "nvidia-l4t-arm64-vllm-cpp-development"
+    nvidia-cuda-12: "cpu-vllm-cpp-development"
+    nvidia-l4t-cuda-12: "cpu-vllm-cpp-development"
 - &crispasr
   name: "crispasr"
   alias: "crispasr"
@@ -512,7 +525,7 @@
   capabilities:
     nvidia: "cuda12-rfdetr"
     intel: "intel-rfdetr"
     #amd: "rocm-rfdetr"
     nvidia-l4t: "nvidia-l4t-arm64-rfdetr"
     metal: "metal-rfdetr"
     default: "cpu-rfdetr"
@@ -4181,7 +4194,7 @@
   capabilities:
     nvidia: "cuda12-rfdetr-development"
     intel: "intel-rfdetr-development"
     #amd: "rocm-rfdetr-development"
     nvidia-l4t: "nvidia-l4t-arm64-rfdetr-development"
     metal: "metal-rfdetr-development"
     default: "cpu-rfdetr-development"

core/gallery/backend_index_capabilities_test.go

@@ -0,0 +1,83 @@
+package gallery_test
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"gopkg.in/yaml.v3"
+
+	"github.com/mudler/LocalAI/core/gallery"
+	"github.com/mudler/LocalAI/pkg/system"
+)
+
+// loadBackendIndex parses backend/index.yaml once for the whole suite.
+var loadBackendIndex = sync.OnceValues(func() (gallery.GalleryElements[*gallery.GalleryBackend], error) {
+	data, err := os.ReadFile(filepath.Join("..", "..", "backend", "index.yaml"))
+	if err != nil {
+		return nil, err
+	}
+	var entries gallery.GalleryElements[*gallery.GalleryBackend]
+	if err := yaml.Unmarshal(data, &entries); err != nil {
+		return nil, err
+	}
+	return entries, nil
+})
+
+var _ = Describe("backend/index.yaml capability maps", func() {
+	var entries gallery.GalleryElements[*gallery.GalleryBackend]
+
+	BeforeEach(func() {
+		var err error
+		entries, err = loadBackendIndex()
+		Expect(err).ToNot(HaveOccurred())
+		Expect(entries).ToNot(BeEmpty())
+	})
+
+	// A capability pointing at a name that does not exist is invisible until a
+	// host with exactly that capability tries to install: FindBestBackendFromMeta
+	// returns nil and the install fails with "no backend found".
+	It("resolves every capability reference to an entry in the index", func() {
+		names := map[string]struct{}{}
+		for _, e := range entries {
+			names[e.Name] = struct{}{}
+		}
+
+		dangling := []string{}
+		for _, e := range entries {
+			for capability, target := range e.CapabilitiesMap {
+				if _, ok := names[target]; !ok {
+					dangling = append(dangling, fmt.Sprintf("  %s -> %s: %q", e.Name, capability, target))
+				}
+			}
+		}
+		Expect(dangling).To(BeEmpty(), "capabilities naming a missing entry:\n%s", strings.Join(dangling, "\n"))
+	})
+
+	// vllm.cpp's CUDA kernels need the CUDA 13 toolchain (12.x nvcc cannot
+	// compile the Blackwell fp4 paths), so CUDA 12 hosts have no GPU build to
+	// install and must land on the CPU one. Assert the fallback is explicit
+	// rather than an accident of the "default" catch-all, so mapping these
+	// capabilities at a CUDA image later is a test failure and not a host that
+	// pulls kernels it cannot run.
+	DescribeTable("routes vllm-cpp hosts to the build their toolchain supports",
+		func(metaName, capability, expected string) {
+			meta := entries.FindByName(metaName)
+			Expect(meta).ToNot(BeNil())
+
+			resolved := meta.FindBestBackendFromMeta(system.NewCapabilityState(capability), entries)
+			Expect(resolved).ToNot(BeNil())
+			Expect(resolved.Name).To(Equal(expected))
+		},
+		Entry("CUDA 12 x86_64 gets the CPU build", "vllm-cpp", "nvidia-cuda-12", "cpu-vllm-cpp"),
+		Entry("CUDA 12 Jetson (AGX Orin) gets the CPU build", "vllm-cpp", "nvidia-l4t-cuda-12", "cpu-vllm-cpp"),
+		Entry("CUDA 13 Jetson (DGX Spark) gets the L4T build", "vllm-cpp", "nvidia-l4t-cuda-13", "nvidia-l4t-arm64-vllm-cpp"),
+		Entry("CUDA 13 x86_64 gets the CUDA build", "vllm-cpp", "nvidia-cuda-13", "cuda13-vllm-cpp"),
+		Entry("development CUDA 12 Jetson gets the CPU build", "vllm-cpp-development", "nvidia-l4t-cuda-12", "cpu-vllm-cpp-development"),
+		Entry("development CUDA 13 Jetson gets the L4T build", "vllm-cpp-development", "nvidia-l4t-cuda-13", "nvidia-l4t-arm64-vllm-cpp-development"),
+	)
+})