From 37158c27dfcfa8abaf67e1b22b701eb0585165e5 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Mon, 8 Jun 2026 15:02:33 +0000 Subject: [PATCH 1/5] chore(llama-cpp): bump to 8f83d6c for mtmd video input support Signed-off-by: Ettore Di Giacinto --- backend/cpp/llama-cpp/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/cpp/llama-cpp/Makefile b/backend/cpp/llama-cpp/Makefile index a33f1ceaa68c..3e84ab59bac7 100644 --- a/backend/cpp/llama-cpp/Makefile +++ b/backend/cpp/llama-cpp/Makefile @@ -1,5 +1,5 @@ -LLAMA_VERSION?=9e3b928fd8c9d14dbf15a8768b9fdd7e5c721d66 +LLAMA_VERSION?=8f83d6c271d194bde2d410145a0ce73bc42e85cd LLAMA_REPO?=https://github.com/ggerganov/llama.cpp CMAKE_ARGS?= From 35ff9358451a7f1fd74152512d3edb66bbcb8a70 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Mon, 8 Jun 2026 15:06:51 +0000 Subject: [PATCH 2/5] feat(llama-cpp): forward video input to mtmd (template + non-template paths) Wire request->videos() into grpc-server.cpp mirroring the existing image and audio handling: a video_data build + non-template files extraction, and input_video chat chunks on the tokenizer-template path. allow_video is auto-set at model load by the vendored upstream chat_params. Signed-off-by: Ettore Di Giacinto --- backend/cpp/llama-cpp/grpc-server.cpp | 73 ++++++++++++++++++++++++++- 1 file changed, 71 insertions(+), 2 deletions(-) diff --git a/backend/cpp/llama-cpp/grpc-server.cpp b/backend/cpp/llama-cpp/grpc-server.cpp index 90a5477a9726..9ec806cd1a7d 100644 --- a/backend/cpp/llama-cpp/grpc-server.cpp +++ b/backend/cpp/llama-cpp/grpc-server.cpp @@ -381,6 +381,15 @@ json parse_options(bool streaming, const backend::PredictOptions* predict, const }); } + // for each video in the request, add the video data + for (int i = 0; i < predict->videos_size(); i++) { + data["video_data"].push_back(json + { + {"id", i}, + {"data", predict->videos(i)}, + }); + } + data["stop"] = predict->stopprompts(); // data["n_probs"] = predict->nprobs(); //TODO: images, @@ -1503,7 +1512,7 @@ class BackendServiceImpl final : public backend::Backend::Service { msg_json["role"] = msg.role(); bool is_last_user_msg = (i == last_user_msg_idx); - bool has_images_or_audio = (request->images_size() > 0 || request->audios_size() > 0); + bool has_images_or_audio = (request->images_size() > 0 || request->audios_size() > 0 || request->videos_size() > 0); // Handle content - can be string, null, or array // For multimodal content, we'll embed images/audio from separate fields @@ -1554,6 +1563,16 @@ class BackendServiceImpl final : public backend::Backend::Service { content_array.push_back(audio_chunk); } } + if (request->videos_size() > 0) { + for (int j = 0; j < request->videos_size(); j++) { + json video_chunk; + video_chunk["type"] = "input_video"; + json input_video; + input_video["data"] = request->videos(j); + video_chunk["input_video"] = input_video; + content_array.push_back(video_chunk); + } + } msg_json["content"] = content_array; } else { // Use content as-is (already array or not last user message) @@ -1588,6 +1607,16 @@ class BackendServiceImpl final : public backend::Backend::Service { content_array.push_back(audio_chunk); } } + if (request->videos_size() > 0) { + for (int j = 0; j < request->videos_size(); j++) { + json video_chunk; + video_chunk["type"] = "input_video"; + json input_video; + input_video["data"] = request->videos(j); + video_chunk["input_video"] = input_video; + content_array.push_back(video_chunk); + } + } msg_json["content"] = content_array; } else if (msg.role() == "tool") { // Tool role messages must have content field set, even if empty @@ -2039,6 +2068,16 @@ class BackendServiceImpl final : public backend::Backend::Service { files.push_back(decoded_data); } } + + const auto &video_data = data.find("video_data"); + if (video_data != data.end() && video_data->is_array()) + { + for (const auto &video : *video_data) + { + auto decoded_data = base64_decode(video["data"].get()); + files.push_back(decoded_data); + } + } } const bool has_mtmd = ctx_server.impl->mctx != nullptr; @@ -2291,7 +2330,7 @@ class BackendServiceImpl final : public backend::Backend::Service { } bool is_last_user_msg = (i == last_user_msg_idx); - bool has_images_or_audio = (request->images_size() > 0 || request->audios_size() > 0); + bool has_images_or_audio = (request->images_size() > 0 || request->audios_size() > 0 || request->videos_size() > 0); // Handle content - can be string, null, or array // For multimodal content, we'll embed images/audio from separate fields @@ -2344,6 +2383,16 @@ class BackendServiceImpl final : public backend::Backend::Service { content_array.push_back(audio_chunk); } } + if (request->videos_size() > 0) { + for (int j = 0; j < request->videos_size(); j++) { + json video_chunk; + video_chunk["type"] = "input_video"; + json input_video; + input_video["data"] = request->videos(j); + video_chunk["input_video"] = input_video; + content_array.push_back(video_chunk); + } + } msg_json["content"] = content_array; } else { // Use content as-is (already array or not last user message) @@ -2383,6 +2432,16 @@ class BackendServiceImpl final : public backend::Backend::Service { content_array.push_back(audio_chunk); } } + if (request->videos_size() > 0) { + for (int j = 0; j < request->videos_size(); j++) { + json video_chunk; + video_chunk["type"] = "input_video"; + json input_video; + input_video["data"] = request->videos(j); + video_chunk["input_video"] = input_video; + content_array.push_back(video_chunk); + } + } msg_json["content"] = content_array; SRV_INF("[CONTENT DEBUG] Predict: Message %d created content array with media\n", i); } else if (!msg.tool_calls().empty()) { @@ -2845,6 +2904,16 @@ class BackendServiceImpl final : public backend::Backend::Service { files.push_back(decoded_data); } } + + const auto &video_data = data.find("video_data"); + if (video_data != data.end() && video_data->is_array()) + { + for (const auto &video : *video_data) + { + auto decoded_data = base64_decode(video["data"].get()); + files.push_back(decoded_data); + } + } } // process files From 23c5c3f7616b27da0af1259fd00149f374f12bcd Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Mon, 8 Jun 2026 15:11:46 +0000 Subject: [PATCH 3/5] feat(ui): add video attachment support to the chat UI Mirror the image/audio attachment path for video: emit video_url content parts, accept video/* in the picker, keep video files as base64, show a film icon badge, and render attached video inline with a