From 8c5f8e9e14702fdeda0cd54662e592014b9f8f53 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sat, 30 May 2026 22:49:31 +0000 Subject: [PATCH 01/29] feat(crispasr): backend source files (Go gRPC server, C-ABI shim, build files) Signed-off-by: Ettore Di Giacinto --- backend/go/crispasr/.gitignore | 5 + backend/go/crispasr/CMakeLists.txt | 19 ++ backend/go/crispasr/Makefile | 125 +++++++ backend/go/crispasr/cpp/crispasr_shim.cpp | 209 ++++++++++++ backend/go/crispasr/cpp/crispasr_shim.h | 30 ++ backend/go/crispasr/gocrispasr.go | 395 ++++++++++++++++++++++ backend/go/crispasr/main.go | 58 ++++ backend/go/crispasr/package.sh | 65 ++++ backend/go/crispasr/run.sh | 52 +++ 9 files changed, 958 insertions(+) create mode 100644 backend/go/crispasr/.gitignore create mode 100644 backend/go/crispasr/CMakeLists.txt create mode 100644 backend/go/crispasr/Makefile create mode 100644 backend/go/crispasr/cpp/crispasr_shim.cpp create mode 100644 backend/go/crispasr/cpp/crispasr_shim.h create mode 100644 backend/go/crispasr/gocrispasr.go create mode 100644 backend/go/crispasr/main.go create mode 100755 backend/go/crispasr/package.sh create mode 100755 backend/go/crispasr/run.sh diff --git a/backend/go/crispasr/.gitignore b/backend/go/crispasr/.gitignore new file mode 100644 index 000000000000..537510174736 --- /dev/null +++ b/backend/go/crispasr/.gitignore @@ -0,0 +1,5 @@ +sources +build* +libgocrispasr*.so +crispasr +package diff --git a/backend/go/crispasr/CMakeLists.txt b/backend/go/crispasr/CMakeLists.txt new file mode 100644 index 000000000000..d557ee981552 --- /dev/null +++ b/backend/go/crispasr/CMakeLists.txt @@ -0,0 +1,19 @@ +cmake_minimum_required(VERSION 3.12) +project(gocrispasr LANGUAGES C CXX) +set(CMAKE_POSITION_INDEPENDENT_CODE ON) +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) + +add_subdirectory(./sources/CrispASR) + +add_library(gocrispasr MODULE cpp/crispasr_shim.cpp) +target_include_directories(gocrispasr PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/sources/CrispASR/include + ${CMAKE_CURRENT_SOURCE_DIR}/sources/CrispASR/ggml/include) +target_link_libraries(gocrispasr PRIVATE crispasr ggml) + +if(CMAKE_CXX_COMPILER_ID MATCHES "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 9.0) + target_link_libraries(gocrispasr PRIVATE stdc++fs) +endif() + +set_property(TARGET gocrispasr PROPERTY CXX_STANDARD 17) +set_target_properties(gocrispasr PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}) diff --git a/backend/go/crispasr/Makefile b/backend/go/crispasr/Makefile new file mode 100644 index 000000000000..ede5530c5641 --- /dev/null +++ b/backend/go/crispasr/Makefile @@ -0,0 +1,125 @@ +CMAKE_ARGS?= +BUILD_TYPE?= +NATIVE?=false + +GOCMD?=go +GO_TAGS?= +JOBS?=$(shell nproc --ignore=1) + +# CrispASR version (release tag) +CRISPASR_REPO?=https://github.com/CrispStrobe/CrispASR +CRISPASR_VERSION?=v0.6.11 +SO_TARGET?=libgocrispasr.so + +CMAKE_ARGS+=-DBUILD_SHARED_LIBS=OFF +# Keep the build lean: no tests/examples/server/SDL2/curl/ffmpeg (the FROM scratch +# image cannot satisfy those runtime deps). All ASR/TTS model backends stay enabled. +CMAKE_ARGS+=-DCRISPASR_BUILD_TESTS=OFF -DCRISPASR_BUILD_EXAMPLES=OFF -DCRISPASR_BUILD_SERVER=OFF +CMAKE_ARGS+=-DCRISPASR_SDL2=OFF -DCRISPASR_CURL=OFF -DCRISPASR_FFMPEG=OFF + +ifeq ($(NATIVE),false) + CMAKE_ARGS+=-DGGML_NATIVE=OFF +endif + +ifeq ($(BUILD_TYPE),cublas) + CMAKE_ARGS+=-DGGML_CUDA=ON +else ifeq ($(BUILD_TYPE),openblas) + CMAKE_ARGS+=-DGGML_BLAS=ON -DGGML_BLAS_VENDOR=OpenBLAS +else ifeq ($(BUILD_TYPE),clblas) + CMAKE_ARGS+=-DGGML_CLBLAST=ON -DCLBlast_DIR=/some/path +else ifeq ($(BUILD_TYPE),hipblas) + CMAKE_ARGS+=-DGGML_HIPBLAS=ON +else ifeq ($(BUILD_TYPE),vulkan) + CMAKE_ARGS+=-DGGML_VULKAN=ON +else ifeq ($(OS),Darwin) + ifneq ($(BUILD_TYPE),metal) + CMAKE_ARGS+=-DGGML_METAL=OFF + else + CMAKE_ARGS+=-DGGML_METAL=ON + CMAKE_ARGS+=-DGGML_METAL_EMBED_LIBRARY=ON + endif +endif + +ifeq ($(BUILD_TYPE),sycl_f16) + CMAKE_ARGS+=-DGGML_SYCL=ON \ + -DCMAKE_C_COMPILER=icx \ + -DCMAKE_CXX_COMPILER=icpx \ + -DGGML_SYCL_F16=ON +endif + +ifeq ($(BUILD_TYPE),sycl_f32) + CMAKE_ARGS+=-DGGML_SYCL=ON \ + -DCMAKE_C_COMPILER=icx \ + -DCMAKE_CXX_COMPILER=icpx +endif + +sources/CrispASR: + mkdir -p sources/CrispASR + cd sources/CrispASR && \ + git init && \ + git remote add origin $(CRISPASR_REPO) && \ + git fetch origin && \ + git checkout $(CRISPASR_VERSION) && \ + git submodule update --init --recursive --depth 1 --single-branch + +# Detect OS +UNAME_S := $(shell uname -s) + +ifeq ($(UNAME_S),Linux) + VARIANT_TARGETS = libgocrispasr-avx.so libgocrispasr-avx2.so libgocrispasr-avx512.so libgocrispasr-fallback.so +else + VARIANT_TARGETS = libgocrispasr-fallback.so +endif + +crispasr: main.go gocrispasr.go $(VARIANT_TARGETS) + CGO_ENABLED=0 $(GOCMD) build -tags "$(GO_TAGS)" -o crispasr ./ + +package: crispasr + bash package.sh + +build: package + +clean: purge + rm -rf libgocrispasr*.so package sources/CrispASR crispasr + +purge: + rm -rf build* + +ifeq ($(UNAME_S),Linux) +libgocrispasr-avx.so: sources/CrispASR + $(MAKE) purge + $(info ${GREEN}I crispasr build info:avx${RESET}) + SO_TARGET=libgocrispasr-avx.so CMAKE_ARGS="$(CMAKE_ARGS) -DGGML_AVX=on -DGGML_AVX2=off -DGGML_AVX512=off -DGGML_FMA=off -DGGML_F16C=off -DGGML_BMI2=off" $(MAKE) libgocrispasr-custom + rm -rfv build* + +libgocrispasr-avx2.so: sources/CrispASR + $(MAKE) purge + $(info ${GREEN}I crispasr build info:avx2${RESET}) + SO_TARGET=libgocrispasr-avx2.so CMAKE_ARGS="$(CMAKE_ARGS) -DGGML_AVX=on -DGGML_AVX2=on -DGGML_AVX512=off -DGGML_FMA=on -DGGML_F16C=on -DGGML_BMI2=on" $(MAKE) libgocrispasr-custom + rm -rfv build* + +libgocrispasr-avx512.so: sources/CrispASR + $(MAKE) purge + $(info ${GREEN}I crispasr build info:avx512${RESET}) + SO_TARGET=libgocrispasr-avx512.so CMAKE_ARGS="$(CMAKE_ARGS) -DGGML_AVX=on -DGGML_AVX2=on -DGGML_AVX512=on -DGGML_FMA=on -DGGML_F16C=on -DGGML_BMI2=on" $(MAKE) libgocrispasr-custom + rm -rfv build* +endif + +libgocrispasr-fallback.so: sources/CrispASR + $(MAKE) purge + $(info ${GREEN}I crispasr build info:fallback${RESET}) + SO_TARGET=libgocrispasr-fallback.so CMAKE_ARGS="$(CMAKE_ARGS) -DGGML_AVX=off -DGGML_AVX2=off -DGGML_AVX512=off -DGGML_FMA=off -DGGML_F16C=off -DGGML_BMI2=off" $(MAKE) libgocrispasr-custom + rm -rfv build* + +libgocrispasr-custom: CMakeLists.txt cpp/crispasr_shim.cpp cpp/crispasr_shim.h + mkdir -p build-$(SO_TARGET) && \ + cd build-$(SO_TARGET) && \ + cmake .. $(CMAKE_ARGS) && \ + cmake --build . --config Release -j$(JOBS) && \ + cd .. && \ + mv build-$(SO_TARGET)/libgocrispasr.so ./$(SO_TARGET) + +test: crispasr + CGO_ENABLED=0 $(GOCMD) test -v ./... + +all: crispasr package diff --git a/backend/go/crispasr/cpp/crispasr_shim.cpp b/backend/go/crispasr/cpp/crispasr_shim.cpp new file mode 100644 index 000000000000..2222f38a3c8e --- /dev/null +++ b/backend/go/crispasr/cpp/crispasr_shim.cpp @@ -0,0 +1,209 @@ +#include "crispasr_shim.h" +#include "ggml-backend.h" +#include "crispasr.h" +#include +#include + +static struct whisper_vad_context *vctx; +static struct whisper_context *ctx; +static std::vector flat_segs; + +static std::atomic g_abort{0}; + +static std::atomic g_go_new_segment_cb{0}; +static std::atomic g_go_new_segment_user_data{0}; + +static bool abort_cb(void * /*user_data*/) { + return g_abort.load(std::memory_order_relaxed) != 0; +} + +static void new_segment_cb(struct whisper_context *cb_ctx, + struct whisper_state * /*state*/, int n_new, + void * /*user_data*/) { + uintptr_t go_cb = g_go_new_segment_cb.load(std::memory_order_relaxed); + if (go_cb == 0) { + return; + } + int total = whisper_full_n_segments(cb_ctx); + int idx_first = total - n_new; + if (idx_first < 0) { + idx_first = 0; + } + uintptr_t ud = g_go_new_segment_user_data.load(std::memory_order_relaxed); + reinterpret_cast(go_cb)(idx_first, n_new, ud); +} + +extern "C" void set_abort(int v) { + g_abort.store(v, std::memory_order_relaxed); +} + +extern "C" void set_new_segment_callback(uintptr_t cb_ptr, uintptr_t user_data) { + g_go_new_segment_cb.store(cb_ptr, std::memory_order_relaxed); + g_go_new_segment_user_data.store(user_data, std::memory_order_relaxed); +} + +static void ggml_log_cb(enum ggml_log_level level, const char *log, + void *data) { + const char *level_str; + + if (!log) { + return; + } + + switch (level) { + case GGML_LOG_LEVEL_DEBUG: + level_str = "DEBUG"; + break; + case GGML_LOG_LEVEL_INFO: + level_str = "INFO"; + break; + case GGML_LOG_LEVEL_WARN: + level_str = "WARN"; + break; + case GGML_LOG_LEVEL_ERROR: + level_str = "ERROR"; + break; + default: /* Potential future-proofing */ + level_str = "?????"; + break; + } + + fprintf(stderr, "[%-5s] ", level_str); + fputs(log, stderr); + fflush(stderr); +} + +int load_model(const char *const model_path) { + whisper_log_set(ggml_log_cb, nullptr); + ggml_backend_load_all(); + + struct whisper_context_params cparams = whisper_context_default_params(); + + ctx = whisper_init_from_file_with_params(model_path, cparams); + if (ctx == nullptr) { + fprintf(stderr, "error: Also failed to init model as transcriber\n"); + return 1; + } + + return 0; +} + +int load_model_vad(const char *const model_path) { + whisper_log_set(ggml_log_cb, nullptr); + ggml_backend_load_all(); + + struct whisper_vad_context_params vcparams = + whisper_vad_default_context_params(); + + // XXX: Overridden to false in upstream due to performance? + // vcparams.use_gpu = true; + + vctx = whisper_vad_init_from_file_with_params(model_path, vcparams); + if (vctx == nullptr) { + fprintf(stderr, "error: Failed to init model as VAD\n"); + return 1; + } + + return 0; +} + +int vad(float pcmf32[], size_t pcmf32_len, float **segs_out, + size_t *segs_out_len) { + if (!whisper_vad_detect_speech(vctx, pcmf32, pcmf32_len)) { + fprintf(stderr, "error: failed to detect speech\n"); + return 1; + } + + struct whisper_vad_params params = whisper_vad_default_params(); + struct whisper_vad_segments *segs = + whisper_vad_segments_from_probs(vctx, params); + size_t segn = whisper_vad_segments_n_segments(segs); + + // fprintf(stderr, "Got segments %zd\n", segn); + + flat_segs.clear(); + + for (int i = 0; i < segn; i++) { + flat_segs.push_back(whisper_vad_segments_get_segment_t0(segs, i)); + flat_segs.push_back(whisper_vad_segments_get_segment_t1(segs, i)); + } + + // fprintf(stderr, "setting out variables: %p=%p -> %p, %p=%zx -> %zx\n", + // segs_out, *segs_out, flat_segs.data(), segs_out_len, *segs_out_len, + // flat_segs.size()); + *segs_out = flat_segs.data(); + *segs_out_len = flat_segs.size(); + + // fprintf(stderr, "freeing segs\n"); + whisper_vad_free_segments(segs); + + // fprintf(stderr, "returning\n"); + return 0; +} + +int transcribe(uint32_t threads, char *lang, bool translate, bool tdrz, + float pcmf32[], size_t pcmf32_len, size_t *segs_out_len, char *prompt) { + whisper_full_params wparams = + whisper_full_default_params(CRISPASR_SAMPLING_GREEDY); + + wparams.n_threads = threads; + if (*lang != '\0') + wparams.language = lang; + else { + wparams.language = nullptr; + } + + wparams.translate = translate; + wparams.debug_mode = true; + wparams.print_progress = true; + wparams.tdrz_enable = tdrz; + wparams.initial_prompt = prompt; + + // Reset stale abort flag from any prior cancelled call, then install the + // ggml abort hook so a subsequent set_abort(1) from Go aborts the next + // compute graph step. + g_abort.store(0, std::memory_order_relaxed); + // Only install the new-segment callback when streaming is requested + // (Go side calls set_new_segment_callback before transcribe()). Leaving + // it always-on is harmless but adds a function-pointer dispatch per + // segment for the offline path. + if (g_go_new_segment_cb.load(std::memory_order_relaxed) != 0) { + wparams.new_segment_callback = new_segment_cb; + wparams.new_segment_callback_user_data = nullptr; + } + wparams.abort_callback = abort_cb; + wparams.abort_callback_user_data = nullptr; + + fprintf(stderr, "info: Enable tdrz: %d\n", tdrz); + fprintf(stderr, "info: Initial prompt: \"%s\"\n", prompt); + + if (whisper_full(ctx, wparams, pcmf32, pcmf32_len)) { + if (g_abort.load(std::memory_order_relaxed)) { + return 2; // aborted by client + } + fprintf(stderr, "error: transcription failed\n"); + return 1; + } + + *segs_out_len = whisper_full_n_segments(ctx); + + return 0; +} + +const char *get_segment_text(int i) { + return whisper_full_get_segment_text(ctx, i); +} + +int64_t get_segment_t0(int i) { return whisper_full_get_segment_t0(ctx, i); } + +int64_t get_segment_t1(int i) { return whisper_full_get_segment_t1(ctx, i); } + +int n_tokens(int i) { return whisper_full_n_tokens(ctx, i); } + +int32_t get_token_id(int i, int j) { + return whisper_full_get_token_id(ctx, i, j); +} + +bool get_segment_speaker_turn_next(int i) { + return whisper_full_get_segment_speaker_turn_next(ctx, i); +} diff --git a/backend/go/crispasr/cpp/crispasr_shim.h b/backend/go/crispasr/cpp/crispasr_shim.h new file mode 100644 index 000000000000..4bac87eef4c8 --- /dev/null +++ b/backend/go/crispasr/cpp/crispasr_shim.h @@ -0,0 +1,30 @@ +#include +#include + +extern "C" { +int load_model(const char *const model_path); +int load_model_vad(const char *const model_path); +int vad(float pcmf32[], size_t pcmf32_size, float **segs_out, + size_t *segs_out_len); +int transcribe(uint32_t threads, char *lang, bool translate, bool tdrz, + float pcmf32[], size_t pcmf32_len, size_t *segs_out_len, + char *prompt); +const char *get_segment_text(int i); +int64_t get_segment_t0(int i); +int64_t get_segment_t1(int i); +int n_tokens(int i); +int32_t get_token_id(int i, int j); +bool get_segment_speaker_turn_next(int i); +void set_abort(int v); + +// Function pointer from Go (returned by purego.NewCallback). Invoked once +// per new-segment event during whisper_full(). The callback runs on the +// decode thread - if Go blocks (slow gRPC consumer), the decode blocks +// too. That is the intended backpressure path. +typedef void (*go_new_segment_cb)(int idx_first, int n_new, uintptr_t user_data); + +// Install the callback used by the next transcribe() call. Pass cb=0 to +// clear. user_data is opaque to C; the Go side uses it to look up +// per-call state. +void set_new_segment_callback(uintptr_t cb_ptr, uintptr_t user_data); +} diff --git a/backend/go/crispasr/gocrispasr.go b/backend/go/crispasr/gocrispasr.go new file mode 100644 index 000000000000..a4bddecbf459 --- /dev/null +++ b/backend/go/crispasr/gocrispasr.go @@ -0,0 +1,395 @@ +package main + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "unsafe" + + "github.com/go-audio/wav" + "github.com/mudler/LocalAI/pkg/grpc/base" + pb "github.com/mudler/LocalAI/pkg/grpc/proto" + "github.com/mudler/LocalAI/pkg/utils" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +var ( + CppLoadModel func(modelPath string) int + CppLoadModelVAD func(modelPath string) int + CppVAD func(pcmf32 []float32, pcmf32Size uintptr, segsOut unsafe.Pointer, segsOutLen unsafe.Pointer) int + CppTranscribe func(threads uint32, lang string, translate bool, diarize bool, pcmf32 []float32, pcmf32Len uintptr, segsOutLen unsafe.Pointer, prompt string) int + CppGetSegmentText func(i int) string + CppGetSegmentStart func(i int) int64 + CppGetSegmentEnd func(i int) int64 + CppNTokens func(i int) int + CppGetTokenID func(i int, j int) int + CppGetSegmentSpeakerTurnNext func(i int) bool + CppSetAbort func(v int) + // Set by main.go via purego.RegisterLibFunc. Installs (or clears with cb=0) + // the C-side trampoline that whisper.cpp invokes per new segment. + CppSetNewSegmentCallback func(cbPtr uintptr, userData uintptr) +) + +// streamCallStates maps per-AudioTranscriptionStream call IDs to the +// state the Go callback needs to emit deltas. Only one entry is ever +// live today (base.SingleThread), but the map shape mirrors +// sherpa-onnx's TTS callback registry and survives a future SingleThread +// removal without a contract change. +var ( + streamCallStates sync.Map // uint64 -> *streamCallState + streamCallSeq atomic.Uint64 + goNewSegmentCb uintptr // purego.NewCallback(onNewSegment) result; set in main.go at boot +) + +type streamCallState struct { + results chan *pb.TranscriptStreamResponse + diarize bool + // nextIdx tracks how many segments we've already emitted. The C + // trampoline passes idx_first = total - n_new, but we walk from + // nextIdx to (idx_first + n_new) defensively in case whisper.cpp ever + // coalesces multiple commits into a single callback invocation. + nextIdx int + // assembled mirrors the literal concat of every Delta sent on results. + // We reuse it as the final TranscriptResult.Text so the e2e + // invariant `final.Text == concat(deltas)` holds exactly. Written from + // the cgo decode thread inside onNewSegment and read by the streaming + // method after CppTranscribe returns; the cgo boundary provides the + // happens-before edge. + assembled strings.Builder +} + +// onNewSegment is the Go side of the C trampoline declared in +// gowhisper.cpp:new_segment_cb. Whisper.cpp invokes it once per +// new-segment event during whisper_full(). Reads segment text via the +// existing CppGetSegment* getters (safe to call against the singleton +// ctx; whisper.cpp is the only writer and it has already published the +// segments by the time this fires). +// +// Sends deltas synchronously: if the channel is full, this blocks the +// whisper decode thread. That's the intended backpressure path - +// dropping deltas would break the concat(deltas) == final.Text invariant +// the e2e suite asserts. +func onNewSegment(idxFirst int32, nNew int32, userData uintptr) { + v, ok := streamCallStates.Load(uint64(userData)) + if !ok { + return // call already torn down (race with cancel + cb fire) + } + state := v.(*streamCallState) + end := int(idxFirst) + int(nNew) + for i := state.nextIdx; i < end; i++ { + txt := strings.ToValidUTF8(strings.Clone(CppGetSegmentText(i)), "�") + txt = strings.TrimSpace(txt) + if state.diarize && CppGetSegmentSpeakerTurnNext(i) { + txt += " [SPEAKER_TURN]" + } + if txt == "" { + state.nextIdx = i + 1 + continue + } + // Prefix subsequent deltas with a single space so the assembled + // stream reads as one space-joined transcript. The first delta has + // no leading space, otherwise concat(deltas) would not match + // final.Text and the e2e invariant would break. + var delta string + if state.assembled.Len() == 0 { + delta = txt + } else { + delta = " " + txt + } + state.results <- &pb.TranscriptStreamResponse{Delta: delta} + state.assembled.WriteString(delta) + state.nextIdx = i + 1 + } +} + +type CrispASR struct { + base.SingleThread +} + +func (w *CrispASR) Load(opts *pb.ModelOptions) error { + vadOnly := false + + for _, oo := range opts.Options { + if oo == "vad_only" { + vadOnly = true + } else { + fmt.Fprintf(os.Stderr, "Unrecognized option: %v\n", oo) + } + } + + if vadOnly { + if ret := CppLoadModelVAD(opts.ModelFile); ret != 0 { + return fmt.Errorf("Failed to load Whisper VAD model") + } + + return nil + } + + if ret := CppLoadModel(opts.ModelFile); ret != 0 { + return fmt.Errorf("Failed to load Whisper transcription model") + } + + return nil +} + +func (w *CrispASR) VAD(req *pb.VADRequest) (pb.VADResponse, error) { + audio := req.Audio + // We expect 0xdeadbeef to be overwritten and if we see it in a stack trace we know it wasn't + segsPtr, segsLen := uintptr(0xdeadbeef), uintptr(0xdeadbeef) + segsPtrPtr, segsLenPtr := unsafe.Pointer(&segsPtr), unsafe.Pointer(&segsLen) + + if ret := CppVAD(audio, uintptr(len(audio)), segsPtrPtr, segsLenPtr); ret != 0 { + return pb.VADResponse{}, fmt.Errorf("Failed VAD") + } + + // Happens when CPP vector has not had any elements pushed to it + if segsPtr == 0 { + return pb.VADResponse{ + Segments: []*pb.VADSegment{}, + }, nil + } + + // unsafeptr warning is caused by segsPtr being on the stack and therefor being subject to stack copying AFAICT + // however the stack shouldn't have grown between setting segsPtr and now, also the memory pointed to is allocated by C++ + segs := unsafe.Slice((*float32)(unsafe.Pointer(segsPtr)), segsLen) + + vadSegments := []*pb.VADSegment{} + for i := range len(segs) >> 1 { + s := segs[2*i] / 100 + t := segs[2*i+1] / 100 + vadSegments = append(vadSegments, &pb.VADSegment{ + Start: s, + End: t, + }) + } + + return pb.VADResponse{ + Segments: vadSegments, + }, nil +} + +func (w *CrispASR) AudioTranscription(ctx context.Context, opts *pb.TranscriptRequest) (pb.TranscriptResult, error) { + if err := ctx.Err(); err != nil { + return pb.TranscriptResult{}, status.Error(codes.Canceled, "transcription cancelled") + } + + dir, err := os.MkdirTemp("", "crispasr") + if err != nil { + return pb.TranscriptResult{}, err + } + defer os.RemoveAll(dir) + + convertedPath := filepath.Join(dir, "converted.wav") + + if err := utils.AudioToWav(opts.Dst, convertedPath); err != nil { + return pb.TranscriptResult{}, err + } + + fh, err := os.Open(convertedPath) + if err != nil { + return pb.TranscriptResult{}, err + } + defer fh.Close() + + d := wav.NewDecoder(fh) + buf, err := d.FullPCMBuffer() + if err != nil { + return pb.TranscriptResult{}, err + } + + data := buf.AsFloat32Buffer().Data + var duration float32 + if buf.Format != nil && buf.Format.SampleRate > 0 { + duration = float32(len(data)) / float32(buf.Format.SampleRate) + } + segsLen := uintptr(0xdeadbeef) + segsLenPtr := unsafe.Pointer(&segsLen) + + // Watcher: flips the C-side abort flag when ctx is cancelled. The + // goroutine is joined synchronously (close(done) signals it to exit, + // wg.Wait() blocks until it has) so a late CppSetAbort(1) cannot fire + // after the function returns and corrupt the next transcription call. + done := make(chan struct{}) + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + select { + case <-ctx.Done(): + CppSetAbort(1) + case <-done: + } + }() + defer func() { + close(done) + wg.Wait() + }() + + ret := CppTranscribe(opts.Threads, opts.Language, opts.Translate, opts.Diarize, data, uintptr(len(data)), segsLenPtr, opts.Prompt) + if ret == 2 { + return pb.TranscriptResult{}, status.Error(codes.Canceled, "transcription cancelled") + } + if ret != 0 { + return pb.TranscriptResult{}, fmt.Errorf("Failed Transcribe") + } + + segments := []*pb.TranscriptSegment{} + text := "" + for i := range int(segsLen) { + // segment start/end conversion factor taken from https://github.com/ggml-org/whisper.cpp/blob/master/examples/cli/cli.cpp#L895 + s := CppGetSegmentStart(i) * (10000000) + t := CppGetSegmentEnd(i) * (10000000) + // whisper.cpp can emit bytes that aren't valid UTF-8 (e.g. a multibyte + // codepoint split across token boundaries); protobuf string fields + // reject those at marshal time. Scrub before the value escapes cgo. + txt := strings.ToValidUTF8(strings.Clone(CppGetSegmentText(i)), "�") + tokens := make([]int32, CppNTokens(i)) + + if opts.Diarize && CppGetSegmentSpeakerTurnNext(i) { + txt += " [SPEAKER_TURN]" + } + + for j := range tokens { + tokens[j] = int32(CppGetTokenID(i, j)) + } + segment := &pb.TranscriptSegment{ + Id: int32(i), + Text: txt, + Start: s, End: t, + Tokens: tokens, + } + + segments = append(segments, segment) + + text += " " + strings.TrimSpace(txt) + } + + return pb.TranscriptResult{ + Segments: segments, + Text: strings.TrimSpace(text), + Language: opts.Language, + Duration: duration, + }, nil +} + +// AudioTranscriptionStream runs whisper_full() and emits deltas via +// whisper.cpp's new_segment_callback as segments are decoded, then a +// final TranscriptResult. The offline AudioTranscription is unchanged; +// both paths share whisper's single-instance ctx and the SingleThread +// concurrency model. +func (w *CrispASR) AudioTranscriptionStream(ctx context.Context, opts *pb.TranscriptRequest, results chan *pb.TranscriptStreamResponse) error { + defer close(results) + + if err := ctx.Err(); err != nil { + return status.Error(codes.Canceled, "transcription cancelled") + } + + dir, err := os.MkdirTemp("", "crispasr") + if err != nil { + return err + } + defer func() { _ = os.RemoveAll(dir) }() + + convertedPath := filepath.Join(dir, "converted.wav") + if err := utils.AudioToWav(opts.Dst, convertedPath); err != nil { + return err + } + + fh, err := os.Open(convertedPath) + if err != nil { + return err + } + defer func() { _ = fh.Close() }() + + d := wav.NewDecoder(fh) + buf, err := d.FullPCMBuffer() + if err != nil { + return err + } + data := buf.AsFloat32Buffer().Data + var duration float32 + if buf.Format != nil && buf.Format.SampleRate > 0 { + duration = float32(len(data)) / float32(buf.Format.SampleRate) + } + + // Register per-call state and install the C-side callback. defer + // teardown so even a panic clears the C pointer (otherwise a stale + // callback fires on the next AudioTranscription call). + callID := streamCallSeq.Add(1) + state := &streamCallState{ + results: results, + diarize: opts.Diarize, + } + streamCallStates.Store(callID, state) + CppSetNewSegmentCallback(goNewSegmentCb, uintptr(callID)) + defer func() { + CppSetNewSegmentCallback(0, 0) + streamCallStates.Delete(callID) + }() + + // Same abort-watcher pattern as AudioTranscription. Joined synchronously + // so a late CppSetAbort(1) cannot fire after this function returns. + done := make(chan struct{}) + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + select { + case <-ctx.Done(): + CppSetAbort(1) + case <-done: + } + }() + defer func() { + close(done) + wg.Wait() + }() + + segsLen := uintptr(0xdeadbeef) + segsLenPtr := unsafe.Pointer(&segsLen) + ret := CppTranscribe(opts.Threads, opts.Language, opts.Translate, opts.Diarize, data, uintptr(len(data)), segsLenPtr, opts.Prompt) + if ret == 2 { + return status.Error(codes.Canceled, "transcription cancelled") + } + if ret != 0 { + return fmt.Errorf("Failed Transcribe") + } + + // Build the final TranscriptResult. Segments[] mirrors the offline + // path so the SSE done event carries the same per-segment shape. + // final.Text reuses the assembled stream so concat(deltas) == final.Text + // holds exactly, matching the e2e contract. + segments := []*pb.TranscriptSegment{} + for i := range int(segsLen) { + s := CppGetSegmentStart(i) * 10000000 + t := CppGetSegmentEnd(i) * 10000000 + txt := strings.ToValidUTF8(strings.Clone(CppGetSegmentText(i)), "�") + tokens := make([]int32, CppNTokens(i)) + if opts.Diarize && CppGetSegmentSpeakerTurnNext(i) { + txt += " [SPEAKER_TURN]" + } + for j := range tokens { + tokens[j] = int32(CppGetTokenID(i, j)) + } + segments = append(segments, &pb.TranscriptSegment{ + Id: int32(i), + Text: txt, + Start: s, End: t, + Tokens: tokens, + }) + } + + final := &pb.TranscriptResult{ + Segments: segments, + Text: state.assembled.String(), + Language: opts.Language, + Duration: duration, + } + results <- &pb.TranscriptStreamResponse{FinalResult: final} + return nil +} diff --git a/backend/go/crispasr/main.go b/backend/go/crispasr/main.go new file mode 100644 index 000000000000..8582a1ca8a91 --- /dev/null +++ b/backend/go/crispasr/main.go @@ -0,0 +1,58 @@ +package main + +// Note: this is started internally by LocalAI and a server is allocated for each model +import ( + "flag" + "os" + + "github.com/ebitengine/purego" + grpc "github.com/mudler/LocalAI/pkg/grpc" +) + +var ( + addr = flag.String("addr", "localhost:50051", "the address to connect to") +) + +type LibFuncs struct { + FuncPtr any + Name string +} + +func main() { + libName := os.Getenv("CRISPASR_LIBRARY") + if libName == "" { + libName = "./libgocrispasr-fallback.so" + } + + lib, err := purego.Dlopen(libName, purego.RTLD_NOW|purego.RTLD_GLOBAL) + if err != nil { + panic(err) + } + + libFuncs := []LibFuncs{ + {&CppLoadModel, "load_model"}, + {&CppLoadModelVAD, "load_model_vad"}, + {&CppVAD, "vad"}, + {&CppTranscribe, "transcribe"}, + {&CppGetSegmentText, "get_segment_text"}, + {&CppGetSegmentStart, "get_segment_t0"}, + {&CppGetSegmentEnd, "get_segment_t1"}, + {&CppNTokens, "n_tokens"}, + {&CppGetTokenID, "get_token_id"}, + {&CppGetSegmentSpeakerTurnNext, "get_segment_speaker_turn_next"}, + {&CppSetAbort, "set_abort"}, + {&CppSetNewSegmentCallback, "set_new_segment_callback"}, + } + + for _, lf := range libFuncs { + purego.RegisterLibFunc(lf.FuncPtr, lib, lf.Name) + } + + goNewSegmentCb = purego.NewCallback(onNewSegment) + + flag.Parse() + + if err := grpc.StartServer(*addr, &CrispASR{}); err != nil { + panic(err) + } +} diff --git a/backend/go/crispasr/package.sh b/backend/go/crispasr/package.sh new file mode 100755 index 000000000000..09e95f13ac1a --- /dev/null +++ b/backend/go/crispasr/package.sh @@ -0,0 +1,65 @@ +#!/bin/bash + +# Script to copy the appropriate libraries based on architecture +# This script is used in the final stage of the Dockerfile + +set -e + +CURDIR=$(dirname "$(realpath $0)") +REPO_ROOT="${CURDIR}/../../.." + +# Create lib directory +mkdir -p $CURDIR/package/lib + +cp -avf $CURDIR/crispasr $CURDIR/package/ +cp -fv $CURDIR/libgocrispasr-*.so $CURDIR/package/ +cp -fv $CURDIR/run.sh $CURDIR/package/ + +# Detect architecture and copy appropriate libraries +if [ -f "/lib64/ld-linux-x86-64.so.2" ]; then + # x86_64 architecture + echo "Detected x86_64 architecture, copying x86_64 libraries..." + cp -arfLv /lib64/ld-linux-x86-64.so.2 $CURDIR/package/lib/ld.so + cp -arfLv /lib/x86_64-linux-gnu/libc.so.6 $CURDIR/package/lib/libc.so.6 + cp -arfLv /lib/x86_64-linux-gnu/libgcc_s.so.1 $CURDIR/package/lib/libgcc_s.so.1 + cp -arfLv /lib/x86_64-linux-gnu/libstdc++.so.6 $CURDIR/package/lib/libstdc++.so.6 + cp -arfLv /lib/x86_64-linux-gnu/libm.so.6 $CURDIR/package/lib/libm.so.6 + cp -arfLv /lib/x86_64-linux-gnu/libgomp.so.1 $CURDIR/package/lib/libgomp.so.1 + cp -arfLv /lib/x86_64-linux-gnu/libgcc_s.so.1 $CURDIR/package/lib/libgcc_s.so.1 + cp -arfLv /lib/x86_64-linux-gnu/libstdc++.so.6 $CURDIR/package/lib/libstdc++.so.6 + cp -arfLv /lib/x86_64-linux-gnu/libdl.so.2 $CURDIR/package/lib/libdl.so.2 + cp -arfLv /lib/x86_64-linux-gnu/librt.so.1 $CURDIR/package/lib/librt.so.1 + cp -arfLv /lib/x86_64-linux-gnu/libpthread.so.0 $CURDIR/package/lib/libpthread.so.0 +elif [ -f "/lib/ld-linux-aarch64.so.1" ]; then + # ARM64 architecture + echo "Detected ARM64 architecture, copying ARM64 libraries..." + cp -arfLv /lib/ld-linux-aarch64.so.1 $CURDIR/package/lib/ld.so + cp -arfLv /lib/aarch64-linux-gnu/libc.so.6 $CURDIR/package/lib/libc.so.6 + cp -arfLv /lib/aarch64-linux-gnu/libgcc_s.so.1 $CURDIR/package/lib/libgcc_s.so.1 + cp -arfLv /lib/aarch64-linux-gnu/libstdc++.so.6 $CURDIR/package/lib/libstdc++.so.6 + cp -arfLv /lib/aarch64-linux-gnu/libm.so.6 $CURDIR/package/lib/libm.so.6 + cp -arfLv /lib/aarch64-linux-gnu/libgomp.so.1 $CURDIR/package/lib/libgomp.so.1 + cp -arfLv /lib/aarch64-linux-gnu/libgcc_s.so.1 $CURDIR/package/lib/libgcc_s.so.1 + cp -arfLv /lib/aarch64-linux-gnu/libstdc++.so.6 $CURDIR/package/lib/libstdc++.so.6 + cp -arfLv /lib/aarch64-linux-gnu/libdl.so.2 $CURDIR/package/lib/libdl.so.2 + cp -arfLv /lib/aarch64-linux-gnu/librt.so.1 $CURDIR/package/lib/librt.so.1 + cp -arfLv /lib/aarch64-linux-gnu/libpthread.so.0 $CURDIR/package/lib/libpthread.so.0 +elif [ $(uname -s) = "Darwin" ]; then + echo "Detected Darwin" +else + echo "Error: Could not detect architecture" + exit 1 +fi + +# Package GPU libraries based on BUILD_TYPE +# The GPU library packaging script will detect BUILD_TYPE and copy appropriate GPU libraries +GPU_LIB_SCRIPT="${REPO_ROOT}/scripts/build/package-gpu-libs.sh" +if [ -f "$GPU_LIB_SCRIPT" ]; then + echo "Packaging GPU libraries for BUILD_TYPE=${BUILD_TYPE:-cpu}..." + source "$GPU_LIB_SCRIPT" "$CURDIR/package/lib" + package_gpu_libs +fi + +echo "Packaging completed successfully" +ls -liah $CURDIR/package/ +ls -liah $CURDIR/package/lib/ diff --git a/backend/go/crispasr/run.sh b/backend/go/crispasr/run.sh new file mode 100755 index 000000000000..942b418a3584 --- /dev/null +++ b/backend/go/crispasr/run.sh @@ -0,0 +1,52 @@ +#!/bin/bash +set -ex + +# Get the absolute current dir where the script is located +CURDIR=$(dirname "$(realpath $0)") + +cd / + +echo "CPU info:" +if [ "$(uname)" != "Darwin" ]; then + grep -e "model\sname" /proc/cpuinfo | head -1 + grep -e "flags" /proc/cpuinfo | head -1 +fi + +LIBRARY="$CURDIR/libgocrispasr-fallback.so" + +if [ "$(uname)" != "Darwin" ]; then + if grep -q -e "\savx\s" /proc/cpuinfo ; then + echo "CPU: AVX found OK" + if [ -e $CURDIR/libgocrispasr-avx.so ]; then + LIBRARY="$CURDIR/libgocrispasr-avx.so" + fi + fi + + if grep -q -e "\savx2\s" /proc/cpuinfo ; then + echo "CPU: AVX2 found OK" + if [ -e $CURDIR/libgocrispasr-avx2.so ]; then + LIBRARY="$CURDIR/libgocrispasr-avx2.so" + fi + fi + + # Check avx 512 + if grep -q -e "\savx512f\s" /proc/cpuinfo ; then + echo "CPU: AVX512F found OK" + if [ -e $CURDIR/libgocrispasr-avx512.so ]; then + LIBRARY="$CURDIR/libgocrispasr-avx512.so" + fi + fi +fi + +export LD_LIBRARY_PATH=$CURDIR/lib:$LD_LIBRARY_PATH +export CRISPASR_LIBRARY=$LIBRARY + +# If there is a lib/ld.so, use it +if [ -f $CURDIR/lib/ld.so ]; then + echo "Using lib/ld.so" + echo "Using library: $LIBRARY" + exec $CURDIR/lib/ld.so $CURDIR/crispasr "$@" +fi + +echo "Using library: $LIBRARY" +exec $CURDIR/crispasr "$@" From 2336043680cdad624b14c01ab23ae6398cc13514 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sat, 30 May 2026 22:55:11 +0000 Subject: [PATCH 02/29] polish(crispasr): brand error strings + fix stale shim comment Signed-off-by: Ettore Di Giacinto --- backend/go/crispasr/gocrispasr.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/go/crispasr/gocrispasr.go b/backend/go/crispasr/gocrispasr.go index a4bddecbf459..74a98879bb34 100644 --- a/backend/go/crispasr/gocrispasr.go +++ b/backend/go/crispasr/gocrispasr.go @@ -64,7 +64,7 @@ type streamCallState struct { } // onNewSegment is the Go side of the C trampoline declared in -// gowhisper.cpp:new_segment_cb. Whisper.cpp invokes it once per +// crispasr_shim.cpp:new_segment_cb. Whisper.cpp invokes it once per // new-segment event during whisper_full(). Reads segment text via the // existing CppGetSegment* getters (safe to call against the singleton // ctx; whisper.cpp is the only writer and it has already published the @@ -124,14 +124,14 @@ func (w *CrispASR) Load(opts *pb.ModelOptions) error { if vadOnly { if ret := CppLoadModelVAD(opts.ModelFile); ret != 0 { - return fmt.Errorf("Failed to load Whisper VAD model") + return fmt.Errorf("Failed to load CrispASR VAD model") } return nil } if ret := CppLoadModel(opts.ModelFile); ret != 0 { - return fmt.Errorf("Failed to load Whisper transcription model") + return fmt.Errorf("Failed to load CrispASR transcription model") } return nil From 2aa2bb70cbb0a87e89f019e25716be0c9606869a Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sat, 30 May 2026 22:59:47 +0000 Subject: [PATCH 03/29] build(crispasr): register backend in root Makefile Mirror the whisper Go backend registration for the new crispasr backend: NOTPARALLEL entry, prepare-test-extra/test-extra hooks, BACKEND_CRISPASR definition, docker-build target generation, and the docker-build-backends aggregate target. Signed-off-by: Ettore Di Giacinto --- Makefile | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index a401762cf695..67c901efbfab 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ # Disable parallel execution for backend builds -.NOTPARALLEL: backends/diffusers backends/llama-cpp backends/turboquant backends/outetts backends/piper backends/stablediffusion-ggml backends/whisper backends/parakeet-cpp backends/faster-whisper backends/silero-vad backends/local-store backends/huggingface backends/rfdetr backends/rfdetr-cpp backends/insightface backends/speaker-recognition backends/kitten-tts backends/kokoro backends/chatterbox backends/llama-cpp-darwin backends/neutts build-darwin-python-backend build-darwin-go-backend backends/mlx backends/diffuser-darwin backends/mlx-vlm backends/mlx-audio backends/mlx-distributed backends/stablediffusion-ggml-darwin backends/vllm backends/vllm-omni backends/sglang backends/moonshine backends/pocket-tts backends/qwen-tts backends/faster-qwen3-tts backends/qwen-asr backends/nemo backends/voxcpm backends/whisperx backends/ace-step backends/acestep-cpp backends/fish-speech backends/voxtral backends/opus backends/trl backends/llama-cpp-quantization backends/kokoros backends/sam3-cpp backends/qwen3-tts-cpp backends/vibevoice-cpp backends/localvqe backends/tinygrad backends/sherpa-onnx backends/ds4 backends/ds4-darwin backends/liquid-audio +.NOTPARALLEL: backends/diffusers backends/llama-cpp backends/turboquant backends/outetts backends/piper backends/stablediffusion-ggml backends/whisper backends/crispasr backends/parakeet-cpp backends/faster-whisper backends/silero-vad backends/local-store backends/huggingface backends/rfdetr backends/rfdetr-cpp backends/insightface backends/speaker-recognition backends/kitten-tts backends/kokoro backends/chatterbox backends/llama-cpp-darwin backends/neutts build-darwin-python-backend build-darwin-go-backend backends/mlx backends/diffuser-darwin backends/mlx-vlm backends/mlx-audio backends/mlx-distributed backends/stablediffusion-ggml-darwin backends/vllm backends/vllm-omni backends/sglang backends/moonshine backends/pocket-tts backends/qwen-tts backends/faster-qwen3-tts backends/qwen-asr backends/nemo backends/voxcpm backends/whisperx backends/ace-step backends/acestep-cpp backends/fish-speech backends/voxtral backends/opus backends/trl backends/llama-cpp-quantization backends/kokoros backends/sam3-cpp backends/qwen3-tts-cpp backends/vibevoice-cpp backends/localvqe backends/tinygrad backends/sherpa-onnx backends/ds4 backends/ds4-darwin backends/liquid-audio GOCMD=go GOTEST=$(GOCMD) test @@ -559,6 +559,7 @@ prepare-test-extra: protogen-python $(MAKE) -C backend/python/speaker-recognition $(MAKE) -C backend/rust/kokoros kokoros-grpc $(MAKE) -C backend/go/rfdetr-cpp + $(MAKE) -C backend/go/crispasr test-extra: prepare-test-extra $(MAKE) -C backend/python/transformers test @@ -586,6 +587,7 @@ test-extra: prepare-test-extra $(MAKE) -C backend/python/speaker-recognition test $(MAKE) -C backend/rust/kokoros test $(MAKE) -C backend/go/rfdetr-cpp test + $(MAKE) -C backend/go/crispasr test ## ## End-to-end gRPC tests that exercise a built backend container image. @@ -1162,6 +1164,7 @@ BACKEND_HUGGINGFACE = huggingface|golang|.|false|true BACKEND_SILERO_VAD = silero-vad|golang|.|false|true BACKEND_STABLEDIFFUSION_GGML = stablediffusion-ggml|golang|.|--progress=plain|true BACKEND_WHISPER = whisper|golang|.|false|true +BACKEND_CRISPASR = crispasr|golang|.|false|true BACKEND_PARAKEET_CPP = parakeet-cpp|golang|.|false|true BACKEND_VOXTRAL = voxtral|golang|.|false|true BACKEND_ACESTEP_CPP = acestep-cpp|golang|.|false|true @@ -1250,6 +1253,7 @@ $(eval $(call generate-docker-build-target,$(BACKEND_HUGGINGFACE))) $(eval $(call generate-docker-build-target,$(BACKEND_SILERO_VAD))) $(eval $(call generate-docker-build-target,$(BACKEND_STABLEDIFFUSION_GGML))) $(eval $(call generate-docker-build-target,$(BACKEND_WHISPER))) +$(eval $(call generate-docker-build-target,$(BACKEND_CRISPASR))) $(eval $(call generate-docker-build-target,$(BACKEND_PARAKEET_CPP))) $(eval $(call generate-docker-build-target,$(BACKEND_VOXTRAL))) $(eval $(call generate-docker-build-target,$(BACKEND_OPUS))) @@ -1300,7 +1304,7 @@ $(eval $(call generate-docker-build-target,$(BACKEND_SHERPA_ONNX))) docker-save-%: backend-images docker save local-ai-backend:$* -o backend-images/$*.tar -docker-build-backends: docker-build-llama-cpp docker-build-ik-llama-cpp docker-build-turboquant docker-build-ds4 docker-build-rerankers docker-build-vllm docker-build-vllm-omni docker-build-sglang docker-build-transformers docker-build-outetts docker-build-diffusers docker-build-kokoro docker-build-faster-whisper docker-build-coqui docker-build-chatterbox docker-build-vibevoice docker-build-liquid-audio docker-build-moonshine docker-build-pocket-tts docker-build-qwen-tts docker-build-fish-speech docker-build-faster-qwen3-tts docker-build-qwen-asr docker-build-nemo docker-build-voxcpm docker-build-whisperx docker-build-ace-step docker-build-acestep-cpp docker-build-voxtral docker-build-mlx-distributed docker-build-trl docker-build-llama-cpp-quantization docker-build-tinygrad docker-build-kokoros docker-build-sam3-cpp docker-build-rfdetr-cpp docker-build-qwen3-tts-cpp docker-build-vibevoice-cpp docker-build-localvqe docker-build-insightface docker-build-speaker-recognition docker-build-sherpa-onnx docker-build-cloud-proxy +docker-build-backends: docker-build-llama-cpp docker-build-ik-llama-cpp docker-build-turboquant docker-build-ds4 docker-build-rerankers docker-build-vllm docker-build-vllm-omni docker-build-sglang docker-build-transformers docker-build-outetts docker-build-diffusers docker-build-kokoro docker-build-faster-whisper docker-build-crispasr docker-build-coqui docker-build-chatterbox docker-build-vibevoice docker-build-liquid-audio docker-build-moonshine docker-build-pocket-tts docker-build-qwen-tts docker-build-fish-speech docker-build-faster-qwen3-tts docker-build-qwen-asr docker-build-nemo docker-build-voxcpm docker-build-whisperx docker-build-ace-step docker-build-acestep-cpp docker-build-voxtral docker-build-mlx-distributed docker-build-trl docker-build-llama-cpp-quantization docker-build-tinygrad docker-build-kokoros docker-build-sam3-cpp docker-build-rfdetr-cpp docker-build-qwen3-tts-cpp docker-build-vibevoice-cpp docker-build-localvqe docker-build-insightface docker-build-speaker-recognition docker-build-sherpa-onnx docker-build-cloud-proxy ######################################################## ### Mock Backend for E2E Tests From 80ebe1c50687b0be72ea451386c93fcac4498c53 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sat, 30 May 2026 22:59:55 +0000 Subject: [PATCH 04/29] ci(crispasr): add backend build matrix entries Mirror the 11 whisper golang Dockerfile matrix entries (CPU amd64/arm64, CUDA 12/13, L4T CUDA 13, Intel SYCL f32/f16, Vulkan amd64/arm64, L4T arm64, ROCm hipblas) with backend and tag-suffix substituted to crispasr. Signed-off-by: Ettore Di Giacinto --- .github/backend-matrix.yml | 148 +++++++++++++++++++++++++++++++++++++ 1 file changed, 148 insertions(+) diff --git a/.github/backend-matrix.yml b/.github/backend-matrix.yml index de41aba52752..1ba8204b8a2d 100644 --- a/.github/backend-matrix.yml +++ b/.github/backend-matrix.yml @@ -716,6 +716,19 @@ include: dockerfile: "./backend/Dockerfile.golang" context: "./" ubuntu-version: '2404' + - build-type: 'cublas' + cuda-major-version: "12" + cuda-minor-version: "8" + platforms: 'linux/amd64' + tag-latest: 'auto' + tag-suffix: '-gpu-nvidia-cuda-12-crispasr' + runs-on: 'ubuntu-latest' + base-image: "ubuntu:24.04" + skip-drivers: 'false' + backend: "crispasr" + dockerfile: "./backend/Dockerfile.golang" + context: "./" + ubuntu-version: '2404' - build-type: 'cublas' cuda-major-version: "12" cuda-minor-version: "8" @@ -1569,6 +1582,19 @@ include: dockerfile: "./backend/Dockerfile.golang" context: "./" ubuntu-version: '2404' + - build-type: 'cublas' + cuda-major-version: "13" + cuda-minor-version: "0" + platforms: 'linux/amd64' + tag-latest: 'auto' + tag-suffix: '-gpu-nvidia-cuda-13-crispasr' + runs-on: 'ubuntu-latest' + base-image: "ubuntu:24.04" + skip-drivers: 'false' + backend: "crispasr" + dockerfile: "./backend/Dockerfile.golang" + context: "./" + ubuntu-version: '2404' - build-type: 'cublas' cuda-major-version: "13" cuda-minor-version: "0" @@ -1595,6 +1621,19 @@ include: backend: "whisper" dockerfile: "./backend/Dockerfile.golang" context: "./" + - build-type: 'cublas' + cuda-major-version: "13" + cuda-minor-version: "0" + platforms: 'linux/arm64' + skip-drivers: 'false' + tag-latest: 'auto' + tag-suffix: '-nvidia-l4t-cuda-13-arm64-crispasr' + base-image: "ubuntu:24.04" + ubuntu-version: '2404' + runs-on: 'ubuntu-24.04-arm' + backend: "crispasr" + dockerfile: "./backend/Dockerfile.golang" + context: "./" - build-type: 'cublas' cuda-major-version: "13" cuda-minor-version: "0" @@ -2889,6 +2928,20 @@ include: dockerfile: "./backend/Dockerfile.golang" context: "./" ubuntu-version: '2404' + - build-type: '' + cuda-major-version: "" + cuda-minor-version: "" + platforms: 'linux/amd64' + platform-tag: 'amd64' + tag-latest: 'auto' + tag-suffix: '-cpu-crispasr' + runs-on: 'ubuntu-latest' + base-image: "ubuntu:24.04" + skip-drivers: 'false' + backend: "crispasr" + dockerfile: "./backend/Dockerfile.golang" + context: "./" + ubuntu-version: '2404' - build-type: '' cuda-major-version: "" cuda-minor-version: "" @@ -2903,6 +2956,20 @@ include: dockerfile: "./backend/Dockerfile.golang" context: "./" ubuntu-version: '2404' + - build-type: '' + cuda-major-version: "" + cuda-minor-version: "" + platforms: 'linux/arm64' + platform-tag: 'arm64' + tag-latest: 'auto' + tag-suffix: '-cpu-crispasr' + runs-on: 'ubuntu-24.04-arm' + base-image: "ubuntu:24.04" + skip-drivers: 'false' + backend: "crispasr" + dockerfile: "./backend/Dockerfile.golang" + context: "./" + ubuntu-version: '2404' - build-type: 'sycl_f32' cuda-major-version: "" cuda-minor-version: "" @@ -2916,6 +2983,19 @@ include: dockerfile: "./backend/Dockerfile.golang" context: "./" ubuntu-version: '2404' + - build-type: 'sycl_f32' + cuda-major-version: "" + cuda-minor-version: "" + platforms: 'linux/amd64' + tag-latest: 'auto' + tag-suffix: '-gpu-intel-sycl-f32-crispasr' + runs-on: 'ubuntu-latest' + base-image: "intel/oneapi-basekit:2025.3.0-0-devel-ubuntu24.04" + skip-drivers: 'false' + backend: "crispasr" + dockerfile: "./backend/Dockerfile.golang" + context: "./" + ubuntu-version: '2404' - build-type: 'sycl_f16' cuda-major-version: "" cuda-minor-version: "" @@ -2929,6 +3009,19 @@ include: dockerfile: "./backend/Dockerfile.golang" context: "./" ubuntu-version: '2404' + - build-type: 'sycl_f16' + cuda-major-version: "" + cuda-minor-version: "" + platforms: 'linux/amd64' + tag-latest: 'auto' + tag-suffix: '-gpu-intel-sycl-f16-crispasr' + runs-on: 'ubuntu-latest' + base-image: "intel/oneapi-basekit:2025.3.0-0-devel-ubuntu24.04" + skip-drivers: 'false' + backend: "crispasr" + dockerfile: "./backend/Dockerfile.golang" + context: "./" + ubuntu-version: '2404' - build-type: 'vulkan' cuda-major-version: "" cuda-minor-version: "" @@ -2943,6 +3036,20 @@ include: dockerfile: "./backend/Dockerfile.golang" context: "./" ubuntu-version: '2404' + - build-type: 'vulkan' + cuda-major-version: "" + cuda-minor-version: "" + platforms: 'linux/amd64' + platform-tag: 'amd64' + tag-latest: 'auto' + tag-suffix: '-gpu-vulkan-crispasr' + runs-on: 'ubuntu-latest' + base-image: "ubuntu:24.04" + skip-drivers: 'false' + backend: "crispasr" + dockerfile: "./backend/Dockerfile.golang" + context: "./" + ubuntu-version: '2404' - build-type: 'vulkan' cuda-major-version: "" cuda-minor-version: "" @@ -2957,6 +3064,20 @@ include: dockerfile: "./backend/Dockerfile.golang" context: "./" ubuntu-version: '2404' + - build-type: 'vulkan' + cuda-major-version: "" + cuda-minor-version: "" + platforms: 'linux/arm64' + platform-tag: 'arm64' + tag-latest: 'auto' + tag-suffix: '-gpu-vulkan-crispasr' + runs-on: 'ubuntu-24.04-arm' + base-image: "ubuntu:24.04" + skip-drivers: 'false' + backend: "crispasr" + dockerfile: "./backend/Dockerfile.golang" + context: "./" + ubuntu-version: '2404' - build-type: 'cublas' cuda-major-version: "12" cuda-minor-version: "0" @@ -2970,6 +3091,19 @@ include: dockerfile: "./backend/Dockerfile.golang" context: "./" ubuntu-version: '2204' + - build-type: 'cublas' + cuda-major-version: "12" + cuda-minor-version: "0" + platforms: 'linux/arm64' + skip-drivers: 'false' + tag-latest: 'auto' + tag-suffix: '-nvidia-l4t-arm64-crispasr' + base-image: "nvcr.io/nvidia/l4t-jetpack:r36.4.0" + runs-on: 'ubuntu-24.04-arm' + backend: "crispasr" + dockerfile: "./backend/Dockerfile.golang" + context: "./" + ubuntu-version: '2204' - build-type: 'hipblas' cuda-major-version: "" cuda-minor-version: "" @@ -2984,6 +3118,20 @@ include: context: "./" ubuntu-version: '2404' # parakeet-cpp + - build-type: 'hipblas' + cuda-major-version: "" + cuda-minor-version: "" + platforms: 'linux/amd64' + tag-latest: 'auto' + tag-suffix: '-gpu-rocm-hipblas-crispasr' + base-image: "rocm/dev-ubuntu-24.04:7.2.1" + runs-on: 'ubuntu-latest' + skip-drivers: 'false' + backend: "crispasr" + dockerfile: "./backend/Dockerfile.golang" + context: "./" + ubuntu-version: '2404' + # parakeet-cpp - build-type: '' cuda-major-version: "" cuda-minor-version: "" From b1c880e0d5f70f57cb944256ffbf96c0ce276378 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sat, 30 May 2026 22:59:55 +0000 Subject: [PATCH 05/29] feat(gallery): add crispasr backend gallery entries Add the crispasr meta anchor and its full set of image gallery entries (cpu, metal, cuda12/13, rocm, intel-sycl f32/f16, vulkan, L4T arm64, L4T cuda13 arm64, plus -development variants), mirroring the whisper backend gallery block. Signed-off-by: Ettore Di Giacinto --- backend/index.yaml | 152 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 152 insertions(+) diff --git a/backend/index.yaml b/backend/index.yaml index 887e2e57e24d..37e6890710e4 100644 --- a/backend/index.yaml +++ b/backend/index.yaml @@ -122,6 +122,33 @@ nvidia-cuda-12: "cuda12-whisper" nvidia-l4t-cuda-12: "nvidia-l4t-arm64-whisper" nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-whisper" +- &crispasr + name: "crispasr" + alias: "crispasr" + license: mit + icon: https://user-images.githubusercontent.com/1991296/235238348-05d0f6a4-da44-4900-a1de-d0707e75b763.jpeg + description: | + CrispASR unified speech engine (whisper.cpp fork on ggml) supporting many ASR architectures (Parakeet, Canary, Voxtral, Qwen3-ASR, Granite, Wav2Vec2, Moonshine, OmniASR, FireRedASR, and more). + urls: + - https://github.com/CrispStrobe/CrispASR + tags: + - audio-transcription + - CPU + - GPU + - CUDA + - HIP + capabilities: + default: "cpu-crispasr" + nvidia: "cuda12-crispasr" + intel: "intel-sycl-f16-crispasr" + metal: "metal-crispasr" + amd: "rocm-crispasr" + vulkan: "vulkan-crispasr" + nvidia-l4t: "nvidia-l4t-arm64-crispasr" + nvidia-cuda-13: "cuda13-crispasr" + nvidia-cuda-12: "cuda12-crispasr" + nvidia-l4t-cuda-12: "nvidia-l4t-arm64-crispasr" + nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-crispasr" - ¶keetcpp name: "parakeet-cpp" alias: "parakeet-cpp" @@ -1957,6 +1984,131 @@ uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-13-whisper" mirrors: - localai/localai-backends:master-gpu-nvidia-cuda-13-whisper +## crispasr +- !!merge <<: *crispasr + name: "crispasr-development" + capabilities: + default: "cpu-crispasr-development" + nvidia: "cuda12-crispasr-development" + intel: "intel-sycl-f16-crispasr-development" + metal: "metal-crispasr-development" + amd: "rocm-crispasr-development" + vulkan: "vulkan-crispasr-development" + nvidia-l4t: "nvidia-l4t-arm64-crispasr-development" + nvidia-cuda-13: "cuda13-crispasr-development" + nvidia-cuda-12: "cuda12-crispasr-development" + nvidia-l4t-cuda-12: "nvidia-l4t-arm64-crispasr-development" + nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-crispasr-development" +- !!merge <<: *crispasr + name: "nvidia-l4t-arm64-crispasr" + uri: "quay.io/go-skynet/local-ai-backends:latest-nvidia-l4t-arm64-crispasr" + mirrors: + - localai/localai-backends:latest-nvidia-l4t-arm64-crispasr +- !!merge <<: *crispasr + name: "nvidia-l4t-arm64-crispasr-development" + uri: "quay.io/go-skynet/local-ai-backends:master-nvidia-l4t-arm64-crispasr" + mirrors: + - localai/localai-backends:master-nvidia-l4t-arm64-crispasr +- !!merge <<: *crispasr + name: "cuda13-nvidia-l4t-arm64-crispasr" + uri: "quay.io/go-skynet/local-ai-backends:latest-nvidia-l4t-cuda-13-arm64-crispasr" + mirrors: + - localai/localai-backends:latest-nvidia-l4t-cuda-13-arm64-crispasr +- !!merge <<: *crispasr + name: "cuda13-nvidia-l4t-arm64-crispasr-development" + uri: "quay.io/go-skynet/local-ai-backends:master-nvidia-l4t-cuda-13-arm64-crispasr" + mirrors: + - localai/localai-backends:master-nvidia-l4t-cuda-13-arm64-crispasr +- !!merge <<: *crispasr + name: "cpu-crispasr" + uri: "quay.io/go-skynet/local-ai-backends:latest-cpu-crispasr" + mirrors: + - localai/localai-backends:latest-cpu-crispasr +- !!merge <<: *crispasr + name: "metal-crispasr" + uri: "quay.io/go-skynet/local-ai-backends:latest-metal-darwin-arm64-crispasr" + mirrors: + - localai/localai-backends:latest-metal-darwin-arm64-crispasr +- !!merge <<: *crispasr + name: "metal-crispasr-development" + uri: "quay.io/go-skynet/local-ai-backends:master-metal-darwin-arm64-crispasr" + mirrors: + - localai/localai-backends:master-metal-darwin-arm64-crispasr +- !!merge <<: *crispasr + name: "cpu-crispasr-development" + uri: "quay.io/go-skynet/local-ai-backends:master-cpu-crispasr" + mirrors: + - localai/localai-backends:master-cpu-crispasr +- !!merge <<: *crispasr + name: "cuda12-crispasr" + uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-12-crispasr" + mirrors: + - localai/localai-backends:latest-gpu-nvidia-cuda-12-crispasr +- !!merge <<: *crispasr + name: "rocm-crispasr" + uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-rocm-hipblas-crispasr" + mirrors: + - localai/localai-backends:latest-gpu-rocm-hipblas-crispasr +- !!merge <<: *crispasr + name: "intel-sycl-f32-crispasr" + uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-intel-sycl-f32-crispasr" + mirrors: + - localai/localai-backends:latest-gpu-intel-sycl-f32-crispasr +- !!merge <<: *crispasr + name: "intel-sycl-f16-crispasr" + uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-intel-sycl-f16-crispasr" + mirrors: + - localai/localai-backends:latest-gpu-intel-sycl-f16-crispasr +- !!merge <<: *crispasr + name: "vulkan-crispasr" + uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-vulkan-crispasr" + mirrors: + - localai/localai-backends:latest-gpu-vulkan-crispasr +- !!merge <<: *crispasr + name: "vulkan-crispasr-development" + uri: "quay.io/go-skynet/local-ai-backends:master-gpu-vulkan-crispasr" + mirrors: + - localai/localai-backends:master-gpu-vulkan-crispasr +- !!merge <<: *crispasr + name: "metal-crispasr" + uri: "quay.io/go-skynet/local-ai-backends:latest-metal-darwin-arm64-crispasr" + mirrors: + - localai/localai-backends:latest-metal-darwin-arm64-crispasr +- !!merge <<: *crispasr + name: "metal-crispasr-development" + uri: "quay.io/go-skynet/local-ai-backends:master-metal-darwin-arm64-crispasr" + mirrors: + - localai/localai-backends:master-metal-darwin-arm64-crispasr +- !!merge <<: *crispasr + name: "cuda12-crispasr-development" + uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-12-crispasr" + mirrors: + - localai/localai-backends:master-gpu-nvidia-cuda-12-crispasr +- !!merge <<: *crispasr + name: "rocm-crispasr-development" + uri: "quay.io/go-skynet/local-ai-backends:master-gpu-rocm-hipblas-crispasr" + mirrors: + - localai/localai-backends:master-gpu-rocm-hipblas-crispasr +- !!merge <<: *crispasr + name: "intel-sycl-f32-crispasr-development" + uri: "quay.io/go-skynet/local-ai-backends:master-gpu-intel-sycl-f32-crispasr" + mirrors: + - localai/localai-backends:master-gpu-intel-sycl-f32-crispasr +- !!merge <<: *crispasr + name: "intel-sycl-f16-crispasr-development" + uri: "quay.io/go-skynet/local-ai-backends:master-gpu-intel-sycl-f16-crispasr" + mirrors: + - localai/localai-backends:master-gpu-intel-sycl-f16-crispasr +- !!merge <<: *crispasr + name: "cuda13-crispasr" + uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-13-crispasr" + mirrors: + - localai/localai-backends:latest-gpu-nvidia-cuda-13-crispasr +- !!merge <<: *crispasr + name: "cuda13-crispasr-development" + uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-13-crispasr" + mirrors: + - localai/localai-backends:master-gpu-nvidia-cuda-13-crispasr ## parakeet-cpp - !!merge <<: *parakeetcpp name: "parakeet-cpp-development" From 96f5d6330e8cc80f72d269b3faf16fab82adc12a Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sat, 30 May 2026 22:59:55 +0000 Subject: [PATCH 06/29] ci(crispasr): bump CRISPASR_VERSION via bump_deps workflow Track CrispStrobe/CrispASR main branch and bump CRISPASR_VERSION in backend/go/crispasr/Makefile. Signed-off-by: Ettore Di Giacinto --- .github/workflows/bump_deps.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/bump_deps.yaml b/.github/workflows/bump_deps.yaml index 95612ae5bfbc..5f1ac0c21525 100644 --- a/.github/workflows/bump_deps.yaml +++ b/.github/workflows/bump_deps.yaml @@ -30,6 +30,10 @@ jobs: variable: "WHISPER_CPP_VERSION" branch: "master" file: "backend/go/whisper/Makefile" + - repository: "CrispStrobe/CrispASR" + variable: "CRISPASR_VERSION" + branch: "main" + file: "backend/go/crispasr/Makefile" - repository: "mudler/parakeet.cpp" variable: "PARAKEET_VERSION" branch: "master" From 20275ec669aef80b175c479d971005d0f39b72f9 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sat, 30 May 2026 23:01:52 +0000 Subject: [PATCH 07/29] build(crispasr): don't wire fixture-gated test into test-extra Mirror the whisper Go backend: its AudioTranscription test is gated on model/audio fixtures and skips in CI, so building crispasr (the heaviest ggml compile in the tree) inside the unit-test lane adds a long compile for zero coverage. The backend image build in backend-matrix.yml remains the authoritative compile check. Signed-off-by: Ettore Di Giacinto --- Makefile | 2 -- 1 file changed, 2 deletions(-) diff --git a/Makefile b/Makefile index 67c901efbfab..5ea1db5bd1f7 100644 --- a/Makefile +++ b/Makefile @@ -559,7 +559,6 @@ prepare-test-extra: protogen-python $(MAKE) -C backend/python/speaker-recognition $(MAKE) -C backend/rust/kokoros kokoros-grpc $(MAKE) -C backend/go/rfdetr-cpp - $(MAKE) -C backend/go/crispasr test-extra: prepare-test-extra $(MAKE) -C backend/python/transformers test @@ -587,7 +586,6 @@ test-extra: prepare-test-extra $(MAKE) -C backend/python/speaker-recognition test $(MAKE) -C backend/rust/kokoros test $(MAKE) -C backend/go/rfdetr-cpp test - $(MAKE) -C backend/go/crispasr test ## ## End-to-end gRPC tests that exercise a built backend container image. From c486e39481b6e00b24f77363a69604dc4d914d6a Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sat, 30 May 2026 23:05:36 +0000 Subject: [PATCH 08/29] ci(crispasr): add darwin metal build entry (mirror whisper) The metal-crispasr gallery entries and capabilities.metal mapping reference -metal-darwin-arm64-crispasr, which is only produced by an includeDarwin entry. Mirror whisper's darwin metal entry so the tag actually gets built. Signed-off-by: Ettore Di Giacinto --- .github/backend-matrix.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/backend-matrix.yml b/.github/backend-matrix.yml index 1ba8204b8a2d..5d0e21dfd24b 100644 --- a/.github/backend-matrix.yml +++ b/.github/backend-matrix.yml @@ -4272,6 +4272,10 @@ includeDarwin: tag-suffix: "-metal-darwin-arm64-whisper" build-type: "metal" lang: "go" + - backend: "crispasr" + tag-suffix: "-metal-darwin-arm64-crispasr" + build-type: "metal" + lang: "go" - backend: "parakeet-cpp" tag-suffix: "-metal-darwin-arm64-parakeet-cpp" build-type: "metal" From 47e022f864934d6ba4217eb8eb0de50502a4b7f4 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sat, 30 May 2026 23:08:53 +0000 Subject: [PATCH 09/29] ci(crispasr): place hipblas matrix entry next to whisper twin Signed-off-by: Ettore Di Giacinto --- .github/backend-matrix.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/backend-matrix.yml b/.github/backend-matrix.yml index 5d0e21dfd24b..9d7e36259a8d 100644 --- a/.github/backend-matrix.yml +++ b/.github/backend-matrix.yml @@ -3117,7 +3117,6 @@ include: dockerfile: "./backend/Dockerfile.golang" context: "./" ubuntu-version: '2404' - # parakeet-cpp - build-type: 'hipblas' cuda-major-version: "" cuda-minor-version: "" From d2579c0c60c82ceff0d00a31965e7d2cb6e841ca Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sat, 30 May 2026 23:10:59 +0000 Subject: [PATCH 10/29] feat(crispasr): register crispasr as pref-only ASR backend + test Signed-off-by: Ettore Di Giacinto --- core/http/endpoints/localai/backend.go | 1 + core/http/endpoints/localai/backend_test.go | 1 + 2 files changed, 2 insertions(+) diff --git a/core/http/endpoints/localai/backend.go b/core/http/endpoints/localai/backend.go index da5808c26388..cbda648d69d2 100644 --- a/core/http/endpoints/localai/backend.go +++ b/core/http/endpoints/localai/backend.go @@ -31,6 +31,7 @@ var knownPrefOnlyBackends = []schema.KnownBackend{ {Name: "mlx-vlm", Modality: "text", AutoDetect: false, Description: "MLX vision-language models (preference-only)"}, // ASR {Name: "whisperx", Modality: "asr", AutoDetect: false, Description: "WhisperX transcription (preference-only)"}, + {Name: "crispasr", Modality: "asr", AutoDetect: false, Description: "CrispASR multi-architecture transcription (preference-only)"}, // TTS {Name: "kokoros", Modality: "tts", AutoDetect: false, Description: "Kokoros TTS (preference-only)"}, {Name: "qwen-tts", Modality: "tts", AutoDetect: false, Description: "Qwen TTS (preference-only)"}, diff --git a/core/http/endpoints/localai/backend_test.go b/core/http/endpoints/localai/backend_test.go index 4a7a9497cac7..0c21bb7b4f6a 100644 --- a/core/http/endpoints/localai/backend_test.go +++ b/core/http/endpoints/localai/backend_test.go @@ -140,6 +140,7 @@ var _ = Describe("Backend Endpoints", func() { expectPrefOnly("trl", "text") expectPrefOnly("mlx-vlm", "text") expectPrefOnly("whisperx", "asr") + expectPrefOnly("crispasr", "asr") expectPrefOnly("kokoros", "tts") expectPrefOnly("qwen-tts", "tts") expectPrefOnly("qwen3-tts-cpp", "tts") From 0f3721dfb4d2496271c906ed5a8f2442b260e431 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sat, 30 May 2026 23:14:39 +0000 Subject: [PATCH 11/29] test(crispasr): port whisper behavioral suite (cancellation + streaming) Signed-off-by: Ettore Di Giacinto --- backend/go/crispasr/gocrispasr_test.go | 174 +++++++++++++++++++++++++ 1 file changed, 174 insertions(+) create mode 100644 backend/go/crispasr/gocrispasr_test.go diff --git a/backend/go/crispasr/gocrispasr_test.go b/backend/go/crispasr/gocrispasr_test.go new file mode 100644 index 000000000000..87fdda45a02e --- /dev/null +++ b/backend/go/crispasr/gocrispasr_test.go @@ -0,0 +1,174 @@ +package main + +import ( + "context" + "os" + "strings" + "sync" + "testing" + "time" + + "github.com/ebitengine/purego" + pb "github.com/mudler/LocalAI/pkg/grpc/proto" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func TestCrispASR(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "CrispASR Backend Suite") +} + +var ( + libLoadOnce sync.Once + libLoadErr error +) + +// ensureLibLoaded mirrors main.go's bootstrap so a Go test can drive the +// bridge without spinning up the gRPC server. Skips the current spec when the +// shared library isn't present (e.g. running before `make backends/whisper`). +func ensureLibLoaded() { + libLoadOnce.Do(func() { + libName := os.Getenv("CRISPASR_LIBRARY") + if libName == "" { + libName = "./libgocrispasr-fallback.so" + } + if _, err := os.Stat(libName); err != nil { + libLoadErr = err + return + } + gosd, err := purego.Dlopen(libName, purego.RTLD_NOW|purego.RTLD_GLOBAL) + if err != nil { + libLoadErr = err + return + } + purego.RegisterLibFunc(&CppLoadModel, gosd, "load_model") + purego.RegisterLibFunc(&CppTranscribe, gosd, "transcribe") + purego.RegisterLibFunc(&CppGetSegmentText, gosd, "get_segment_text") + purego.RegisterLibFunc(&CppGetSegmentStart, gosd, "get_segment_t0") + purego.RegisterLibFunc(&CppGetSegmentEnd, gosd, "get_segment_t1") + purego.RegisterLibFunc(&CppNTokens, gosd, "n_tokens") + purego.RegisterLibFunc(&CppGetTokenID, gosd, "get_token_id") + purego.RegisterLibFunc(&CppGetSegmentSpeakerTurnNext, gosd, "get_segment_speaker_turn_next") + purego.RegisterLibFunc(&CppSetAbort, gosd, "set_abort") + purego.RegisterLibFunc(&CppSetNewSegmentCallback, gosd, "set_new_segment_callback") + }) + if libLoadErr != nil { + Skip("whisper library not loadable: " + libLoadErr.Error()) + } +} + +// fixturesOrSkip returns the model + audio paths or skips the spec if either +// env var is unset. The test never runs in default CI — it requires a real +// whisper model and a long audio file (~3 minutes) on disk. +func fixturesOrSkip() (string, string) { + modelPath := os.Getenv("CRISPASR_MODEL_PATH") + audioPath := os.Getenv("CRISPASR_AUDIO_PATH") + if modelPath == "" || audioPath == "" { + Skip("set WHISPER_MODEL_PATH and WHISPER_AUDIO_PATH to run this spec") + } + return modelPath, audioPath +} + +var _ = Describe("CrispASR", func() { + Context("AudioTranscription cancellation", func() { + It("returns codes.Canceled and resets the abort flag for the next call", func() { + modelPath, audioPath := fixturesOrSkip() + ensureLibLoaded() + + w := &CrispASR{} + Expect(w.Load(&pb.ModelOptions{ModelFile: modelPath})).To(Succeed()) + + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() + + start := time.Now() + _, err := w.AudioTranscription(ctx, &pb.TranscriptRequest{ + Dst: audioPath, + Threads: 4, + Language: "en", + }) + elapsed := time.Since(start) + + Expect(err).To(HaveOccurred(), "transcription completed in %s without cancel — try a longer audio file", elapsed) + st, ok := status.FromError(err) + Expect(ok).To(BeTrue(), "expected gRPC status error, got %v", err) + Expect(st.Code()).To(Equal(codes.Canceled), "expected codes.Canceled, got %v", err) + Expect(elapsed).To(BeNumerically("<", 5*time.Second), "cancellation took %s, expected <5s", elapsed) + + // Subsequent transcription must succeed — proves g_abort reset. + res, err := w.AudioTranscription(context.Background(), &pb.TranscriptRequest{ + Dst: audioPath, + Threads: 4, + Language: "en", + }) + Expect(err).ToNot(HaveOccurred(), "post-cancel transcription failed") + Expect(res.Text).ToNot(BeEmpty(), "post-cancel transcription returned empty text") + }) + }) + + Context("AudioTranscriptionStream", func() { + It("emits multiple deltas progressively for a multi-segment clip", func() { + modelPath, audioPath := fixturesOrSkip() + ensureLibLoaded() + + // The streaming method dispatches through the package-level + // goNewSegmentCb. main.go normally builds it; in this test + // process main() is never called, so build it here lazily. + // purego.NewCallback returns a stable pointer; calling it once + // per process is correct. + if goNewSegmentCb == 0 { + goNewSegmentCb = purego.NewCallback(onNewSegment) + } + + w := &CrispASR{} + Expect(w.Load(&pb.ModelOptions{ModelFile: modelPath})).To(Succeed()) + + results := make(chan *pb.TranscriptStreamResponse, 64) + done := make(chan error, 1) + go func() { + done <- w.AudioTranscriptionStream(context.Background(), &pb.TranscriptRequest{ + Dst: audioPath, + Threads: 4, + Language: "en", + Stream: true, + }, results) + }() + + var deltas []string + var assembled strings.Builder + var finalText string + var finalSegmentCount int + for chunk := range results { + if d := chunk.GetDelta(); d != "" { + deltas = append(deltas, d) + assembled.WriteString(d) + } + if final := chunk.GetFinalResult(); final != nil { + finalText = final.GetText() + finalSegmentCount = len(final.GetSegments()) + } + } + Expect(<-done).ToNot(HaveOccurred()) + + // The whisper-specific bar: real streaming via new_segment_callback + // fires once per decoded segment, so a multi-segment clip MUST + // produce >=2 delta events. A faked-streaming impl (run + // whisper_full to completion, then walk the segment list) would + // also pass len(deltas) >= 1, which is why the generic e2e spec + // is not strict enough. + Expect(len(deltas)).To(BeNumerically(">=", 2), + "expected multiple deltas from a multi-segment clip, got %d (assembled=%q)", + len(deltas), assembled.String()) + Expect(finalSegmentCount).To(BeNumerically(">=", 2), + "expected final to carry multiple segments") + Expect(assembled.String()).To(Equal(finalText), + "concat(deltas) must equal final.Text") + }) + }) +}) From 7c1b4fe594d40e6dfe8872b0831992b9cb823ecc Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sat, 30 May 2026 23:15:27 +0000 Subject: [PATCH 12/29] test(crispasr): fix skip message env var names to CRISPASR_* Signed-off-by: Ettore Di Giacinto --- backend/go/crispasr/gocrispasr_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/go/crispasr/gocrispasr_test.go b/backend/go/crispasr/gocrispasr_test.go index 87fdda45a02e..d9b263235f2e 100644 --- a/backend/go/crispasr/gocrispasr_test.go +++ b/backend/go/crispasr/gocrispasr_test.go @@ -67,7 +67,7 @@ func fixturesOrSkip() (string, string) { modelPath := os.Getenv("CRISPASR_MODEL_PATH") audioPath := os.Getenv("CRISPASR_AUDIO_PATH") if modelPath == "" || audioPath == "" { - Skip("set WHISPER_MODEL_PATH and WHISPER_AUDIO_PATH to run this spec") + Skip("set CRISPASR_MODEL_PATH and CRISPASR_AUDIO_PATH to run this spec") } return modelPath, audioPath } From 88099946f0e3d4272dfbbd2df0f890d255b9763d Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sat, 30 May 2026 23:58:26 +0000 Subject: [PATCH 13/29] feat(crispasr): switch shim to crispasr_session_* multi-architecture API The shim used whisper_full(), which in CrispASR is the whisper-only path: libcrispasr only transcribes Whisper GGUFs through it. Multi-architecture transcription (Parakeet, Voxtral, Qwen3-ASR, Canary, Granite, FunASR, Paraformer, SenseVoice, ...) goes through the crispasr_session_* C-ABI, which auto-detects the architecture from the GGUF and dispatches to the matching backend. Rewrite the C shim around crispasr_session_open / _transcribe_lang / _result_* and add get_backend() so the selected backend is logged. load_model now takes a threads param (session_open binds n_threads at open). The session result is segment+word based with no token IDs and no per-decode callback, so drop n_tokens / get_token_id / get_segment_speaker_turn_next / set_new_segment_callback. set_abort is kept for API parity but is best-effort: the session transcribe is blocking with no abort hook. Update the purego bindings and gocrispasr.go to match: tokens are left empty, speaker-turn handling is removed, and AudioTranscriptionStream emits one delta per non-empty segment after the blocking decode returns (no progressive streaming via the session API), preserving the concat(deltas) == final.Text invariant. crispasr_session_set_translate is exported by libcrispasr but not declared in crispasr.h, so it is forward-declared in the shim alongside the open/transcribe/result functions. Signed-off-by: Ettore Di Giacinto --- backend/go/crispasr/cpp/crispasr_shim.cpp | 158 +++++++++---------- backend/go/crispasr/cpp/crispasr_shim.h | 19 +-- backend/go/crispasr/gocrispasr.go | 177 ++++++---------------- backend/go/crispasr/main.go | 7 +- 4 files changed, 123 insertions(+), 238 deletions(-) diff --git a/backend/go/crispasr/cpp/crispasr_shim.cpp b/backend/go/crispasr/cpp/crispasr_shim.cpp index 2222f38a3c8e..b1b23231ff99 100644 --- a/backend/go/crispasr/cpp/crispasr_shim.cpp +++ b/backend/go/crispasr/cpp/crispasr_shim.cpp @@ -4,42 +4,37 @@ #include #include +// Opaque session types. crispasr.h declares `struct crispasr_session;` but not +// the result type nor the open/transcribe/result accessors — those are +// CA_EXPORT extern "C" symbols in src/crispasr_c_api.cpp, so we forward-declare +// exactly the ones we use. Signatures verified against +// sources/CrispASR/src/crispasr_c_api.cpp. +struct crispasr_session_result; +extern "C" { +crispasr_session *crispasr_session_open(const char *model_path, int n_threads); +void crispasr_session_close(crispasr_session *s); +const char *crispasr_session_backend(crispasr_session *s); +int crispasr_session_set_translate(crispasr_session *s, int enable); +crispasr_session_result *crispasr_session_transcribe_lang( + crispasr_session *s, const float *pcm, int n_samples, const char *language); +int crispasr_session_result_n_segments(crispasr_session_result *r); +const char *crispasr_session_result_segment_text(crispasr_session_result *r, + int i); +int64_t crispasr_session_result_segment_t0(crispasr_session_result *r, int i); +int64_t crispasr_session_result_segment_t1(crispasr_session_result *r, int i); +void crispasr_session_result_free(crispasr_session_result *r); +} + +static crispasr_session *g_session = nullptr; +static crispasr_session_result *g_result = nullptr; + static struct whisper_vad_context *vctx; -static struct whisper_context *ctx; static std::vector flat_segs; static std::atomic g_abort{0}; -static std::atomic g_go_new_segment_cb{0}; -static std::atomic g_go_new_segment_user_data{0}; - -static bool abort_cb(void * /*user_data*/) { - return g_abort.load(std::memory_order_relaxed) != 0; -} - -static void new_segment_cb(struct whisper_context *cb_ctx, - struct whisper_state * /*state*/, int n_new, - void * /*user_data*/) { - uintptr_t go_cb = g_go_new_segment_cb.load(std::memory_order_relaxed); - if (go_cb == 0) { - return; - } - int total = whisper_full_n_segments(cb_ctx); - int idx_first = total - n_new; - if (idx_first < 0) { - idx_first = 0; - } - uintptr_t ud = g_go_new_segment_user_data.load(std::memory_order_relaxed); - reinterpret_cast(go_cb)(idx_first, n_new, ud); -} - extern "C" void set_abort(int v) { - g_abort.store(v, std::memory_order_relaxed); -} - -extern "C" void set_new_segment_callback(uintptr_t cb_ptr, uintptr_t user_data) { - g_go_new_segment_cb.store(cb_ptr, std::memory_order_relaxed); - g_go_new_segment_user_data.store(user_data, std::memory_order_relaxed); + g_abort.store(v, std::memory_order_relaxed); } static void ggml_log_cb(enum ggml_log_level level, const char *log, @@ -73,18 +68,18 @@ static void ggml_log_cb(enum ggml_log_level level, const char *log, fflush(stderr); } -int load_model(const char *const model_path) { +int load_model(const char *const model_path, int threads) { whisper_log_set(ggml_log_cb, nullptr); ggml_backend_load_all(); - struct whisper_context_params cparams = whisper_context_default_params(); - - ctx = whisper_init_from_file_with_params(model_path, cparams); - if (ctx == nullptr) { - fprintf(stderr, "error: Also failed to init model as transcriber\n"); + g_session = crispasr_session_open(model_path, threads); + if (g_session == nullptr) { + fprintf(stderr, "error: failed to open CrispASR session for model\n"); return 1; } + fprintf(stderr, "info: CrispASR backend selected: %s\n", + crispasr_session_backend(g_session)); return 0; } @@ -141,69 +136,66 @@ int vad(float pcmf32[], size_t pcmf32_len, float **segs_out, return 0; } -int transcribe(uint32_t threads, char *lang, bool translate, bool tdrz, - float pcmf32[], size_t pcmf32_len, size_t *segs_out_len, char *prompt) { - whisper_full_params wparams = - whisper_full_default_params(CRISPASR_SAMPLING_GREEDY); - - wparams.n_threads = threads; - if (*lang != '\0') - wparams.language = lang; - else { - wparams.language = nullptr; +// threads, diarize and prompt are accepted for Go-side API parity but unused +// in Phase 1: the thread count is fixed at session open, and diarization and +// the initial prompt are separate CrispASR features not yet wired through the +// session ASR path. +int transcribe(uint32_t threads, char *lang, bool translate, bool diarize, + float pcmf32[], size_t pcmf32_len, size_t *segs_out_len, + char *prompt) { + (void)threads; + (void)diarize; + (void)prompt; + + if (!g_session) { + return 1; } - wparams.translate = translate; - wparams.debug_mode = true; - wparams.print_progress = true; - wparams.tdrz_enable = tdrz; - wparams.initial_prompt = prompt; - - // Reset stale abort flag from any prior cancelled call, then install the - // ggml abort hook so a subsequent set_abort(1) from Go aborts the next - // compute graph step. + // Reset stale abort flag from any prior cancelled call. set_abort remains + // best-effort: the session transcribe call is blocking and exposes no abort + // hook, so a mid-decode abort cannot interrupt it. g_abort.store(0, std::memory_order_relaxed); - // Only install the new-segment callback when streaming is requested - // (Go side calls set_new_segment_callback before transcribe()). Leaving - // it always-on is harmless but adds a function-pointer dispatch per - // segment for the offline path. - if (g_go_new_segment_cb.load(std::memory_order_relaxed) != 0) { - wparams.new_segment_callback = new_segment_cb; - wparams.new_segment_callback_user_data = nullptr; - } - wparams.abort_callback = abort_cb; - wparams.abort_callback_user_data = nullptr; - fprintf(stderr, "info: Enable tdrz: %d\n", tdrz); - fprintf(stderr, "info: Initial prompt: \"%s\"\n", prompt); + crispasr_session_set_translate(g_session, translate ? 1 : 0); - if (whisper_full(ctx, wparams, pcmf32, pcmf32_len)) { - if (g_abort.load(std::memory_order_relaxed)) { - return 2; // aborted by client - } + if (g_result) { + crispasr_session_result_free(g_result); + g_result = nullptr; + } + + const char *language = (lang && *lang) ? lang : nullptr; + g_result = crispasr_session_transcribe_lang(g_session, pcmf32, (int)pcmf32_len, + language); + if (!g_result) { fprintf(stderr, "error: transcription failed\n"); return 1; } - *segs_out_len = whisper_full_n_segments(ctx); - + *segs_out_len = crispasr_session_result_n_segments(g_result); return 0; } const char *get_segment_text(int i) { - return whisper_full_get_segment_text(ctx, i); + if (!g_result) { + return ""; + } + return crispasr_session_result_segment_text(g_result, i); } -int64_t get_segment_t0(int i) { return whisper_full_get_segment_t0(ctx, i); } - -int64_t get_segment_t1(int i) { return whisper_full_get_segment_t1(ctx, i); } - -int n_tokens(int i) { return whisper_full_n_tokens(ctx, i); } +int64_t get_segment_t0(int i) { + if (!g_result) { + return 0; + } + return crispasr_session_result_segment_t0(g_result, i); +} -int32_t get_token_id(int i, int j) { - return whisper_full_get_token_id(ctx, i, j); +int64_t get_segment_t1(int i) { + if (!g_result) { + return 0; + } + return crispasr_session_result_segment_t1(g_result, i); } -bool get_segment_speaker_turn_next(int i) { - return whisper_full_get_segment_speaker_turn_next(ctx, i); +const char *get_backend(void) { + return g_session ? crispasr_session_backend(g_session) : ""; } diff --git a/backend/go/crispasr/cpp/crispasr_shim.h b/backend/go/crispasr/cpp/crispasr_shim.h index 4bac87eef4c8..fa01097423cb 100644 --- a/backend/go/crispasr/cpp/crispasr_shim.h +++ b/backend/go/crispasr/cpp/crispasr_shim.h @@ -2,29 +2,16 @@ #include extern "C" { -int load_model(const char *const model_path); +int load_model(const char *const model_path, int threads); int load_model_vad(const char *const model_path); int vad(float pcmf32[], size_t pcmf32_size, float **segs_out, size_t *segs_out_len); -int transcribe(uint32_t threads, char *lang, bool translate, bool tdrz, +int transcribe(uint32_t threads, char *lang, bool translate, bool diarize, float pcmf32[], size_t pcmf32_len, size_t *segs_out_len, char *prompt); const char *get_segment_text(int i); int64_t get_segment_t0(int i); int64_t get_segment_t1(int i); -int n_tokens(int i); -int32_t get_token_id(int i, int j); -bool get_segment_speaker_turn_next(int i); +const char *get_backend(void); void set_abort(int v); - -// Function pointer from Go (returned by purego.NewCallback). Invoked once -// per new-segment event during whisper_full(). The callback runs on the -// decode thread - if Go blocks (slow gRPC consumer), the decode blocks -// too. That is the intended backpressure path. -typedef void (*go_new_segment_cb)(int idx_first, int n_new, uintptr_t user_data); - -// Install the callback used by the next transcribe() call. Pass cb=0 to -// clear. user_data is opaque to C; the Go side uses it to look up -// per-call state. -void set_new_segment_callback(uintptr_t cb_ptr, uintptr_t user_data); } diff --git a/backend/go/crispasr/gocrispasr.go b/backend/go/crispasr/gocrispasr.go index 74a98879bb34..17d93502aa1c 100644 --- a/backend/go/crispasr/gocrispasr.go +++ b/backend/go/crispasr/gocrispasr.go @@ -7,7 +7,6 @@ import ( "path/filepath" "strings" "sync" - "sync/atomic" "unsafe" "github.com/go-audio/wav" @@ -19,94 +18,17 @@ import ( ) var ( - CppLoadModel func(modelPath string) int - CppLoadModelVAD func(modelPath string) int - CppVAD func(pcmf32 []float32, pcmf32Size uintptr, segsOut unsafe.Pointer, segsOutLen unsafe.Pointer) int - CppTranscribe func(threads uint32, lang string, translate bool, diarize bool, pcmf32 []float32, pcmf32Len uintptr, segsOutLen unsafe.Pointer, prompt string) int - CppGetSegmentText func(i int) string - CppGetSegmentStart func(i int) int64 - CppGetSegmentEnd func(i int) int64 - CppNTokens func(i int) int - CppGetTokenID func(i int, j int) int - CppGetSegmentSpeakerTurnNext func(i int) bool - CppSetAbort func(v int) - // Set by main.go via purego.RegisterLibFunc. Installs (or clears with cb=0) - // the C-side trampoline that whisper.cpp invokes per new segment. - CppSetNewSegmentCallback func(cbPtr uintptr, userData uintptr) + CppLoadModel func(modelPath string, threads int) int + CppLoadModelVAD func(modelPath string) int + CppVAD func(pcmf32 []float32, pcmf32Size uintptr, segsOut unsafe.Pointer, segsOutLen unsafe.Pointer) int + CppTranscribe func(threads uint32, lang string, translate bool, diarize bool, pcmf32 []float32, pcmf32Len uintptr, segsOutLen unsafe.Pointer, prompt string) int + CppGetSegmentText func(i int) string + CppGetSegmentStart func(i int) int64 + CppGetSegmentEnd func(i int) int64 + CppGetBackend func() string + CppSetAbort func(v int) ) -// streamCallStates maps per-AudioTranscriptionStream call IDs to the -// state the Go callback needs to emit deltas. Only one entry is ever -// live today (base.SingleThread), but the map shape mirrors -// sherpa-onnx's TTS callback registry and survives a future SingleThread -// removal without a contract change. -var ( - streamCallStates sync.Map // uint64 -> *streamCallState - streamCallSeq atomic.Uint64 - goNewSegmentCb uintptr // purego.NewCallback(onNewSegment) result; set in main.go at boot -) - -type streamCallState struct { - results chan *pb.TranscriptStreamResponse - diarize bool - // nextIdx tracks how many segments we've already emitted. The C - // trampoline passes idx_first = total - n_new, but we walk from - // nextIdx to (idx_first + n_new) defensively in case whisper.cpp ever - // coalesces multiple commits into a single callback invocation. - nextIdx int - // assembled mirrors the literal concat of every Delta sent on results. - // We reuse it as the final TranscriptResult.Text so the e2e - // invariant `final.Text == concat(deltas)` holds exactly. Written from - // the cgo decode thread inside onNewSegment and read by the streaming - // method after CppTranscribe returns; the cgo boundary provides the - // happens-before edge. - assembled strings.Builder -} - -// onNewSegment is the Go side of the C trampoline declared in -// crispasr_shim.cpp:new_segment_cb. Whisper.cpp invokes it once per -// new-segment event during whisper_full(). Reads segment text via the -// existing CppGetSegment* getters (safe to call against the singleton -// ctx; whisper.cpp is the only writer and it has already published the -// segments by the time this fires). -// -// Sends deltas synchronously: if the channel is full, this blocks the -// whisper decode thread. That's the intended backpressure path - -// dropping deltas would break the concat(deltas) == final.Text invariant -// the e2e suite asserts. -func onNewSegment(idxFirst int32, nNew int32, userData uintptr) { - v, ok := streamCallStates.Load(uint64(userData)) - if !ok { - return // call already torn down (race with cancel + cb fire) - } - state := v.(*streamCallState) - end := int(idxFirst) + int(nNew) - for i := state.nextIdx; i < end; i++ { - txt := strings.ToValidUTF8(strings.Clone(CppGetSegmentText(i)), "�") - txt = strings.TrimSpace(txt) - if state.diarize && CppGetSegmentSpeakerTurnNext(i) { - txt += " [SPEAKER_TURN]" - } - if txt == "" { - state.nextIdx = i + 1 - continue - } - // Prefix subsequent deltas with a single space so the assembled - // stream reads as one space-joined transcript. The first delta has - // no leading space, otherwise concat(deltas) would not match - // final.Text and the e2e invariant would break. - var delta string - if state.assembled.Len() == 0 { - delta = txt - } else { - delta = " " + txt - } - state.results <- &pb.TranscriptStreamResponse{Delta: delta} - state.assembled.WriteString(delta) - state.nextIdx = i + 1 - } -} - type CrispASR struct { base.SingleThread } @@ -130,10 +52,12 @@ func (w *CrispASR) Load(opts *pb.ModelOptions) error { return nil } - if ret := CppLoadModel(opts.ModelFile); ret != 0 { + if ret := CppLoadModel(opts.ModelFile, int(opts.Threads)); ret != 0 { return fmt.Errorf("Failed to load CrispASR transcription model") } + fmt.Fprintf(os.Stderr, "CrispASR backend selected: %s\n", CppGetBackend()) + return nil } @@ -244,24 +168,17 @@ func (w *CrispASR) AudioTranscription(ctx context.Context, opts *pb.TranscriptRe // segment start/end conversion factor taken from https://github.com/ggml-org/whisper.cpp/blob/master/examples/cli/cli.cpp#L895 s := CppGetSegmentStart(i) * (10000000) t := CppGetSegmentEnd(i) * (10000000) - // whisper.cpp can emit bytes that aren't valid UTF-8 (e.g. a multibyte - // codepoint split across token boundaries); protobuf string fields - // reject those at marshal time. Scrub before the value escapes cgo. + // The session result can emit bytes that aren't valid UTF-8 (e.g. a + // multibyte codepoint split across token boundaries); protobuf string + // fields reject those at marshal time. Scrub before the value escapes + // cgo. The session result is segment+word based and exposes no token + // IDs, so Tokens is left empty. txt := strings.ToValidUTF8(strings.Clone(CppGetSegmentText(i)), "�") - tokens := make([]int32, CppNTokens(i)) - - if opts.Diarize && CppGetSegmentSpeakerTurnNext(i) { - txt += " [SPEAKER_TURN]" - } - for j := range tokens { - tokens[j] = int32(CppGetTokenID(i, j)) - } segment := &pb.TranscriptSegment{ Id: int32(i), Text: txt, Start: s, End: t, - Tokens: tokens, } segments = append(segments, segment) @@ -277,11 +194,13 @@ func (w *CrispASR) AudioTranscription(ctx context.Context, opts *pb.TranscriptRe }, nil } -// AudioTranscriptionStream runs whisper_full() and emits deltas via -// whisper.cpp's new_segment_callback as segments are decoded, then a -// final TranscriptResult. The offline AudioTranscription is unchanged; -// both paths share whisper's single-instance ctx and the SingleThread -// concurrency model. +// AudioTranscriptionStream runs the session transcribe to completion and then +// emits one delta per non-empty segment, followed by a final TranscriptResult. +// Progressive/real-time streaming isn't available via the session API (there +// is no per-decode callback), so deltas are emitted per-segment after the +// blocking decode returns rather than as segments are produced. The offline +// AudioTranscription is unchanged; both paths share the session and the +// SingleThread concurrency model. func (w *CrispASR) AudioTranscriptionStream(ctx context.Context, opts *pb.TranscriptRequest, results chan *pb.TranscriptStreamResponse) error { defer close(results) @@ -317,23 +236,9 @@ func (w *CrispASR) AudioTranscriptionStream(ctx context.Context, opts *pb.Transc duration = float32(len(data)) / float32(buf.Format.SampleRate) } - // Register per-call state and install the C-side callback. defer - // teardown so even a panic clears the C pointer (otherwise a stale - // callback fires on the next AudioTranscription call). - callID := streamCallSeq.Add(1) - state := &streamCallState{ - results: results, - diarize: opts.Diarize, - } - streamCallStates.Store(callID, state) - CppSetNewSegmentCallback(goNewSegmentCb, uintptr(callID)) - defer func() { - CppSetNewSegmentCallback(0, 0) - streamCallStates.Delete(callID) - }() - // Same abort-watcher pattern as AudioTranscription. Joined synchronously // so a late CppSetAbort(1) cannot fire after this function returns. + // Best-effort only: the session transcribe is blocking with no abort hook. done := make(chan struct{}) var wg sync.WaitGroup wg.Add(1) @@ -360,33 +265,39 @@ func (w *CrispASR) AudioTranscriptionStream(ctx context.Context, opts *pb.Transc return fmt.Errorf("Failed Transcribe") } - // Build the final TranscriptResult. Segments[] mirrors the offline - // path so the SSE done event carries the same per-segment shape. - // final.Text reuses the assembled stream so concat(deltas) == final.Text - // holds exactly, matching the e2e contract. + // Walk the segments once: emit a delta per non-empty segment and build the + // final TranscriptResult.Segments alongside. The first delta has no leading + // space and subsequent ones are prefixed with a single space, so + // concat(deltas) == final.Text exactly, matching the e2e contract. segments := []*pb.TranscriptSegment{} + var assembled strings.Builder for i := range int(segsLen) { s := CppGetSegmentStart(i) * 10000000 t := CppGetSegmentEnd(i) * 10000000 txt := strings.ToValidUTF8(strings.Clone(CppGetSegmentText(i)), "�") - tokens := make([]int32, CppNTokens(i)) - if opts.Diarize && CppGetSegmentSpeakerTurnNext(i) { - txt += " [SPEAKER_TURN]" - } - for j := range tokens { - tokens[j] = int32(CppGetTokenID(i, j)) - } segments = append(segments, &pb.TranscriptSegment{ Id: int32(i), Text: txt, Start: s, End: t, - Tokens: tokens, }) + + trimmed := strings.TrimSpace(txt) + if trimmed == "" { + continue + } + var delta string + if assembled.Len() == 0 { + delta = trimmed + } else { + delta = " " + trimmed + } + results <- &pb.TranscriptStreamResponse{Delta: delta} + assembled.WriteString(delta) } final := &pb.TranscriptResult{ Segments: segments, - Text: state.assembled.String(), + Text: assembled.String(), Language: opts.Language, Duration: duration, } diff --git a/backend/go/crispasr/main.go b/backend/go/crispasr/main.go index 8582a1ca8a91..4d0f2b92b2ab 100644 --- a/backend/go/crispasr/main.go +++ b/backend/go/crispasr/main.go @@ -37,19 +37,14 @@ func main() { {&CppGetSegmentText, "get_segment_text"}, {&CppGetSegmentStart, "get_segment_t0"}, {&CppGetSegmentEnd, "get_segment_t1"}, - {&CppNTokens, "n_tokens"}, - {&CppGetTokenID, "get_token_id"}, - {&CppGetSegmentSpeakerTurnNext, "get_segment_speaker_turn_next"}, + {&CppGetBackend, "get_backend"}, {&CppSetAbort, "set_abort"}, - {&CppSetNewSegmentCallback, "set_new_segment_callback"}, } for _, lf := range libFuncs { purego.RegisterLibFunc(lf.FuncPtr, lib, lf.Name) } - goNewSegmentCb = purego.NewCallback(onNewSegment) - flag.Parse() if err := grpc.StartServer(*addr, &CrispASR{}); err != nil { From 748df02a8cb4612768692444407e3d836f272f66 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sat, 30 May 2026 23:58:37 +0000 Subject: [PATCH 14/29] build(crispasr): link full CrispASR backend set for multi-arch support The shim's crispasr_session_* dispatch calls into the per-architecture backend libs (parakeet, voxtral, qwen3_asr, canary, funasr, paraformer, sensevoice, ...), which CrispASR builds as static archives. Linking only crispasr + ggml dead-stripped every backend object from the final module (nm backend-symbol count: 0), leaving a whisper-only .so. Link the same backend set as crispasr-cli so the static archives are pulled in. After this the module carries the backend symbols (nm count 407, .so grows from ~2.1MB to ~6.7MB) and the session API can dispatch to every compiled-in architecture. Also rewrite ${CMAKE_SOURCE_DIR}/examples/talk-llama to ${PROJECT_SOURCE_DIR}/... in the vendored src/CMakeLists.txt: CrispASR locates its vendored llama.cpp via ${CMAKE_SOURCE_DIR}, which is wrong when CrispASR is add_subdirectory'd (CMAKE_SOURCE_DIR points at this backend dir, not the CrispASR root). PROJECT_SOURCE_DIR is correct both standalone and as a subproject; the sed is idempotent. Signed-off-by: Ettore Di Giacinto --- backend/go/crispasr/CMakeLists.txt | 13 ++++++++++++- backend/go/crispasr/Makefile | 7 +++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/backend/go/crispasr/CMakeLists.txt b/backend/go/crispasr/CMakeLists.txt index d557ee981552..5ed6a9c3f0bb 100644 --- a/backend/go/crispasr/CMakeLists.txt +++ b/backend/go/crispasr/CMakeLists.txt @@ -9,7 +9,18 @@ add_library(gocrispasr MODULE cpp/crispasr_shim.cpp) target_include_directories(gocrispasr PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/sources/CrispASR/include ${CMAKE_CURRENT_SOURCE_DIR}/sources/CrispASR/ggml/include) -target_link_libraries(gocrispasr PRIVATE crispasr ggml) +# Link the same backend set as crispasr-cli (examples/cli/CMakeLists.txt) so +# the session API can dispatch to every compiled-in architecture, not just +# whisper. crispasr is the referencer; the backend static libs supply the +# per-architecture symbols; ggml is the math/runtime base. +target_link_libraries(gocrispasr PRIVATE + crispasr + parakeet canary canary_ctc cohere granite_speech granite_nle + voxtral voxtral4b qwen3_asr qwen3_tts orpheus chatterbox indextts + kokoro voxcpm2_tts m2m100 t5_translate wav2vec2-ggml vibevoice + silero-lid pyannote-seg funasr paraformer sensevoice + crisp_audio + ggml) if(CMAKE_CXX_COMPILER_ID MATCHES "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 9.0) target_link_libraries(gocrispasr PRIVATE stdc++fs) diff --git a/backend/go/crispasr/Makefile b/backend/go/crispasr/Makefile index ede5530c5641..3d57067b0fce 100644 --- a/backend/go/crispasr/Makefile +++ b/backend/go/crispasr/Makefile @@ -61,6 +61,13 @@ sources/CrispASR: git fetch origin && \ git checkout $(CRISPASR_VERSION) && \ git submodule update --init --recursive --depth 1 --single-branch + # CrispASR's src/CMakeLists.txt locates its vendored llama.cpp + # (crispasr-llama-core, used by the chat C-ABI) via ${CMAKE_SOURCE_DIR}, + # which assumes CrispASR is the top-level CMake project. We add_subdirectory + # it, so ${CMAKE_SOURCE_DIR} is THIS backend dir and the talk-llama sources + # aren't found. Rewrite to ${PROJECT_SOURCE_DIR} (the crispasr project root), + # which is correct both standalone and as a subproject. Idempotent. + sed -i 's#\$${CMAKE_SOURCE_DIR}/examples/talk-llama#\$${PROJECT_SOURCE_DIR}/examples/talk-llama#' sources/CrispASR/src/CMakeLists.txt # Detect OS UNAME_S := $(shell uname -s) From 6cd92c4e4fa9dcd7a2207b0e9c9f8efee52bebc8 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sat, 30 May 2026 23:58:46 +0000 Subject: [PATCH 15/29] test(crispasr): adapt suite to session API (blocking, no decode callback) Register the new symbol set (drop the removed token/speaker/callback funcs, add get_backend; load_model now takes 2 args). The session transcribe is blocking with no abort hook, so a mid-decode cancel can't interrupt it: change the cancellation spec to cancel the context before the call and assert codes.Canceled from the pre-call ctx.Err() check, dropping the <5s mid-decode timing assertion. The streaming spec still holds with per-segment post-decode emission (>=2 deltas, concat(deltas) == final.Text). Signed-off-by: Ettore Di Giacinto --- backend/go/crispasr/gocrispasr_test.go | 42 ++++++++------------------ 1 file changed, 12 insertions(+), 30 deletions(-) diff --git a/backend/go/crispasr/gocrispasr_test.go b/backend/go/crispasr/gocrispasr_test.go index d9b263235f2e..658d4081dffb 100644 --- a/backend/go/crispasr/gocrispasr_test.go +++ b/backend/go/crispasr/gocrispasr_test.go @@ -6,7 +6,6 @@ import ( "strings" "sync" "testing" - "time" "github.com/ebitengine/purego" pb "github.com/mudler/LocalAI/pkg/grpc/proto" @@ -49,11 +48,8 @@ func ensureLibLoaded() { purego.RegisterLibFunc(&CppGetSegmentText, gosd, "get_segment_text") purego.RegisterLibFunc(&CppGetSegmentStart, gosd, "get_segment_t0") purego.RegisterLibFunc(&CppGetSegmentEnd, gosd, "get_segment_t1") - purego.RegisterLibFunc(&CppNTokens, gosd, "n_tokens") - purego.RegisterLibFunc(&CppGetTokenID, gosd, "get_token_id") - purego.RegisterLibFunc(&CppGetSegmentSpeakerTurnNext, gosd, "get_segment_speaker_turn_next") + purego.RegisterLibFunc(&CppGetBackend, gosd, "get_backend") purego.RegisterLibFunc(&CppSetAbort, gosd, "set_abort") - purego.RegisterLibFunc(&CppSetNewSegmentCallback, gosd, "set_new_segment_callback") }) if libLoadErr != nil { Skip("whisper library not loadable: " + libLoadErr.Error()) @@ -74,32 +70,29 @@ func fixturesOrSkip() (string, string) { var _ = Describe("CrispASR", func() { Context("AudioTranscription cancellation", func() { - It("returns codes.Canceled and resets the abort flag for the next call", func() { + It("returns codes.Canceled on a pre-cancelled context and still succeeds afterwards", func() { modelPath, audioPath := fixturesOrSkip() ensureLibLoaded() w := &CrispASR{} Expect(w.Load(&pb.ModelOptions{ModelFile: modelPath})).To(Succeed()) + // The session transcribe is blocking and exposes no abort hook, so + // a mid-decode cancel can't interrupt it. The contract we can rely + // on is the pre-call ctx.Err() check: a context cancelled before + // the call must yield codes.Canceled without starting a decode. ctx, cancel := context.WithCancel(context.Background()) - go func() { - time.Sleep(100 * time.Millisecond) - cancel() - }() + cancel() - start := time.Now() _, err := w.AudioTranscription(ctx, &pb.TranscriptRequest{ Dst: audioPath, Threads: 4, Language: "en", }) - elapsed := time.Since(start) - - Expect(err).To(HaveOccurred(), "transcription completed in %s without cancel — try a longer audio file", elapsed) + Expect(err).To(HaveOccurred(), "expected pre-cancelled context to fail") st, ok := status.FromError(err) Expect(ok).To(BeTrue(), "expected gRPC status error, got %v", err) Expect(st.Code()).To(Equal(codes.Canceled), "expected codes.Canceled, got %v", err) - Expect(elapsed).To(BeNumerically("<", 5*time.Second), "cancellation took %s, expected <5s", elapsed) // Subsequent transcription must succeed — proves g_abort reset. res, err := w.AudioTranscription(context.Background(), &pb.TranscriptRequest{ @@ -117,15 +110,6 @@ var _ = Describe("CrispASR", func() { modelPath, audioPath := fixturesOrSkip() ensureLibLoaded() - // The streaming method dispatches through the package-level - // goNewSegmentCb. main.go normally builds it; in this test - // process main() is never called, so build it here lazily. - // purego.NewCallback returns a stable pointer; calling it once - // per process is correct. - if goNewSegmentCb == 0 { - goNewSegmentCb = purego.NewCallback(onNewSegment) - } - w := &CrispASR{} Expect(w.Load(&pb.ModelOptions{ModelFile: modelPath})).To(Succeed()) @@ -156,12 +140,10 @@ var _ = Describe("CrispASR", func() { } Expect(<-done).ToNot(HaveOccurred()) - // The whisper-specific bar: real streaming via new_segment_callback - // fires once per decoded segment, so a multi-segment clip MUST - // produce >=2 delta events. A faked-streaming impl (run - // whisper_full to completion, then walk the segment list) would - // also pass len(deltas) >= 1, which is why the generic e2e spec - // is not strict enough. + // One delta per non-empty segment is emitted after the blocking + // decode returns (the session API has no per-decode callback), so a + // multi-segment clip MUST produce >=2 delta events, and + // concat(deltas) MUST equal final.Text exactly. Expect(len(deltas)).To(BeNumerically(">=", 2), "expected multiple deltas from a multi-segment clip, got %d (assembled=%q)", len(deltas), assembled.String()) From 8dfe33b13df8ddc7b783bca70443d6a436992f15 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sun, 31 May 2026 00:09:50 +0000 Subject: [PATCH 16/29] feat(gallery): add CrispASR ASR model entries (-crispasr) Signed-off-by: Ettore Di Giacinto --- gallery/canary-crispasr.yaml | 7 + gallery/cohere-crispasr.yaml | 7 + gallery/data2vec-crispasr.yaml | 7 + gallery/fastconformer-ctc-crispasr.yaml | 7 + gallery/firered-asr-crispasr.yaml | 7 + gallery/fun-asr-mlt-nano-crispasr.yaml | 7 + gallery/funasr-crispasr.yaml | 7 + gallery/glm-asr-crispasr.yaml | 7 + gallery/granite-4.1-crispasr.yaml | 7 + gallery/granite-4.1-nar-crispasr.yaml | 7 + gallery/granite-4.1-plus-crispasr.yaml | 7 + gallery/granite-crispasr.yaml | 7 + gallery/hubert-crispasr.yaml | 7 + gallery/index.yaml | 941 ++++++++++++++++++++ gallery/kyutai-stt-crispasr.yaml | 7 + gallery/mega-asr-crispasr.yaml | 7 + gallery/mimo-asr-crispasr.yaml | 7 + gallery/moonshine-crispasr.yaml | 7 + gallery/moonshine-de-crispasr.yaml | 7 + gallery/moonshine-streaming-crispasr.yaml | 7 + gallery/moonshine-tiny-de-crispasr.yaml | 7 + gallery/omniasr-crispasr.yaml | 7 + gallery/omniasr-llm-1b-crispasr.yaml | 7 + gallery/omniasr-llm-crispasr.yaml | 7 + gallery/paraformer-crispasr.yaml | 7 + gallery/parakeet-crispasr.yaml | 7 + gallery/parakeet-ctc-0.6b-crispasr.yaml | 7 + gallery/parakeet-ctc-1.1b-crispasr.yaml | 7 + gallery/parakeet-ja-crispasr.yaml | 7 + gallery/parakeet-rnnt-0.6b-crispasr.yaml | 7 + gallery/parakeet-rnnt-1.1b-crispasr.yaml | 7 + gallery/parakeet-tdt-1.1b-crispasr.yaml | 7 + gallery/parakeet-tdt_ctc-1.1b-crispasr.yaml | 7 + gallery/parakeet-tdt_ctc-110m-crispasr.yaml | 7 + gallery/parakeet-v2-crispasr.yaml | 7 + gallery/qwen3-1.7b-crispasr.yaml | 7 + gallery/qwen3-crispasr.yaml | 7 + gallery/sensevoice-crispasr.yaml | 7 + gallery/vibevoice-crispasr.yaml | 7 + gallery/voxtral-crispasr.yaml | 7 + gallery/voxtral4b-crispasr.yaml | 7 + gallery/wav2vec2-crispasr.yaml | 7 + gallery/wav2vec2-de-crispasr.yaml | 7 + 43 files changed, 1235 insertions(+) create mode 100644 gallery/canary-crispasr.yaml create mode 100644 gallery/cohere-crispasr.yaml create mode 100644 gallery/data2vec-crispasr.yaml create mode 100644 gallery/fastconformer-ctc-crispasr.yaml create mode 100644 gallery/firered-asr-crispasr.yaml create mode 100644 gallery/fun-asr-mlt-nano-crispasr.yaml create mode 100644 gallery/funasr-crispasr.yaml create mode 100644 gallery/glm-asr-crispasr.yaml create mode 100644 gallery/granite-4.1-crispasr.yaml create mode 100644 gallery/granite-4.1-nar-crispasr.yaml create mode 100644 gallery/granite-4.1-plus-crispasr.yaml create mode 100644 gallery/granite-crispasr.yaml create mode 100644 gallery/hubert-crispasr.yaml create mode 100644 gallery/kyutai-stt-crispasr.yaml create mode 100644 gallery/mega-asr-crispasr.yaml create mode 100644 gallery/mimo-asr-crispasr.yaml create mode 100644 gallery/moonshine-crispasr.yaml create mode 100644 gallery/moonshine-de-crispasr.yaml create mode 100644 gallery/moonshine-streaming-crispasr.yaml create mode 100644 gallery/moonshine-tiny-de-crispasr.yaml create mode 100644 gallery/omniasr-crispasr.yaml create mode 100644 gallery/omniasr-llm-1b-crispasr.yaml create mode 100644 gallery/omniasr-llm-crispasr.yaml create mode 100644 gallery/paraformer-crispasr.yaml create mode 100644 gallery/parakeet-crispasr.yaml create mode 100644 gallery/parakeet-ctc-0.6b-crispasr.yaml create mode 100644 gallery/parakeet-ctc-1.1b-crispasr.yaml create mode 100644 gallery/parakeet-ja-crispasr.yaml create mode 100644 gallery/parakeet-rnnt-0.6b-crispasr.yaml create mode 100644 gallery/parakeet-rnnt-1.1b-crispasr.yaml create mode 100644 gallery/parakeet-tdt-1.1b-crispasr.yaml create mode 100644 gallery/parakeet-tdt_ctc-1.1b-crispasr.yaml create mode 100644 gallery/parakeet-tdt_ctc-110m-crispasr.yaml create mode 100644 gallery/parakeet-v2-crispasr.yaml create mode 100644 gallery/qwen3-1.7b-crispasr.yaml create mode 100644 gallery/qwen3-crispasr.yaml create mode 100644 gallery/sensevoice-crispasr.yaml create mode 100644 gallery/vibevoice-crispasr.yaml create mode 100644 gallery/voxtral-crispasr.yaml create mode 100644 gallery/voxtral4b-crispasr.yaml create mode 100644 gallery/wav2vec2-crispasr.yaml create mode 100644 gallery/wav2vec2-de-crispasr.yaml diff --git a/gallery/canary-crispasr.yaml b/gallery/canary-crispasr.yaml new file mode 100644 index 000000000000..d7743d8c3eb0 --- /dev/null +++ b/gallery/canary-crispasr.yaml @@ -0,0 +1,7 @@ +--- +name: "canary-crispasr" + +config_file: | + backend: crispasr + parameters: + model: canary-1b-v2-q4_k.gguf diff --git a/gallery/cohere-crispasr.yaml b/gallery/cohere-crispasr.yaml new file mode 100644 index 000000000000..7f5a885409ec --- /dev/null +++ b/gallery/cohere-crispasr.yaml @@ -0,0 +1,7 @@ +--- +name: "cohere-crispasr" + +config_file: | + backend: crispasr + parameters: + model: cohere-transcribe-q4_k.gguf diff --git a/gallery/data2vec-crispasr.yaml b/gallery/data2vec-crispasr.yaml new file mode 100644 index 000000000000..cadef474aee8 --- /dev/null +++ b/gallery/data2vec-crispasr.yaml @@ -0,0 +1,7 @@ +--- +name: "data2vec-crispasr" + +config_file: | + backend: crispasr + parameters: + model: data2vec-audio-base-960h-q4_k.gguf diff --git a/gallery/fastconformer-ctc-crispasr.yaml b/gallery/fastconformer-ctc-crispasr.yaml new file mode 100644 index 000000000000..a3ae51f83776 --- /dev/null +++ b/gallery/fastconformer-ctc-crispasr.yaml @@ -0,0 +1,7 @@ +--- +name: "fastconformer-ctc-crispasr" + +config_file: | + backend: crispasr + parameters: + model: stt-en-fastconformer-ctc-large-q4_k.gguf diff --git a/gallery/firered-asr-crispasr.yaml b/gallery/firered-asr-crispasr.yaml new file mode 100644 index 000000000000..aa6a318bd5d2 --- /dev/null +++ b/gallery/firered-asr-crispasr.yaml @@ -0,0 +1,7 @@ +--- +name: "firered-asr-crispasr" + +config_file: | + backend: crispasr + parameters: + model: firered-asr2-aed-q4_k.gguf diff --git a/gallery/fun-asr-mlt-nano-crispasr.yaml b/gallery/fun-asr-mlt-nano-crispasr.yaml new file mode 100644 index 000000000000..44df86fa51f0 --- /dev/null +++ b/gallery/fun-asr-mlt-nano-crispasr.yaml @@ -0,0 +1,7 @@ +--- +name: "fun-asr-mlt-nano-crispasr" + +config_file: | + backend: crispasr + parameters: + model: funasr-mlt-nano-2512-f16.gguf diff --git a/gallery/funasr-crispasr.yaml b/gallery/funasr-crispasr.yaml new file mode 100644 index 000000000000..f1c6bf2b4929 --- /dev/null +++ b/gallery/funasr-crispasr.yaml @@ -0,0 +1,7 @@ +--- +name: "funasr-crispasr" + +config_file: | + backend: crispasr + parameters: + model: funasr-nano-2512-f16.gguf diff --git a/gallery/glm-asr-crispasr.yaml b/gallery/glm-asr-crispasr.yaml new file mode 100644 index 000000000000..9186ab34953f --- /dev/null +++ b/gallery/glm-asr-crispasr.yaml @@ -0,0 +1,7 @@ +--- +name: "glm-asr-crispasr" + +config_file: | + backend: crispasr + parameters: + model: glm-asr-nano-q4_k.gguf diff --git a/gallery/granite-4.1-crispasr.yaml b/gallery/granite-4.1-crispasr.yaml new file mode 100644 index 000000000000..fd9222f3bb9f --- /dev/null +++ b/gallery/granite-4.1-crispasr.yaml @@ -0,0 +1,7 @@ +--- +name: "granite-4.1-crispasr" + +config_file: | + backend: crispasr + parameters: + model: granite-speech-4.1-2b-q4_k.gguf diff --git a/gallery/granite-4.1-nar-crispasr.yaml b/gallery/granite-4.1-nar-crispasr.yaml new file mode 100644 index 000000000000..cf2ec73cff22 --- /dev/null +++ b/gallery/granite-4.1-nar-crispasr.yaml @@ -0,0 +1,7 @@ +--- +name: "granite-4.1-nar-crispasr" + +config_file: | + backend: crispasr + parameters: + model: granite-speech-4.1-2b-nar-q4_k.gguf diff --git a/gallery/granite-4.1-plus-crispasr.yaml b/gallery/granite-4.1-plus-crispasr.yaml new file mode 100644 index 000000000000..3be970fc87ca --- /dev/null +++ b/gallery/granite-4.1-plus-crispasr.yaml @@ -0,0 +1,7 @@ +--- +name: "granite-4.1-plus-crispasr" + +config_file: | + backend: crispasr + parameters: + model: granite-speech-4.1-2b-plus-q4_k.gguf diff --git a/gallery/granite-crispasr.yaml b/gallery/granite-crispasr.yaml new file mode 100644 index 000000000000..7142ca86829f --- /dev/null +++ b/gallery/granite-crispasr.yaml @@ -0,0 +1,7 @@ +--- +name: "granite-crispasr" + +config_file: | + backend: crispasr + parameters: + model: granite-speech-4.0-1b-q4_k.gguf diff --git a/gallery/hubert-crispasr.yaml b/gallery/hubert-crispasr.yaml new file mode 100644 index 000000000000..aae3c91a5d2c --- /dev/null +++ b/gallery/hubert-crispasr.yaml @@ -0,0 +1,7 @@ +--- +name: "hubert-crispasr" + +config_file: | + backend: crispasr + parameters: + model: hubert-large-ls960-ft-q4_k.gguf diff --git a/gallery/index.yaml b/gallery/index.yaml index ecd648f326bf..53c24e530e59 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -31771,3 +31771,944 @@ - filename: parakeet-cpp/tdt_ctc-1.1b-f16.gguf uri: huggingface://mudler/parakeet-cpp-gguf/tdt_ctc-1.1b-f16.gguf sha256: cd53f64eefac2623a12f2f118ef50b56622dc3012f42c815c6adf0d08292f387 + +- name: parakeet-crispasr + url: github:mudler/LocalAI/gallery/parakeet-crispasr.yaml@master + urls: + - https://huggingface.co/cstr/parakeet-tdt-0.6b-v3-GGUF + description: | + NVIDIA Parakeet TDT 0.6B v3 (FastConformer + Token-and-Duration Transducer), 25-language ASR. Runs via the CrispASR backend. Default GGUF size ~467 MB. + tags: + - crispasr + - asr + - speech-recognition + - stt + - gguf + overrides: + backend: crispasr + known_usecases: + - transcript + name: parakeet-crispasr + parameters: + model: parakeet-tdt-0.6b-v3-q4_k.gguf + files: + - filename: parakeet-tdt-0.6b-v3-q4_k.gguf + uri: huggingface://cstr/parakeet-tdt-0.6b-v3-GGUF/parakeet-tdt-0.6b-v3-q4_k.gguf +- name: parakeet-v2-crispasr + url: github:mudler/LocalAI/gallery/parakeet-v2-crispasr.yaml@master + urls: + - https://huggingface.co/cstr/parakeet-tdt-0.6b-v2-GGUF + description: | + NVIDIA Parakeet TDT 0.6B v2 (FastConformer + TDT), English-only ASR. Runs via the CrispASR backend. Default GGUF size ~468 MB. + tags: + - crispasr + - asr + - speech-recognition + - stt + - gguf + overrides: + backend: crispasr + known_usecases: + - transcript + name: parakeet-v2-crispasr + parameters: + model: parakeet-tdt-0.6b-v2-q4_k.gguf + files: + - filename: parakeet-tdt-0.6b-v2-q4_k.gguf + uri: huggingface://cstr/parakeet-tdt-0.6b-v2-GGUF/parakeet-tdt-0.6b-v2-q4_k.gguf +- name: parakeet-ja-crispasr + url: github:mudler/LocalAI/gallery/parakeet-ja-crispasr.yaml@master + urls: + - https://huggingface.co/cstr/parakeet-tdt-0.6b-ja-GGUF + description: | + NVIDIA Parakeet TDT 0.6B Japanese ASR (F16 default; Q4_K is quantisation-sensitive for this model). Runs via the CrispASR backend. Default GGUF size ~1.24 GB. + tags: + - crispasr + - asr + - speech-recognition + - stt + - gguf + overrides: + backend: crispasr + known_usecases: + - transcript + name: parakeet-ja-crispasr + parameters: + model: parakeet-tdt-0.6b-ja.gguf + files: + - filename: parakeet-tdt-0.6b-ja.gguf + uri: huggingface://cstr/parakeet-tdt-0.6b-ja-GGUF/parakeet-tdt-0.6b-ja.gguf +- name: parakeet-tdt-1.1b-crispasr + url: github:mudler/LocalAI/gallery/parakeet-tdt-1.1b-crispasr.yaml@master + urls: + - https://huggingface.co/cstr/parakeet-tdt-1.1b-GGUF + description: | + NVIDIA Parakeet TDT 1.1B (42-layer FastConformer encoder), English-only ASR. Runs via the CrispASR backend. Default GGUF size ~808 MB. + tags: + - crispasr + - asr + - speech-recognition + - stt + - gguf + overrides: + backend: crispasr + known_usecases: + - transcript + name: parakeet-tdt-1.1b-crispasr + parameters: + model: parakeet-tdt-1.1b-q4_k.gguf + files: + - filename: parakeet-tdt-1.1b-q4_k.gguf + uri: huggingface://cstr/parakeet-tdt-1.1b-GGUF/parakeet-tdt-1.1b-q4_k.gguf +- name: parakeet-tdt_ctc-110m-crispasr + url: github:mudler/LocalAI/gallery/parakeet-tdt_ctc-110m-crispasr.yaml@master + urls: + - https://huggingface.co/cstr/parakeet-tdt_ctc-110m-GGUF + description: | + NVIDIA Parakeet hybrid TDT+CTC 110M (smallest, CTC decode), English-only ASR. Runs via the CrispASR backend. Default GGUF size ~91 MB. + tags: + - crispasr + - asr + - speech-recognition + - stt + - gguf + overrides: + backend: crispasr + known_usecases: + - transcript + name: parakeet-tdt_ctc-110m-crispasr + parameters: + model: parakeet-tdt_ctc-110m-q4_k.gguf + files: + - filename: parakeet-tdt_ctc-110m-q4_k.gguf + uri: huggingface://cstr/parakeet-tdt_ctc-110m-GGUF/parakeet-tdt_ctc-110m-q4_k.gguf +- name: parakeet-tdt_ctc-1.1b-crispasr + url: github:mudler/LocalAI/gallery/parakeet-tdt_ctc-1.1b-crispasr.yaml@master + urls: + - https://huggingface.co/cstr/parakeet-tdt_ctc-1.1b-GGUF + description: | + NVIDIA Parakeet hybrid TDT+CTC 1.1B (multilingual, casing + punctuation) ASR. Runs via the CrispASR backend. Default GGUF size ~810 MB. + tags: + - crispasr + - asr + - speech-recognition + - stt + - gguf + overrides: + backend: crispasr + known_usecases: + - transcript + name: parakeet-tdt_ctc-1.1b-crispasr + parameters: + model: parakeet-tdt_ctc-1.1b-q4_k.gguf + files: + - filename: parakeet-tdt_ctc-1.1b-q4_k.gguf + uri: huggingface://cstr/parakeet-tdt_ctc-1.1b-GGUF/parakeet-tdt_ctc-1.1b-q4_k.gguf +- name: parakeet-rnnt-0.6b-crispasr + url: github:mudler/LocalAI/gallery/parakeet-rnnt-0.6b-crispasr.yaml@master + urls: + - https://huggingface.co/cstr/parakeet-rnnt-0.6b-GGUF + description: | + NVIDIA Parakeet RNN-Transducer 0.6B (24-layer FastConformer) ASR. Runs via the CrispASR backend. Default GGUF size ~447 MB. + tags: + - crispasr + - asr + - speech-recognition + - stt + - gguf + overrides: + backend: crispasr + known_usecases: + - transcript + name: parakeet-rnnt-0.6b-crispasr + parameters: + model: parakeet-rnnt-0.6b-q4_k.gguf + files: + - filename: parakeet-rnnt-0.6b-q4_k.gguf + uri: huggingface://cstr/parakeet-rnnt-0.6b-GGUF/parakeet-rnnt-0.6b-q4_k.gguf +- name: parakeet-rnnt-1.1b-crispasr + url: github:mudler/LocalAI/gallery/parakeet-rnnt-1.1b-crispasr.yaml@master + urls: + - https://huggingface.co/cstr/parakeet-rnnt-1.1b-GGUF + description: | + NVIDIA Parakeet RNN-Transducer 1.1B (42-layer FastConformer) ASR. Runs via the CrispASR backend. Default GGUF size ~770 MB. + tags: + - crispasr + - asr + - speech-recognition + - stt + - gguf + overrides: + backend: crispasr + known_usecases: + - transcript + name: parakeet-rnnt-1.1b-crispasr + parameters: + model: parakeet-rnnt-1.1b-q4_k.gguf + files: + - filename: parakeet-rnnt-1.1b-q4_k.gguf + uri: huggingface://cstr/parakeet-rnnt-1.1b-GGUF/parakeet-rnnt-1.1b-q4_k.gguf +- name: parakeet-ctc-0.6b-crispasr + url: github:mudler/LocalAI/gallery/parakeet-ctc-0.6b-crispasr.yaml@master + urls: + - https://huggingface.co/cstr/parakeet-ctc-0.6b-GGUF + description: | + NVIDIA Parakeet CTC 0.6B (FastConformer-CTC), English-only ASR. Runs via the CrispASR backend. Default GGUF size ~455 MB. + tags: + - crispasr + - asr + - speech-recognition + - stt + - gguf + overrides: + backend: crispasr + known_usecases: + - transcript + name: parakeet-ctc-0.6b-crispasr + parameters: + model: parakeet-ctc-0.6b-q4_k.gguf + files: + - filename: parakeet-ctc-0.6b-q4_k.gguf + uri: huggingface://cstr/parakeet-ctc-0.6b-GGUF/parakeet-ctc-0.6b-q4_k.gguf +- name: parakeet-ctc-1.1b-crispasr + url: github:mudler/LocalAI/gallery/parakeet-ctc-1.1b-crispasr.yaml@master + urls: + - https://huggingface.co/cstr/parakeet-ctc-1.1b-GGUF + description: | + NVIDIA Parakeet CTC 1.1B (FastConformer-CTC), English-only ASR. Runs via the CrispASR backend. Default GGUF size ~795 MB. + tags: + - crispasr + - asr + - speech-recognition + - stt + - gguf + overrides: + backend: crispasr + known_usecases: + - transcript + name: parakeet-ctc-1.1b-crispasr + parameters: + model: parakeet-ctc-1.1b-q4_k.gguf + files: + - filename: parakeet-ctc-1.1b-q4_k.gguf + uri: huggingface://cstr/parakeet-ctc-1.1b-GGUF/parakeet-ctc-1.1b-q4_k.gguf +- name: fastconformer-ctc-crispasr + url: github:mudler/LocalAI/gallery/fastconformer-ctc-crispasr.yaml@master + urls: + - https://huggingface.co/cstr/stt-en-fastconformer-ctc-large-GGUF + description: | + NVIDIA STT-EN FastConformer-CTC Large, English ASR. Runs via the CrispASR backend. Default GGUF size ~83 MB. + tags: + - crispasr + - asr + - speech-recognition + - stt + - gguf + overrides: + backend: crispasr + known_usecases: + - transcript + name: fastconformer-ctc-crispasr + parameters: + model: stt-en-fastconformer-ctc-large-q4_k.gguf + files: + - filename: stt-en-fastconformer-ctc-large-q4_k.gguf + uri: huggingface://cstr/stt-en-fastconformer-ctc-large-GGUF/stt-en-fastconformer-ctc-large-q4_k.gguf +- name: canary-crispasr + url: github:mudler/LocalAI/gallery/canary-crispasr.yaml@master + urls: + - https://huggingface.co/cstr/canary-1b-v2-GGUF + description: | + NVIDIA Canary 1B v2 (FastConformer encoder-decoder), multilingual ASR + translation. Runs via the CrispASR backend. Default GGUF size ~600 MB. + tags: + - crispasr + - asr + - speech-recognition + - stt + - gguf + overrides: + backend: crispasr + known_usecases: + - transcript + name: canary-crispasr + parameters: + model: canary-1b-v2-q4_k.gguf + files: + - filename: canary-1b-v2-q4_k.gguf + uri: huggingface://cstr/canary-1b-v2-GGUF/canary-1b-v2-q4_k.gguf +- name: voxtral-crispasr + url: github:mudler/LocalAI/gallery/voxtral-crispasr.yaml@master + urls: + - https://huggingface.co/cstr/voxtral-mini-3b-2507-GGUF + description: | + Mistral Voxtral Mini 3B (audio LLM) ASR. Runs via the CrispASR backend. Default GGUF size ~2.5 GB. + tags: + - crispasr + - asr + - speech-recognition + - stt + - gguf + overrides: + backend: crispasr + known_usecases: + - transcript + name: voxtral-crispasr + parameters: + model: voxtral-mini-3b-2507-q4_k.gguf + files: + - filename: voxtral-mini-3b-2507-q4_k.gguf + uri: huggingface://cstr/voxtral-mini-3b-2507-GGUF/voxtral-mini-3b-2507-q4_k.gguf +- name: voxtral4b-crispasr + url: github:mudler/LocalAI/gallery/voxtral4b-crispasr.yaml@master + urls: + - https://huggingface.co/cstr/voxtral-mini-4b-realtime-GGUF + description: | + Mistral Voxtral Mini 4B Realtime (audio LLM) ASR. Runs via the CrispASR backend. Default GGUF size ~3.3 GB. + tags: + - crispasr + - asr + - speech-recognition + - stt + - gguf + overrides: + backend: crispasr + known_usecases: + - transcript + name: voxtral4b-crispasr + parameters: + model: voxtral-mini-4b-realtime-q4_k.gguf + files: + - filename: voxtral-mini-4b-realtime-q4_k.gguf + uri: huggingface://cstr/voxtral-mini-4b-realtime-GGUF/voxtral-mini-4b-realtime-q4_k.gguf +- name: granite-crispasr + url: github:mudler/LocalAI/gallery/granite-crispasr.yaml@master + urls: + - https://huggingface.co/cstr/granite-speech-4.0-1b-GGUF + description: | + IBM Granite Speech 4.0 1B ASR. Runs via the CrispASR backend. Default GGUF size ~2.94 GB. + tags: + - crispasr + - asr + - speech-recognition + - stt + - gguf + overrides: + backend: crispasr + known_usecases: + - transcript + name: granite-crispasr + parameters: + model: granite-speech-4.0-1b-q4_k.gguf + files: + - filename: granite-speech-4.0-1b-q4_k.gguf + uri: huggingface://cstr/granite-speech-4.0-1b-GGUF/granite-speech-4.0-1b-q4_k.gguf +- name: granite-4.1-crispasr + url: github:mudler/LocalAI/gallery/granite-4.1-crispasr.yaml@master + urls: + - https://huggingface.co/cstr/granite-speech-4.1-2b-GGUF + description: | + IBM Granite Speech 4.1 2B ASR. Runs via the CrispASR backend. Default GGUF size ~2.94 GB. + tags: + - crispasr + - asr + - speech-recognition + - stt + - gguf + overrides: + backend: crispasr + known_usecases: + - transcript + name: granite-4.1-crispasr + parameters: + model: granite-speech-4.1-2b-q4_k.gguf + files: + - filename: granite-speech-4.1-2b-q4_k.gguf + uri: huggingface://cstr/granite-speech-4.1-2b-GGUF/granite-speech-4.1-2b-q4_k.gguf +- name: granite-4.1-plus-crispasr + url: github:mudler/LocalAI/gallery/granite-4.1-plus-crispasr.yaml@master + urls: + - https://huggingface.co/cstr/granite-speech-4.1-2b-plus-GGUF + description: | + IBM Granite Speech 4.1 2B Plus ASR. Runs via the CrispASR backend. Default GGUF size ~2.96 GB. + tags: + - crispasr + - asr + - speech-recognition + - stt + - gguf + overrides: + backend: crispasr + known_usecases: + - transcript + name: granite-4.1-plus-crispasr + parameters: + model: granite-speech-4.1-2b-plus-q4_k.gguf + files: + - filename: granite-speech-4.1-2b-plus-q4_k.gguf + uri: huggingface://cstr/granite-speech-4.1-2b-plus-GGUF/granite-speech-4.1-2b-plus-q4_k.gguf +- name: granite-4.1-nar-crispasr + url: github:mudler/LocalAI/gallery/granite-4.1-nar-crispasr.yaml@master + urls: + - https://huggingface.co/cstr/granite-speech-4.1-2b-nar-GGUF + description: | + IBM Granite Speech 4.1 2B NAR (non-autoregressive) ASR. Runs via the CrispASR backend. Default GGUF size ~3.2 GB. + tags: + - crispasr + - asr + - speech-recognition + - stt + - gguf + overrides: + backend: crispasr + known_usecases: + - transcript + name: granite-4.1-nar-crispasr + parameters: + model: granite-speech-4.1-2b-nar-q4_k.gguf + files: + - filename: granite-speech-4.1-2b-nar-q4_k.gguf + uri: huggingface://cstr/granite-speech-4.1-2b-nar-GGUF/granite-speech-4.1-2b-nar-q4_k.gguf +- name: qwen3-crispasr + url: github:mudler/LocalAI/gallery/qwen3-crispasr.yaml@master + urls: + - https://huggingface.co/cstr/qwen3-asr-0.6b-GGUF + description: | + Qwen3-ASR 0.6B ASR. Runs via the CrispASR backend. Default GGUF size ~500 MB. + tags: + - crispasr + - asr + - speech-recognition + - stt + - gguf + overrides: + backend: crispasr + known_usecases: + - transcript + name: qwen3-crispasr + parameters: + model: qwen3-asr-0.6b-q4_k.gguf + files: + - filename: qwen3-asr-0.6b-q4_k.gguf + uri: huggingface://cstr/qwen3-asr-0.6b-GGUF/qwen3-asr-0.6b-q4_k.gguf +- name: qwen3-1.7b-crispasr + url: github:mudler/LocalAI/gallery/qwen3-1.7b-crispasr.yaml@master + urls: + - https://huggingface.co/cstr/qwen3-asr-1.7b-GGUF + description: | + Qwen3-ASR 1.7B ASR. Runs via the CrispASR backend. Default GGUF size ~1.3 GB. + tags: + - crispasr + - asr + - speech-recognition + - stt + - gguf + overrides: + backend: crispasr + known_usecases: + - transcript + name: qwen3-1.7b-crispasr + parameters: + model: qwen3-asr-1.7b-q4_k.gguf + files: + - filename: qwen3-asr-1.7b-q4_k.gguf + uri: huggingface://cstr/qwen3-asr-1.7b-GGUF/qwen3-asr-1.7b-q4_k.gguf +- name: mega-asr-crispasr + url: github:mudler/LocalAI/gallery/mega-asr-crispasr.yaml@master + urls: + - https://huggingface.co/cstr/mega-asr-GGUF + description: | + Mega-ASR 1.7B (Qwen3-ASR-1.7B with robustness LoRA merged) ASR. Runs via the CrispASR backend. Default GGUF size ~1.3 GB. + tags: + - crispasr + - asr + - speech-recognition + - stt + - gguf + overrides: + backend: crispasr + known_usecases: + - transcript + name: mega-asr-crispasr + parameters: + model: mega-asr-1.7b-q4_k.gguf + files: + - filename: mega-asr-1.7b-q4_k.gguf + uri: huggingface://cstr/mega-asr-GGUF/mega-asr-1.7b-q4_k.gguf +- name: funasr-crispasr + url: github:mudler/LocalAI/gallery/funasr-crispasr.yaml@master + urls: + - https://huggingface.co/cstr/funasr-nano-GGUF + description: | + FunAudioLLM Fun-ASR-Nano-2512 (SANM encoder + Qwen3-0.6B decoder), zh/yue/en/ja/ko ASR. Runs via the CrispASR backend. Default GGUF size ~1.98 GB (F16 only). + license: "FunASR Model License v1.1 (commercial use OK with attribution)" + tags: + - crispasr + - asr + - speech-recognition + - stt + - gguf + overrides: + backend: crispasr + known_usecases: + - transcript + name: funasr-crispasr + parameters: + model: funasr-nano-2512-f16.gguf + files: + - filename: funasr-nano-2512-f16.gguf + uri: huggingface://cstr/funasr-nano-GGUF/funasr-nano-2512-f16.gguf +- name: fun-asr-mlt-nano-crispasr + url: github:mudler/LocalAI/gallery/fun-asr-mlt-nano-crispasr.yaml@master + urls: + - https://huggingface.co/cstr/funasr-mlt-nano-GGUF + description: | + FunAudioLLM Fun-ASR-MLT-Nano-2512 multilingual (31 languages) ASR. Runs via the CrispASR backend. Default GGUF size ~1.98 GB (F16 only). + license: "FunASR Model License v1.1 (commercial use OK with attribution)" + tags: + - crispasr + - asr + - speech-recognition + - stt + - gguf + overrides: + backend: crispasr + known_usecases: + - transcript + name: fun-asr-mlt-nano-crispasr + parameters: + model: funasr-mlt-nano-2512-f16.gguf + files: + - filename: funasr-mlt-nano-2512-f16.gguf + uri: huggingface://cstr/funasr-mlt-nano-GGUF/funasr-mlt-nano-2512-f16.gguf +- name: sensevoice-crispasr + url: github:mudler/LocalAI/gallery/sensevoice-crispasr.yaml@master + urls: + - https://huggingface.co/cstr/sensevoice-small-GGUF + description: | + FunAudioLLM SenseVoiceSmall (encoder-only multi-task: ASR + language + emotion + audio-event tags). Runs via the CrispASR backend. Default GGUF size ~129 MB. + license: "FunASR Model License v1.1 (commercial use OK with attribution)" + tags: + - crispasr + - asr + - speech-recognition + - stt + - gguf + overrides: + backend: crispasr + known_usecases: + - transcript + name: sensevoice-crispasr + parameters: + model: sensevoice-small-q4_k.gguf + files: + - filename: sensevoice-small-q4_k.gguf + uri: huggingface://cstr/sensevoice-small-GGUF/sensevoice-small-q4_k.gguf +- name: paraformer-crispasr + url: github:mudler/LocalAI/gallery/paraformer-crispasr.yaml@master + urls: + - https://huggingface.co/cstr/paraformer-zh-GGUF + description: | + FunASR Paraformer-zh (NAR-ASR, 220M), zh+en character-level ASR. Runs via the CrispASR backend. Default GGUF size ~123 MB. + license: "FunASR Model License (commercial use OK with attribution)" + tags: + - crispasr + - asr + - speech-recognition + - stt + - gguf + overrides: + backend: crispasr + known_usecases: + - transcript + name: paraformer-crispasr + parameters: + model: paraformer-zh-q4_k.gguf + files: + - filename: paraformer-zh-q4_k.gguf + uri: huggingface://cstr/paraformer-zh-GGUF/paraformer-zh-q4_k.gguf +- name: cohere-crispasr + url: github:mudler/LocalAI/gallery/cohere-crispasr.yaml@master + urls: + - https://huggingface.co/cstr/cohere-transcribe-03-2026-GGUF + description: | + Cohere Transcribe (03-2026) ASR. Runs via the CrispASR backend. Default GGUF size ~550 MB. + tags: + - crispasr + - asr + - speech-recognition + - stt + - gguf + overrides: + backend: crispasr + known_usecases: + - transcript + name: cohere-crispasr + parameters: + model: cohere-transcribe-q4_k.gguf + files: + - filename: cohere-transcribe-q4_k.gguf + uri: huggingface://cstr/cohere-transcribe-03-2026-GGUF/cohere-transcribe-q4_k.gguf +- name: wav2vec2-crispasr + url: github:mudler/LocalAI/gallery/wav2vec2-crispasr.yaml@master + urls: + - https://huggingface.co/cstr/wav2vec2-large-xlsr-53-english-GGUF + description: | + wav2vec2 Large XLSR-53 English (CTC) ASR. Runs via the CrispASR backend. Default GGUF size ~212 MB. + tags: + - crispasr + - asr + - speech-recognition + - stt + - gguf + overrides: + backend: crispasr + known_usecases: + - transcript + name: wav2vec2-crispasr + parameters: + model: wav2vec2-xlsr-en-q4_k.gguf + files: + - filename: wav2vec2-xlsr-en-q4_k.gguf + uri: huggingface://cstr/wav2vec2-large-xlsr-53-english-GGUF/wav2vec2-xlsr-en-q4_k.gguf +- name: wav2vec2-de-crispasr + url: github:mudler/LocalAI/gallery/wav2vec2-de-crispasr.yaml@master + urls: + - https://huggingface.co/cstr/wav2vec2-large-xlsr-53-german-GGUF + description: | + wav2vec2 Large XLSR-53 German (CTC) ASR. Runs via the CrispASR backend. Default GGUF size ~222 MB. + tags: + - crispasr + - asr + - speech-recognition + - stt + - gguf + overrides: + backend: crispasr + known_usecases: + - transcript + name: wav2vec2-de-crispasr + parameters: + model: wav2vec2-large-xlsr-53-german-q4_k.gguf + files: + - filename: wav2vec2-large-xlsr-53-german-q4_k.gguf + uri: huggingface://cstr/wav2vec2-large-xlsr-53-german-GGUF/wav2vec2-large-xlsr-53-german-q4_k.gguf +- name: hubert-crispasr + url: github:mudler/LocalAI/gallery/hubert-crispasr.yaml@master + urls: + - https://huggingface.co/cstr/hubert-large-ls960-ft-GGUF + description: | + HuBERT Large (LS960 fine-tuned, CTC) English ASR. Runs via the CrispASR backend. Default GGUF size ~200 MB. + tags: + - crispasr + - asr + - speech-recognition + - stt + - gguf + overrides: + backend: crispasr + known_usecases: + - transcript + name: hubert-crispasr + parameters: + model: hubert-large-ls960-ft-q4_k.gguf + files: + - filename: hubert-large-ls960-ft-q4_k.gguf + uri: huggingface://cstr/hubert-large-ls960-ft-GGUF/hubert-large-ls960-ft-q4_k.gguf +- name: data2vec-crispasr + url: github:mudler/LocalAI/gallery/data2vec-crispasr.yaml@master + urls: + - https://huggingface.co/cstr/data2vec-audio-960h-GGUF + description: | + data2vec-audio Base (960h, CTC) English ASR. Runs via the CrispASR backend. Default GGUF size ~60 MB. + tags: + - crispasr + - asr + - speech-recognition + - stt + - gguf + overrides: + backend: crispasr + known_usecases: + - transcript + name: data2vec-crispasr + parameters: + model: data2vec-audio-base-960h-q4_k.gguf + files: + - filename: data2vec-audio-base-960h-q4_k.gguf + uri: huggingface://cstr/data2vec-audio-960h-GGUF/data2vec-audio-base-960h-q4_k.gguf +- name: omniasr-crispasr + url: github:mudler/LocalAI/gallery/omniasr-crispasr.yaml@master + urls: + - https://huggingface.co/cstr/omniASR-CTC-1B-v2-GGUF + description: | + omniASR-CTC 1B v2 ASR. Runs via the CrispASR backend. Default GGUF size ~658 MB. + tags: + - crispasr + - asr + - speech-recognition + - stt + - gguf + overrides: + backend: crispasr + known_usecases: + - transcript + name: omniasr-crispasr + parameters: + model: omniasr-ctc-1b-v2-q4_k.gguf + files: + - filename: omniasr-ctc-1b-v2-q4_k.gguf + uri: huggingface://cstr/omniASR-CTC-1B-v2-GGUF/omniasr-ctc-1b-v2-q4_k.gguf +- name: omniasr-llm-crispasr + url: github:mudler/LocalAI/gallery/omniasr-llm-crispasr.yaml@master + urls: + - https://huggingface.co/cstr/omniasr-llm-300m-v2-GGUF + description: | + omniASR-LLM 300M v2 ASR. Runs via the CrispASR backend. Default GGUF size ~1019 MB. + tags: + - crispasr + - asr + - speech-recognition + - stt + - gguf + overrides: + backend: crispasr + known_usecases: + - transcript + name: omniasr-llm-crispasr + parameters: + model: omniasr-llm-300m-v2-q4_k.gguf + files: + - filename: omniasr-llm-300m-v2-q4_k.gguf + uri: huggingface://cstr/omniasr-llm-300m-v2-GGUF/omniasr-llm-300m-v2-q4_k.gguf +- name: omniasr-llm-1b-crispasr + url: github:mudler/LocalAI/gallery/omniasr-llm-1b-crispasr.yaml@master + urls: + - https://huggingface.co/cstr/omniasr-llm-1b-GGUF + description: | + omniASR-LLM 1B ASR. Runs via the CrispASR backend. Default GGUF size ~1300 MB. + tags: + - crispasr + - asr + - speech-recognition + - stt + - gguf + overrides: + backend: crispasr + known_usecases: + - transcript + name: omniasr-llm-1b-crispasr + parameters: + model: omniasr-llm-1b-q4_k.gguf + files: + - filename: omniasr-llm-1b-q4_k.gguf + uri: huggingface://cstr/omniasr-llm-1b-GGUF/omniasr-llm-1b-q4_k.gguf +- name: mimo-asr-crispasr + url: github:mudler/LocalAI/gallery/mimo-asr-crispasr.yaml@master + urls: + - https://huggingface.co/cstr/mimo-asr-GGUF + description: | + MiMo-ASR (audio LLM) ASR. Runs via the CrispASR backend. Default GGUF size ~4.2 GB. + tags: + - crispasr + - asr + - speech-recognition + - stt + - gguf + overrides: + backend: crispasr + known_usecases: + - transcript + name: mimo-asr-crispasr + parameters: + model: mimo-asr-q4_k.gguf + files: + - filename: mimo-asr-q4_k.gguf + uri: huggingface://cstr/mimo-asr-GGUF/mimo-asr-q4_k.gguf + - filename: mimo-tokenizer-q4_k.gguf + uri: huggingface://cstr/mimo-tokenizer-GGUF/mimo-tokenizer-q4_k.gguf +- name: vibevoice-crispasr + url: github:mudler/LocalAI/gallery/vibevoice-crispasr.yaml@master + urls: + - https://huggingface.co/cstr/vibevoice-asr-GGUF + description: | + VibeVoice ASR. Runs via the CrispASR backend. Default GGUF size ~4.5 GB. + tags: + - crispasr + - asr + - speech-recognition + - stt + - gguf + overrides: + backend: crispasr + known_usecases: + - transcript + name: vibevoice-crispasr + parameters: + model: vibevoice-asr-q4_k.gguf + files: + - filename: vibevoice-asr-q4_k.gguf + uri: huggingface://cstr/vibevoice-asr-GGUF/vibevoice-asr-q4_k.gguf +- name: firered-asr-crispasr + url: github:mudler/LocalAI/gallery/firered-asr-crispasr.yaml@master + urls: + - https://huggingface.co/cstr/firered-asr2-aed-GGUF + description: | + FireRed-ASR2 AED (attention encoder-decoder) ASR. Runs via the CrispASR backend. Default GGUF size ~918 MB. + tags: + - crispasr + - asr + - speech-recognition + - stt + - gguf + overrides: + backend: crispasr + known_usecases: + - transcript + name: firered-asr-crispasr + parameters: + model: firered-asr2-aed-q4_k.gguf + files: + - filename: firered-asr2-aed-q4_k.gguf + uri: huggingface://cstr/firered-asr2-aed-GGUF/firered-asr2-aed-q4_k.gguf +- name: kyutai-stt-crispasr + url: github:mudler/LocalAI/gallery/kyutai-stt-crispasr.yaml@master + urls: + - https://huggingface.co/cstr/kyutai-stt-1b-GGUF + description: | + Kyutai STT 1B (streaming) ASR. Runs via the CrispASR backend. Default GGUF size ~636 MB. + tags: + - crispasr + - asr + - speech-recognition + - stt + - gguf + overrides: + backend: crispasr + known_usecases: + - transcript + name: kyutai-stt-crispasr + parameters: + model: kyutai-stt-1b-q4_k.gguf + files: + - filename: kyutai-stt-1b-q4_k.gguf + uri: huggingface://cstr/kyutai-stt-1b-GGUF/kyutai-stt-1b-q4_k.gguf +- name: glm-asr-crispasr + url: github:mudler/LocalAI/gallery/glm-asr-crispasr.yaml@master + urls: + - https://huggingface.co/cstr/glm-asr-nano-GGUF + description: | + GLM-ASR Nano ASR. Runs via the CrispASR backend. Default GGUF size ~1.2 GB. + tags: + - crispasr + - asr + - speech-recognition + - stt + - gguf + overrides: + backend: crispasr + known_usecases: + - transcript + name: glm-asr-crispasr + parameters: + model: glm-asr-nano-q4_k.gguf + files: + - filename: glm-asr-nano-q4_k.gguf + uri: huggingface://cstr/glm-asr-nano-GGUF/glm-asr-nano-q4_k.gguf +- name: moonshine-crispasr + url: github:mudler/LocalAI/gallery/moonshine-crispasr.yaml@master + urls: + - https://huggingface.co/cstr/moonshine-tiny-GGUF + description: | + Moonshine Tiny (Useful Sensors) English ASR. Runs via the CrispASR backend. Default GGUF size ~20 MB. + tags: + - crispasr + - asr + - speech-recognition + - stt + - gguf + overrides: + backend: crispasr + known_usecases: + - transcript + name: moonshine-crispasr + parameters: + model: moonshine-tiny-q4_k.gguf + files: + - filename: moonshine-tiny-q4_k.gguf + uri: huggingface://cstr/moonshine-tiny-GGUF/moonshine-tiny-q4_k.gguf + - filename: tokenizer.bin + uri: huggingface://cstr/moonshine-tiny-GGUF/tokenizer.bin +- name: moonshine-de-crispasr + url: github:mudler/LocalAI/gallery/moonshine-de-crispasr.yaml@master + urls: + - https://huggingface.co/cstr/moonshine-base-de-fidoriel-GGUF + description: | + Moonshine Base German (fidoriel fine-tune) ASR. Runs via the CrispASR backend. Default GGUF size ~39 MB. + license: "CC-BY-NC-SA-4.0" + tags: + - crispasr + - asr + - speech-recognition + - stt + - gguf + overrides: + backend: crispasr + known_usecases: + - transcript + name: moonshine-de-crispasr + parameters: + model: moonshine-base-de-fidoriel-q4_k.gguf + files: + - filename: moonshine-base-de-fidoriel-q4_k.gguf + uri: huggingface://cstr/moonshine-base-de-fidoriel-GGUF/moonshine-base-de-fidoriel-q4_k.gguf + - filename: tokenizer.bin + uri: huggingface://cstr/moonshine-base-de-fidoriel-GGUF/tokenizer.bin +- name: moonshine-tiny-de-crispasr + url: github:mudler/LocalAI/gallery/moonshine-tiny-de-crispasr.yaml@master + urls: + - https://huggingface.co/cstr/moonshine-tiny-de-fidoriel-GGUF + description: | + Moonshine Tiny German (fidoriel fine-tune) ASR. Runs via the CrispASR backend. Default GGUF size ~17 MB. + license: "CC-BY-NC-SA-4.0" + tags: + - crispasr + - asr + - speech-recognition + - stt + - gguf + overrides: + backend: crispasr + known_usecases: + - transcript + name: moonshine-tiny-de-crispasr + parameters: + model: moonshine-tiny-de-fidoriel-q4_k.gguf + files: + - filename: moonshine-tiny-de-fidoriel-q4_k.gguf + uri: huggingface://cstr/moonshine-tiny-de-fidoriel-GGUF/moonshine-tiny-de-fidoriel-q4_k.gguf + - filename: tokenizer.bin + uri: huggingface://cstr/moonshine-tiny-de-fidoriel-GGUF/tokenizer.bin +- name: moonshine-streaming-crispasr + url: github:mudler/LocalAI/gallery/moonshine-streaming-crispasr.yaml@master + urls: + - https://huggingface.co/cstr/moonshine-streaming-tiny-GGUF + description: | + Moonshine Streaming Tiny English ASR. Runs via the CrispASR backend. Default GGUF size ~31 MB. + tags: + - crispasr + - asr + - speech-recognition + - stt + - gguf + overrides: + backend: crispasr + known_usecases: + - transcript + name: moonshine-streaming-crispasr + parameters: + model: moonshine-streaming-tiny-q4_k.gguf + files: + - filename: moonshine-streaming-tiny-q4_k.gguf + uri: huggingface://cstr/moonshine-streaming-tiny-GGUF/moonshine-streaming-tiny-q4_k.gguf + - filename: tokenizer.bin + uri: huggingface://cstr/moonshine-streaming-tiny-GGUF/tokenizer.bin diff --git a/gallery/kyutai-stt-crispasr.yaml b/gallery/kyutai-stt-crispasr.yaml new file mode 100644 index 000000000000..903c459612d4 --- /dev/null +++ b/gallery/kyutai-stt-crispasr.yaml @@ -0,0 +1,7 @@ +--- +name: "kyutai-stt-crispasr" + +config_file: | + backend: crispasr + parameters: + model: kyutai-stt-1b-q4_k.gguf diff --git a/gallery/mega-asr-crispasr.yaml b/gallery/mega-asr-crispasr.yaml new file mode 100644 index 000000000000..b26ea4b05ad3 --- /dev/null +++ b/gallery/mega-asr-crispasr.yaml @@ -0,0 +1,7 @@ +--- +name: "mega-asr-crispasr" + +config_file: | + backend: crispasr + parameters: + model: mega-asr-1.7b-q4_k.gguf diff --git a/gallery/mimo-asr-crispasr.yaml b/gallery/mimo-asr-crispasr.yaml new file mode 100644 index 000000000000..3bab1e151478 --- /dev/null +++ b/gallery/mimo-asr-crispasr.yaml @@ -0,0 +1,7 @@ +--- +name: "mimo-asr-crispasr" + +config_file: | + backend: crispasr + parameters: + model: mimo-asr-q4_k.gguf diff --git a/gallery/moonshine-crispasr.yaml b/gallery/moonshine-crispasr.yaml new file mode 100644 index 000000000000..27c27dcdde27 --- /dev/null +++ b/gallery/moonshine-crispasr.yaml @@ -0,0 +1,7 @@ +--- +name: "moonshine-crispasr" + +config_file: | + backend: crispasr + parameters: + model: moonshine-tiny-q4_k.gguf diff --git a/gallery/moonshine-de-crispasr.yaml b/gallery/moonshine-de-crispasr.yaml new file mode 100644 index 000000000000..1ccc03273274 --- /dev/null +++ b/gallery/moonshine-de-crispasr.yaml @@ -0,0 +1,7 @@ +--- +name: "moonshine-de-crispasr" + +config_file: | + backend: crispasr + parameters: + model: moonshine-base-de-fidoriel-q4_k.gguf diff --git a/gallery/moonshine-streaming-crispasr.yaml b/gallery/moonshine-streaming-crispasr.yaml new file mode 100644 index 000000000000..b5708a731b68 --- /dev/null +++ b/gallery/moonshine-streaming-crispasr.yaml @@ -0,0 +1,7 @@ +--- +name: "moonshine-streaming-crispasr" + +config_file: | + backend: crispasr + parameters: + model: moonshine-streaming-tiny-q4_k.gguf diff --git a/gallery/moonshine-tiny-de-crispasr.yaml b/gallery/moonshine-tiny-de-crispasr.yaml new file mode 100644 index 000000000000..08f15391e7cb --- /dev/null +++ b/gallery/moonshine-tiny-de-crispasr.yaml @@ -0,0 +1,7 @@ +--- +name: "moonshine-tiny-de-crispasr" + +config_file: | + backend: crispasr + parameters: + model: moonshine-tiny-de-fidoriel-q4_k.gguf diff --git a/gallery/omniasr-crispasr.yaml b/gallery/omniasr-crispasr.yaml new file mode 100644 index 000000000000..4f70cfb89fa1 --- /dev/null +++ b/gallery/omniasr-crispasr.yaml @@ -0,0 +1,7 @@ +--- +name: "omniasr-crispasr" + +config_file: | + backend: crispasr + parameters: + model: omniasr-ctc-1b-v2-q4_k.gguf diff --git a/gallery/omniasr-llm-1b-crispasr.yaml b/gallery/omniasr-llm-1b-crispasr.yaml new file mode 100644 index 000000000000..e7b0db8b099e --- /dev/null +++ b/gallery/omniasr-llm-1b-crispasr.yaml @@ -0,0 +1,7 @@ +--- +name: "omniasr-llm-1b-crispasr" + +config_file: | + backend: crispasr + parameters: + model: omniasr-llm-1b-q4_k.gguf diff --git a/gallery/omniasr-llm-crispasr.yaml b/gallery/omniasr-llm-crispasr.yaml new file mode 100644 index 000000000000..fcb950763ecc --- /dev/null +++ b/gallery/omniasr-llm-crispasr.yaml @@ -0,0 +1,7 @@ +--- +name: "omniasr-llm-crispasr" + +config_file: | + backend: crispasr + parameters: + model: omniasr-llm-300m-v2-q4_k.gguf diff --git a/gallery/paraformer-crispasr.yaml b/gallery/paraformer-crispasr.yaml new file mode 100644 index 000000000000..045374dadc25 --- /dev/null +++ b/gallery/paraformer-crispasr.yaml @@ -0,0 +1,7 @@ +--- +name: "paraformer-crispasr" + +config_file: | + backend: crispasr + parameters: + model: paraformer-zh-q4_k.gguf diff --git a/gallery/parakeet-crispasr.yaml b/gallery/parakeet-crispasr.yaml new file mode 100644 index 000000000000..6490290d4167 --- /dev/null +++ b/gallery/parakeet-crispasr.yaml @@ -0,0 +1,7 @@ +--- +name: "parakeet-crispasr" + +config_file: | + backend: crispasr + parameters: + model: parakeet-tdt-0.6b-v3-q4_k.gguf diff --git a/gallery/parakeet-ctc-0.6b-crispasr.yaml b/gallery/parakeet-ctc-0.6b-crispasr.yaml new file mode 100644 index 000000000000..aec82e23f09e --- /dev/null +++ b/gallery/parakeet-ctc-0.6b-crispasr.yaml @@ -0,0 +1,7 @@ +--- +name: "parakeet-ctc-0.6b-crispasr" + +config_file: | + backend: crispasr + parameters: + model: parakeet-ctc-0.6b-q4_k.gguf diff --git a/gallery/parakeet-ctc-1.1b-crispasr.yaml b/gallery/parakeet-ctc-1.1b-crispasr.yaml new file mode 100644 index 000000000000..47cc4299173d --- /dev/null +++ b/gallery/parakeet-ctc-1.1b-crispasr.yaml @@ -0,0 +1,7 @@ +--- +name: "parakeet-ctc-1.1b-crispasr" + +config_file: | + backend: crispasr + parameters: + model: parakeet-ctc-1.1b-q4_k.gguf diff --git a/gallery/parakeet-ja-crispasr.yaml b/gallery/parakeet-ja-crispasr.yaml new file mode 100644 index 000000000000..a441f9bb0000 --- /dev/null +++ b/gallery/parakeet-ja-crispasr.yaml @@ -0,0 +1,7 @@ +--- +name: "parakeet-ja-crispasr" + +config_file: | + backend: crispasr + parameters: + model: parakeet-tdt-0.6b-ja.gguf diff --git a/gallery/parakeet-rnnt-0.6b-crispasr.yaml b/gallery/parakeet-rnnt-0.6b-crispasr.yaml new file mode 100644 index 000000000000..67203508de7a --- /dev/null +++ b/gallery/parakeet-rnnt-0.6b-crispasr.yaml @@ -0,0 +1,7 @@ +--- +name: "parakeet-rnnt-0.6b-crispasr" + +config_file: | + backend: crispasr + parameters: + model: parakeet-rnnt-0.6b-q4_k.gguf diff --git a/gallery/parakeet-rnnt-1.1b-crispasr.yaml b/gallery/parakeet-rnnt-1.1b-crispasr.yaml new file mode 100644 index 000000000000..800d683c9dee --- /dev/null +++ b/gallery/parakeet-rnnt-1.1b-crispasr.yaml @@ -0,0 +1,7 @@ +--- +name: "parakeet-rnnt-1.1b-crispasr" + +config_file: | + backend: crispasr + parameters: + model: parakeet-rnnt-1.1b-q4_k.gguf diff --git a/gallery/parakeet-tdt-1.1b-crispasr.yaml b/gallery/parakeet-tdt-1.1b-crispasr.yaml new file mode 100644 index 000000000000..92cad7967fb3 --- /dev/null +++ b/gallery/parakeet-tdt-1.1b-crispasr.yaml @@ -0,0 +1,7 @@ +--- +name: "parakeet-tdt-1.1b-crispasr" + +config_file: | + backend: crispasr + parameters: + model: parakeet-tdt-1.1b-q4_k.gguf diff --git a/gallery/parakeet-tdt_ctc-1.1b-crispasr.yaml b/gallery/parakeet-tdt_ctc-1.1b-crispasr.yaml new file mode 100644 index 000000000000..b71c5e95eb7a --- /dev/null +++ b/gallery/parakeet-tdt_ctc-1.1b-crispasr.yaml @@ -0,0 +1,7 @@ +--- +name: "parakeet-tdt_ctc-1.1b-crispasr" + +config_file: | + backend: crispasr + parameters: + model: parakeet-tdt_ctc-1.1b-q4_k.gguf diff --git a/gallery/parakeet-tdt_ctc-110m-crispasr.yaml b/gallery/parakeet-tdt_ctc-110m-crispasr.yaml new file mode 100644 index 000000000000..4bf39a1c3225 --- /dev/null +++ b/gallery/parakeet-tdt_ctc-110m-crispasr.yaml @@ -0,0 +1,7 @@ +--- +name: "parakeet-tdt_ctc-110m-crispasr" + +config_file: | + backend: crispasr + parameters: + model: parakeet-tdt_ctc-110m-q4_k.gguf diff --git a/gallery/parakeet-v2-crispasr.yaml b/gallery/parakeet-v2-crispasr.yaml new file mode 100644 index 000000000000..4094b420ca3a --- /dev/null +++ b/gallery/parakeet-v2-crispasr.yaml @@ -0,0 +1,7 @@ +--- +name: "parakeet-v2-crispasr" + +config_file: | + backend: crispasr + parameters: + model: parakeet-tdt-0.6b-v2-q4_k.gguf diff --git a/gallery/qwen3-1.7b-crispasr.yaml b/gallery/qwen3-1.7b-crispasr.yaml new file mode 100644 index 000000000000..0458badf242a --- /dev/null +++ b/gallery/qwen3-1.7b-crispasr.yaml @@ -0,0 +1,7 @@ +--- +name: "qwen3-1.7b-crispasr" + +config_file: | + backend: crispasr + parameters: + model: qwen3-asr-1.7b-q4_k.gguf diff --git a/gallery/qwen3-crispasr.yaml b/gallery/qwen3-crispasr.yaml new file mode 100644 index 000000000000..5bb037538bbf --- /dev/null +++ b/gallery/qwen3-crispasr.yaml @@ -0,0 +1,7 @@ +--- +name: "qwen3-crispasr" + +config_file: | + backend: crispasr + parameters: + model: qwen3-asr-0.6b-q4_k.gguf diff --git a/gallery/sensevoice-crispasr.yaml b/gallery/sensevoice-crispasr.yaml new file mode 100644 index 000000000000..2be065049e35 --- /dev/null +++ b/gallery/sensevoice-crispasr.yaml @@ -0,0 +1,7 @@ +--- +name: "sensevoice-crispasr" + +config_file: | + backend: crispasr + parameters: + model: sensevoice-small-q4_k.gguf diff --git a/gallery/vibevoice-crispasr.yaml b/gallery/vibevoice-crispasr.yaml new file mode 100644 index 000000000000..ed3817ed973c --- /dev/null +++ b/gallery/vibevoice-crispasr.yaml @@ -0,0 +1,7 @@ +--- +name: "vibevoice-crispasr" + +config_file: | + backend: crispasr + parameters: + model: vibevoice-asr-q4_k.gguf diff --git a/gallery/voxtral-crispasr.yaml b/gallery/voxtral-crispasr.yaml new file mode 100644 index 000000000000..319d7f50b1dc --- /dev/null +++ b/gallery/voxtral-crispasr.yaml @@ -0,0 +1,7 @@ +--- +name: "voxtral-crispasr" + +config_file: | + backend: crispasr + parameters: + model: voxtral-mini-3b-2507-q4_k.gguf diff --git a/gallery/voxtral4b-crispasr.yaml b/gallery/voxtral4b-crispasr.yaml new file mode 100644 index 000000000000..64f601e7c2b8 --- /dev/null +++ b/gallery/voxtral4b-crispasr.yaml @@ -0,0 +1,7 @@ +--- +name: "voxtral4b-crispasr" + +config_file: | + backend: crispasr + parameters: + model: voxtral-mini-4b-realtime-q4_k.gguf diff --git a/gallery/wav2vec2-crispasr.yaml b/gallery/wav2vec2-crispasr.yaml new file mode 100644 index 000000000000..66afe60bd720 --- /dev/null +++ b/gallery/wav2vec2-crispasr.yaml @@ -0,0 +1,7 @@ +--- +name: "wav2vec2-crispasr" + +config_file: | + backend: crispasr + parameters: + model: wav2vec2-xlsr-en-q4_k.gguf diff --git a/gallery/wav2vec2-de-crispasr.yaml b/gallery/wav2vec2-de-crispasr.yaml new file mode 100644 index 000000000000..e4ac5999160b --- /dev/null +++ b/gallery/wav2vec2-de-crispasr.yaml @@ -0,0 +1,7 @@ +--- +name: "wav2vec2-de-crispasr" + +config_file: | + backend: crispasr + parameters: + model: wav2vec2-large-xlsr-53-german-q4_k.gguf From 39b8ae808d0a5c453ca616f4fa7f27fcc06d80eb Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sun, 31 May 2026 00:18:04 +0000 Subject: [PATCH 17/29] fix(gallery): keep only session-auto-detectable CrispASR ASR models The crispasr backend loads models via crispasr_session_open, which auto-detects the backend from the GGUF general.architecture using crispasr_detect_backend_from_gguf. Architectures not in that detect map cannot be opened, so those gallery entries fail to load. Removed entries whose architecture is not wired into CrispASR v0.6.11's session auto-detect router (they can be re-added when upstream maps them): - Not in the detect map: data2vec, firered-asr, funasr, fun-asr-mlt-nano, glm-asr, hubert, kyutai-stt, mega-asr, mimo-asr, moonshine{,-de,-streaming,-tiny-de}, omniasr{,-llm,-llm-1b}, paraformer, sensevoice. - Pending verification (filename-heuristic routed, not arch-detected): parakeet-ctc-0.6b, parakeet-ctc-1.1b. Their GGUFs are routed to the fastconformer-ctc backend by a filename heuristic in the model registry, which implies general.architecture is not a mapped string. Kept the parakeet rnnt/tdt_ctc variants: convert-parakeet-to-gguf.py writes general.architecture="parakeet" unconditionally and encodes the rnnt/ctc distinction in metadata fields, so they session-auto-detect. Signed-off-by: Ettore Di Giacinto --- gallery/data2vec-crispasr.yaml | 7 - gallery/firered-asr-crispasr.yaml | 7 - gallery/fun-asr-mlt-nano-crispasr.yaml | 7 - gallery/funasr-crispasr.yaml | 7 - gallery/glm-asr-crispasr.yaml | 7 - gallery/hubert-crispasr.yaml | 7 - gallery/index.yaml | 456 ---------------------- gallery/kyutai-stt-crispasr.yaml | 7 - gallery/mega-asr-crispasr.yaml | 7 - gallery/mimo-asr-crispasr.yaml | 7 - gallery/moonshine-crispasr.yaml | 7 - gallery/moonshine-de-crispasr.yaml | 7 - gallery/moonshine-streaming-crispasr.yaml | 7 - gallery/moonshine-tiny-de-crispasr.yaml | 7 - gallery/omniasr-crispasr.yaml | 7 - gallery/omniasr-llm-1b-crispasr.yaml | 7 - gallery/omniasr-llm-crispasr.yaml | 7 - gallery/paraformer-crispasr.yaml | 7 - gallery/parakeet-ctc-0.6b-crispasr.yaml | 7 - gallery/parakeet-ctc-1.1b-crispasr.yaml | 7 - gallery/sensevoice-crispasr.yaml | 7 - 21 files changed, 596 deletions(-) delete mode 100644 gallery/data2vec-crispasr.yaml delete mode 100644 gallery/firered-asr-crispasr.yaml delete mode 100644 gallery/fun-asr-mlt-nano-crispasr.yaml delete mode 100644 gallery/funasr-crispasr.yaml delete mode 100644 gallery/glm-asr-crispasr.yaml delete mode 100644 gallery/hubert-crispasr.yaml delete mode 100644 gallery/kyutai-stt-crispasr.yaml delete mode 100644 gallery/mega-asr-crispasr.yaml delete mode 100644 gallery/mimo-asr-crispasr.yaml delete mode 100644 gallery/moonshine-crispasr.yaml delete mode 100644 gallery/moonshine-de-crispasr.yaml delete mode 100644 gallery/moonshine-streaming-crispasr.yaml delete mode 100644 gallery/moonshine-tiny-de-crispasr.yaml delete mode 100644 gallery/omniasr-crispasr.yaml delete mode 100644 gallery/omniasr-llm-1b-crispasr.yaml delete mode 100644 gallery/omniasr-llm-crispasr.yaml delete mode 100644 gallery/paraformer-crispasr.yaml delete mode 100644 gallery/parakeet-ctc-0.6b-crispasr.yaml delete mode 100644 gallery/parakeet-ctc-1.1b-crispasr.yaml delete mode 100644 gallery/sensevoice-crispasr.yaml diff --git a/gallery/data2vec-crispasr.yaml b/gallery/data2vec-crispasr.yaml deleted file mode 100644 index cadef474aee8..000000000000 --- a/gallery/data2vec-crispasr.yaml +++ /dev/null @@ -1,7 +0,0 @@ ---- -name: "data2vec-crispasr" - -config_file: | - backend: crispasr - parameters: - model: data2vec-audio-base-960h-q4_k.gguf diff --git a/gallery/firered-asr-crispasr.yaml b/gallery/firered-asr-crispasr.yaml deleted file mode 100644 index aa6a318bd5d2..000000000000 --- a/gallery/firered-asr-crispasr.yaml +++ /dev/null @@ -1,7 +0,0 @@ ---- -name: "firered-asr-crispasr" - -config_file: | - backend: crispasr - parameters: - model: firered-asr2-aed-q4_k.gguf diff --git a/gallery/fun-asr-mlt-nano-crispasr.yaml b/gallery/fun-asr-mlt-nano-crispasr.yaml deleted file mode 100644 index 44df86fa51f0..000000000000 --- a/gallery/fun-asr-mlt-nano-crispasr.yaml +++ /dev/null @@ -1,7 +0,0 @@ ---- -name: "fun-asr-mlt-nano-crispasr" - -config_file: | - backend: crispasr - parameters: - model: funasr-mlt-nano-2512-f16.gguf diff --git a/gallery/funasr-crispasr.yaml b/gallery/funasr-crispasr.yaml deleted file mode 100644 index f1c6bf2b4929..000000000000 --- a/gallery/funasr-crispasr.yaml +++ /dev/null @@ -1,7 +0,0 @@ ---- -name: "funasr-crispasr" - -config_file: | - backend: crispasr - parameters: - model: funasr-nano-2512-f16.gguf diff --git a/gallery/glm-asr-crispasr.yaml b/gallery/glm-asr-crispasr.yaml deleted file mode 100644 index 9186ab34953f..000000000000 --- a/gallery/glm-asr-crispasr.yaml +++ /dev/null @@ -1,7 +0,0 @@ ---- -name: "glm-asr-crispasr" - -config_file: | - backend: crispasr - parameters: - model: glm-asr-nano-q4_k.gguf diff --git a/gallery/hubert-crispasr.yaml b/gallery/hubert-crispasr.yaml deleted file mode 100644 index aae3c91a5d2c..000000000000 --- a/gallery/hubert-crispasr.yaml +++ /dev/null @@ -1,7 +0,0 @@ ---- -name: "hubert-crispasr" - -config_file: | - backend: crispasr - parameters: - model: hubert-large-ls960-ft-q4_k.gguf diff --git a/gallery/index.yaml b/gallery/index.yaml index 53c24e530e59..9da2a1c3d891 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -31948,50 +31948,6 @@ files: - filename: parakeet-rnnt-1.1b-q4_k.gguf uri: huggingface://cstr/parakeet-rnnt-1.1b-GGUF/parakeet-rnnt-1.1b-q4_k.gguf -- name: parakeet-ctc-0.6b-crispasr - url: github:mudler/LocalAI/gallery/parakeet-ctc-0.6b-crispasr.yaml@master - urls: - - https://huggingface.co/cstr/parakeet-ctc-0.6b-GGUF - description: | - NVIDIA Parakeet CTC 0.6B (FastConformer-CTC), English-only ASR. Runs via the CrispASR backend. Default GGUF size ~455 MB. - tags: - - crispasr - - asr - - speech-recognition - - stt - - gguf - overrides: - backend: crispasr - known_usecases: - - transcript - name: parakeet-ctc-0.6b-crispasr - parameters: - model: parakeet-ctc-0.6b-q4_k.gguf - files: - - filename: parakeet-ctc-0.6b-q4_k.gguf - uri: huggingface://cstr/parakeet-ctc-0.6b-GGUF/parakeet-ctc-0.6b-q4_k.gguf -- name: parakeet-ctc-1.1b-crispasr - url: github:mudler/LocalAI/gallery/parakeet-ctc-1.1b-crispasr.yaml@master - urls: - - https://huggingface.co/cstr/parakeet-ctc-1.1b-GGUF - description: | - NVIDIA Parakeet CTC 1.1B (FastConformer-CTC), English-only ASR. Runs via the CrispASR backend. Default GGUF size ~795 MB. - tags: - - crispasr - - asr - - speech-recognition - - stt - - gguf - overrides: - backend: crispasr - known_usecases: - - transcript - name: parakeet-ctc-1.1b-crispasr - parameters: - model: parakeet-ctc-1.1b-q4_k.gguf - files: - - filename: parakeet-ctc-1.1b-q4_k.gguf - uri: huggingface://cstr/parakeet-ctc-1.1b-GGUF/parakeet-ctc-1.1b-q4_k.gguf - name: fastconformer-ctc-crispasr url: github:mudler/LocalAI/gallery/fastconformer-ctc-crispasr.yaml@master urls: @@ -32212,120 +32168,6 @@ files: - filename: qwen3-asr-1.7b-q4_k.gguf uri: huggingface://cstr/qwen3-asr-1.7b-GGUF/qwen3-asr-1.7b-q4_k.gguf -- name: mega-asr-crispasr - url: github:mudler/LocalAI/gallery/mega-asr-crispasr.yaml@master - urls: - - https://huggingface.co/cstr/mega-asr-GGUF - description: | - Mega-ASR 1.7B (Qwen3-ASR-1.7B with robustness LoRA merged) ASR. Runs via the CrispASR backend. Default GGUF size ~1.3 GB. - tags: - - crispasr - - asr - - speech-recognition - - stt - - gguf - overrides: - backend: crispasr - known_usecases: - - transcript - name: mega-asr-crispasr - parameters: - model: mega-asr-1.7b-q4_k.gguf - files: - - filename: mega-asr-1.7b-q4_k.gguf - uri: huggingface://cstr/mega-asr-GGUF/mega-asr-1.7b-q4_k.gguf -- name: funasr-crispasr - url: github:mudler/LocalAI/gallery/funasr-crispasr.yaml@master - urls: - - https://huggingface.co/cstr/funasr-nano-GGUF - description: | - FunAudioLLM Fun-ASR-Nano-2512 (SANM encoder + Qwen3-0.6B decoder), zh/yue/en/ja/ko ASR. Runs via the CrispASR backend. Default GGUF size ~1.98 GB (F16 only). - license: "FunASR Model License v1.1 (commercial use OK with attribution)" - tags: - - crispasr - - asr - - speech-recognition - - stt - - gguf - overrides: - backend: crispasr - known_usecases: - - transcript - name: funasr-crispasr - parameters: - model: funasr-nano-2512-f16.gguf - files: - - filename: funasr-nano-2512-f16.gguf - uri: huggingface://cstr/funasr-nano-GGUF/funasr-nano-2512-f16.gguf -- name: fun-asr-mlt-nano-crispasr - url: github:mudler/LocalAI/gallery/fun-asr-mlt-nano-crispasr.yaml@master - urls: - - https://huggingface.co/cstr/funasr-mlt-nano-GGUF - description: | - FunAudioLLM Fun-ASR-MLT-Nano-2512 multilingual (31 languages) ASR. Runs via the CrispASR backend. Default GGUF size ~1.98 GB (F16 only). - license: "FunASR Model License v1.1 (commercial use OK with attribution)" - tags: - - crispasr - - asr - - speech-recognition - - stt - - gguf - overrides: - backend: crispasr - known_usecases: - - transcript - name: fun-asr-mlt-nano-crispasr - parameters: - model: funasr-mlt-nano-2512-f16.gguf - files: - - filename: funasr-mlt-nano-2512-f16.gguf - uri: huggingface://cstr/funasr-mlt-nano-GGUF/funasr-mlt-nano-2512-f16.gguf -- name: sensevoice-crispasr - url: github:mudler/LocalAI/gallery/sensevoice-crispasr.yaml@master - urls: - - https://huggingface.co/cstr/sensevoice-small-GGUF - description: | - FunAudioLLM SenseVoiceSmall (encoder-only multi-task: ASR + language + emotion + audio-event tags). Runs via the CrispASR backend. Default GGUF size ~129 MB. - license: "FunASR Model License v1.1 (commercial use OK with attribution)" - tags: - - crispasr - - asr - - speech-recognition - - stt - - gguf - overrides: - backend: crispasr - known_usecases: - - transcript - name: sensevoice-crispasr - parameters: - model: sensevoice-small-q4_k.gguf - files: - - filename: sensevoice-small-q4_k.gguf - uri: huggingface://cstr/sensevoice-small-GGUF/sensevoice-small-q4_k.gguf -- name: paraformer-crispasr - url: github:mudler/LocalAI/gallery/paraformer-crispasr.yaml@master - urls: - - https://huggingface.co/cstr/paraformer-zh-GGUF - description: | - FunASR Paraformer-zh (NAR-ASR, 220M), zh+en character-level ASR. Runs via the CrispASR backend. Default GGUF size ~123 MB. - license: "FunASR Model License (commercial use OK with attribution)" - tags: - - crispasr - - asr - - speech-recognition - - stt - - gguf - overrides: - backend: crispasr - known_usecases: - - transcript - name: paraformer-crispasr - parameters: - model: paraformer-zh-q4_k.gguf - files: - - filename: paraformer-zh-q4_k.gguf - uri: huggingface://cstr/paraformer-zh-GGUF/paraformer-zh-q4_k.gguf - name: cohere-crispasr url: github:mudler/LocalAI/gallery/cohere-crispasr.yaml@master urls: @@ -32392,140 +32234,6 @@ files: - filename: wav2vec2-large-xlsr-53-german-q4_k.gguf uri: huggingface://cstr/wav2vec2-large-xlsr-53-german-GGUF/wav2vec2-large-xlsr-53-german-q4_k.gguf -- name: hubert-crispasr - url: github:mudler/LocalAI/gallery/hubert-crispasr.yaml@master - urls: - - https://huggingface.co/cstr/hubert-large-ls960-ft-GGUF - description: | - HuBERT Large (LS960 fine-tuned, CTC) English ASR. Runs via the CrispASR backend. Default GGUF size ~200 MB. - tags: - - crispasr - - asr - - speech-recognition - - stt - - gguf - overrides: - backend: crispasr - known_usecases: - - transcript - name: hubert-crispasr - parameters: - model: hubert-large-ls960-ft-q4_k.gguf - files: - - filename: hubert-large-ls960-ft-q4_k.gguf - uri: huggingface://cstr/hubert-large-ls960-ft-GGUF/hubert-large-ls960-ft-q4_k.gguf -- name: data2vec-crispasr - url: github:mudler/LocalAI/gallery/data2vec-crispasr.yaml@master - urls: - - https://huggingface.co/cstr/data2vec-audio-960h-GGUF - description: | - data2vec-audio Base (960h, CTC) English ASR. Runs via the CrispASR backend. Default GGUF size ~60 MB. - tags: - - crispasr - - asr - - speech-recognition - - stt - - gguf - overrides: - backend: crispasr - known_usecases: - - transcript - name: data2vec-crispasr - parameters: - model: data2vec-audio-base-960h-q4_k.gguf - files: - - filename: data2vec-audio-base-960h-q4_k.gguf - uri: huggingface://cstr/data2vec-audio-960h-GGUF/data2vec-audio-base-960h-q4_k.gguf -- name: omniasr-crispasr - url: github:mudler/LocalAI/gallery/omniasr-crispasr.yaml@master - urls: - - https://huggingface.co/cstr/omniASR-CTC-1B-v2-GGUF - description: | - omniASR-CTC 1B v2 ASR. Runs via the CrispASR backend. Default GGUF size ~658 MB. - tags: - - crispasr - - asr - - speech-recognition - - stt - - gguf - overrides: - backend: crispasr - known_usecases: - - transcript - name: omniasr-crispasr - parameters: - model: omniasr-ctc-1b-v2-q4_k.gguf - files: - - filename: omniasr-ctc-1b-v2-q4_k.gguf - uri: huggingface://cstr/omniASR-CTC-1B-v2-GGUF/omniasr-ctc-1b-v2-q4_k.gguf -- name: omniasr-llm-crispasr - url: github:mudler/LocalAI/gallery/omniasr-llm-crispasr.yaml@master - urls: - - https://huggingface.co/cstr/omniasr-llm-300m-v2-GGUF - description: | - omniASR-LLM 300M v2 ASR. Runs via the CrispASR backend. Default GGUF size ~1019 MB. - tags: - - crispasr - - asr - - speech-recognition - - stt - - gguf - overrides: - backend: crispasr - known_usecases: - - transcript - name: omniasr-llm-crispasr - parameters: - model: omniasr-llm-300m-v2-q4_k.gguf - files: - - filename: omniasr-llm-300m-v2-q4_k.gguf - uri: huggingface://cstr/omniasr-llm-300m-v2-GGUF/omniasr-llm-300m-v2-q4_k.gguf -- name: omniasr-llm-1b-crispasr - url: github:mudler/LocalAI/gallery/omniasr-llm-1b-crispasr.yaml@master - urls: - - https://huggingface.co/cstr/omniasr-llm-1b-GGUF - description: | - omniASR-LLM 1B ASR. Runs via the CrispASR backend. Default GGUF size ~1300 MB. - tags: - - crispasr - - asr - - speech-recognition - - stt - - gguf - overrides: - backend: crispasr - known_usecases: - - transcript - name: omniasr-llm-1b-crispasr - parameters: - model: omniasr-llm-1b-q4_k.gguf - files: - - filename: omniasr-llm-1b-q4_k.gguf - uri: huggingface://cstr/omniasr-llm-1b-GGUF/omniasr-llm-1b-q4_k.gguf -- name: mimo-asr-crispasr - url: github:mudler/LocalAI/gallery/mimo-asr-crispasr.yaml@master - urls: - - https://huggingface.co/cstr/mimo-asr-GGUF - description: | - MiMo-ASR (audio LLM) ASR. Runs via the CrispASR backend. Default GGUF size ~4.2 GB. - tags: - - crispasr - - asr - - speech-recognition - - stt - - gguf - overrides: - backend: crispasr - known_usecases: - - transcript - name: mimo-asr-crispasr - parameters: - model: mimo-asr-q4_k.gguf - files: - - filename: mimo-asr-q4_k.gguf - uri: huggingface://cstr/mimo-asr-GGUF/mimo-asr-q4_k.gguf - - filename: mimo-tokenizer-q4_k.gguf - uri: huggingface://cstr/mimo-tokenizer-GGUF/mimo-tokenizer-q4_k.gguf - name: vibevoice-crispasr url: github:mudler/LocalAI/gallery/vibevoice-crispasr.yaml@master urls: @@ -32548,167 +32256,3 @@ files: - filename: vibevoice-asr-q4_k.gguf uri: huggingface://cstr/vibevoice-asr-GGUF/vibevoice-asr-q4_k.gguf -- name: firered-asr-crispasr - url: github:mudler/LocalAI/gallery/firered-asr-crispasr.yaml@master - urls: - - https://huggingface.co/cstr/firered-asr2-aed-GGUF - description: | - FireRed-ASR2 AED (attention encoder-decoder) ASR. Runs via the CrispASR backend. Default GGUF size ~918 MB. - tags: - - crispasr - - asr - - speech-recognition - - stt - - gguf - overrides: - backend: crispasr - known_usecases: - - transcript - name: firered-asr-crispasr - parameters: - model: firered-asr2-aed-q4_k.gguf - files: - - filename: firered-asr2-aed-q4_k.gguf - uri: huggingface://cstr/firered-asr2-aed-GGUF/firered-asr2-aed-q4_k.gguf -- name: kyutai-stt-crispasr - url: github:mudler/LocalAI/gallery/kyutai-stt-crispasr.yaml@master - urls: - - https://huggingface.co/cstr/kyutai-stt-1b-GGUF - description: | - Kyutai STT 1B (streaming) ASR. Runs via the CrispASR backend. Default GGUF size ~636 MB. - tags: - - crispasr - - asr - - speech-recognition - - stt - - gguf - overrides: - backend: crispasr - known_usecases: - - transcript - name: kyutai-stt-crispasr - parameters: - model: kyutai-stt-1b-q4_k.gguf - files: - - filename: kyutai-stt-1b-q4_k.gguf - uri: huggingface://cstr/kyutai-stt-1b-GGUF/kyutai-stt-1b-q4_k.gguf -- name: glm-asr-crispasr - url: github:mudler/LocalAI/gallery/glm-asr-crispasr.yaml@master - urls: - - https://huggingface.co/cstr/glm-asr-nano-GGUF - description: | - GLM-ASR Nano ASR. Runs via the CrispASR backend. Default GGUF size ~1.2 GB. - tags: - - crispasr - - asr - - speech-recognition - - stt - - gguf - overrides: - backend: crispasr - known_usecases: - - transcript - name: glm-asr-crispasr - parameters: - model: glm-asr-nano-q4_k.gguf - files: - - filename: glm-asr-nano-q4_k.gguf - uri: huggingface://cstr/glm-asr-nano-GGUF/glm-asr-nano-q4_k.gguf -- name: moonshine-crispasr - url: github:mudler/LocalAI/gallery/moonshine-crispasr.yaml@master - urls: - - https://huggingface.co/cstr/moonshine-tiny-GGUF - description: | - Moonshine Tiny (Useful Sensors) English ASR. Runs via the CrispASR backend. Default GGUF size ~20 MB. - tags: - - crispasr - - asr - - speech-recognition - - stt - - gguf - overrides: - backend: crispasr - known_usecases: - - transcript - name: moonshine-crispasr - parameters: - model: moonshine-tiny-q4_k.gguf - files: - - filename: moonshine-tiny-q4_k.gguf - uri: huggingface://cstr/moonshine-tiny-GGUF/moonshine-tiny-q4_k.gguf - - filename: tokenizer.bin - uri: huggingface://cstr/moonshine-tiny-GGUF/tokenizer.bin -- name: moonshine-de-crispasr - url: github:mudler/LocalAI/gallery/moonshine-de-crispasr.yaml@master - urls: - - https://huggingface.co/cstr/moonshine-base-de-fidoriel-GGUF - description: | - Moonshine Base German (fidoriel fine-tune) ASR. Runs via the CrispASR backend. Default GGUF size ~39 MB. - license: "CC-BY-NC-SA-4.0" - tags: - - crispasr - - asr - - speech-recognition - - stt - - gguf - overrides: - backend: crispasr - known_usecases: - - transcript - name: moonshine-de-crispasr - parameters: - model: moonshine-base-de-fidoriel-q4_k.gguf - files: - - filename: moonshine-base-de-fidoriel-q4_k.gguf - uri: huggingface://cstr/moonshine-base-de-fidoriel-GGUF/moonshine-base-de-fidoriel-q4_k.gguf - - filename: tokenizer.bin - uri: huggingface://cstr/moonshine-base-de-fidoriel-GGUF/tokenizer.bin -- name: moonshine-tiny-de-crispasr - url: github:mudler/LocalAI/gallery/moonshine-tiny-de-crispasr.yaml@master - urls: - - https://huggingface.co/cstr/moonshine-tiny-de-fidoriel-GGUF - description: | - Moonshine Tiny German (fidoriel fine-tune) ASR. Runs via the CrispASR backend. Default GGUF size ~17 MB. - license: "CC-BY-NC-SA-4.0" - tags: - - crispasr - - asr - - speech-recognition - - stt - - gguf - overrides: - backend: crispasr - known_usecases: - - transcript - name: moonshine-tiny-de-crispasr - parameters: - model: moonshine-tiny-de-fidoriel-q4_k.gguf - files: - - filename: moonshine-tiny-de-fidoriel-q4_k.gguf - uri: huggingface://cstr/moonshine-tiny-de-fidoriel-GGUF/moonshine-tiny-de-fidoriel-q4_k.gguf - - filename: tokenizer.bin - uri: huggingface://cstr/moonshine-tiny-de-fidoriel-GGUF/tokenizer.bin -- name: moonshine-streaming-crispasr - url: github:mudler/LocalAI/gallery/moonshine-streaming-crispasr.yaml@master - urls: - - https://huggingface.co/cstr/moonshine-streaming-tiny-GGUF - description: | - Moonshine Streaming Tiny English ASR. Runs via the CrispASR backend. Default GGUF size ~31 MB. - tags: - - crispasr - - asr - - speech-recognition - - stt - - gguf - overrides: - backend: crispasr - known_usecases: - - transcript - name: moonshine-streaming-crispasr - parameters: - model: moonshine-streaming-tiny-q4_k.gguf - files: - - filename: moonshine-streaming-tiny-q4_k.gguf - uri: huggingface://cstr/moonshine-streaming-tiny-GGUF/moonshine-streaming-tiny-q4_k.gguf - - filename: tokenizer.bin - uri: huggingface://cstr/moonshine-streaming-tiny-GGUF/tokenizer.bin diff --git a/gallery/kyutai-stt-crispasr.yaml b/gallery/kyutai-stt-crispasr.yaml deleted file mode 100644 index 903c459612d4..000000000000 --- a/gallery/kyutai-stt-crispasr.yaml +++ /dev/null @@ -1,7 +0,0 @@ ---- -name: "kyutai-stt-crispasr" - -config_file: | - backend: crispasr - parameters: - model: kyutai-stt-1b-q4_k.gguf diff --git a/gallery/mega-asr-crispasr.yaml b/gallery/mega-asr-crispasr.yaml deleted file mode 100644 index b26ea4b05ad3..000000000000 --- a/gallery/mega-asr-crispasr.yaml +++ /dev/null @@ -1,7 +0,0 @@ ---- -name: "mega-asr-crispasr" - -config_file: | - backend: crispasr - parameters: - model: mega-asr-1.7b-q4_k.gguf diff --git a/gallery/mimo-asr-crispasr.yaml b/gallery/mimo-asr-crispasr.yaml deleted file mode 100644 index 3bab1e151478..000000000000 --- a/gallery/mimo-asr-crispasr.yaml +++ /dev/null @@ -1,7 +0,0 @@ ---- -name: "mimo-asr-crispasr" - -config_file: | - backend: crispasr - parameters: - model: mimo-asr-q4_k.gguf diff --git a/gallery/moonshine-crispasr.yaml b/gallery/moonshine-crispasr.yaml deleted file mode 100644 index 27c27dcdde27..000000000000 --- a/gallery/moonshine-crispasr.yaml +++ /dev/null @@ -1,7 +0,0 @@ ---- -name: "moonshine-crispasr" - -config_file: | - backend: crispasr - parameters: - model: moonshine-tiny-q4_k.gguf diff --git a/gallery/moonshine-de-crispasr.yaml b/gallery/moonshine-de-crispasr.yaml deleted file mode 100644 index 1ccc03273274..000000000000 --- a/gallery/moonshine-de-crispasr.yaml +++ /dev/null @@ -1,7 +0,0 @@ ---- -name: "moonshine-de-crispasr" - -config_file: | - backend: crispasr - parameters: - model: moonshine-base-de-fidoriel-q4_k.gguf diff --git a/gallery/moonshine-streaming-crispasr.yaml b/gallery/moonshine-streaming-crispasr.yaml deleted file mode 100644 index b5708a731b68..000000000000 --- a/gallery/moonshine-streaming-crispasr.yaml +++ /dev/null @@ -1,7 +0,0 @@ ---- -name: "moonshine-streaming-crispasr" - -config_file: | - backend: crispasr - parameters: - model: moonshine-streaming-tiny-q4_k.gguf diff --git a/gallery/moonshine-tiny-de-crispasr.yaml b/gallery/moonshine-tiny-de-crispasr.yaml deleted file mode 100644 index 08f15391e7cb..000000000000 --- a/gallery/moonshine-tiny-de-crispasr.yaml +++ /dev/null @@ -1,7 +0,0 @@ ---- -name: "moonshine-tiny-de-crispasr" - -config_file: | - backend: crispasr - parameters: - model: moonshine-tiny-de-fidoriel-q4_k.gguf diff --git a/gallery/omniasr-crispasr.yaml b/gallery/omniasr-crispasr.yaml deleted file mode 100644 index 4f70cfb89fa1..000000000000 --- a/gallery/omniasr-crispasr.yaml +++ /dev/null @@ -1,7 +0,0 @@ ---- -name: "omniasr-crispasr" - -config_file: | - backend: crispasr - parameters: - model: omniasr-ctc-1b-v2-q4_k.gguf diff --git a/gallery/omniasr-llm-1b-crispasr.yaml b/gallery/omniasr-llm-1b-crispasr.yaml deleted file mode 100644 index e7b0db8b099e..000000000000 --- a/gallery/omniasr-llm-1b-crispasr.yaml +++ /dev/null @@ -1,7 +0,0 @@ ---- -name: "omniasr-llm-1b-crispasr" - -config_file: | - backend: crispasr - parameters: - model: omniasr-llm-1b-q4_k.gguf diff --git a/gallery/omniasr-llm-crispasr.yaml b/gallery/omniasr-llm-crispasr.yaml deleted file mode 100644 index fcb950763ecc..000000000000 --- a/gallery/omniasr-llm-crispasr.yaml +++ /dev/null @@ -1,7 +0,0 @@ ---- -name: "omniasr-llm-crispasr" - -config_file: | - backend: crispasr - parameters: - model: omniasr-llm-300m-v2-q4_k.gguf diff --git a/gallery/paraformer-crispasr.yaml b/gallery/paraformer-crispasr.yaml deleted file mode 100644 index 045374dadc25..000000000000 --- a/gallery/paraformer-crispasr.yaml +++ /dev/null @@ -1,7 +0,0 @@ ---- -name: "paraformer-crispasr" - -config_file: | - backend: crispasr - parameters: - model: paraformer-zh-q4_k.gguf diff --git a/gallery/parakeet-ctc-0.6b-crispasr.yaml b/gallery/parakeet-ctc-0.6b-crispasr.yaml deleted file mode 100644 index aec82e23f09e..000000000000 --- a/gallery/parakeet-ctc-0.6b-crispasr.yaml +++ /dev/null @@ -1,7 +0,0 @@ ---- -name: "parakeet-ctc-0.6b-crispasr" - -config_file: | - backend: crispasr - parameters: - model: parakeet-ctc-0.6b-q4_k.gguf diff --git a/gallery/parakeet-ctc-1.1b-crispasr.yaml b/gallery/parakeet-ctc-1.1b-crispasr.yaml deleted file mode 100644 index 47cc4299173d..000000000000 --- a/gallery/parakeet-ctc-1.1b-crispasr.yaml +++ /dev/null @@ -1,7 +0,0 @@ ---- -name: "parakeet-ctc-1.1b-crispasr" - -config_file: | - backend: crispasr - parameters: - model: parakeet-ctc-1.1b-q4_k.gguf diff --git a/gallery/sensevoice-crispasr.yaml b/gallery/sensevoice-crispasr.yaml deleted file mode 100644 index 2be065049e35..000000000000 --- a/gallery/sensevoice-crispasr.yaml +++ /dev/null @@ -1,7 +0,0 @@ ---- -name: "sensevoice-crispasr" - -config_file: | - backend: crispasr - parameters: - model: sensevoice-small-q4_k.gguf From 8d6e7aff4f39824a633600e108128b62f25b471b Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sun, 31 May 2026 07:11:38 +0000 Subject: [PATCH 18/29] feat(crispasr): TTS synthesis via crispasr_session_synthesize (24kHz) Add tts_synthesize/tts_free/tts_set_voice to the C-ABI shim. They reuse the already-open g_session (crispasr_session_open auto-detects a TTS model) and dispatch to the upstream synthesis call, which returns malloc'd 24 kHz mono float PCM. Orpheus needs a SNAC codec path that we do not set, so it returns NULL here and surfaces as an error Go-side. Signed-off-by: Ettore Di Giacinto --- backend/go/crispasr/cpp/crispasr_shim.cpp | 22 ++++++++++++++++++++++ backend/go/crispasr/cpp/crispasr_shim.h | 3 +++ 2 files changed, 25 insertions(+) diff --git a/backend/go/crispasr/cpp/crispasr_shim.cpp b/backend/go/crispasr/cpp/crispasr_shim.cpp index b1b23231ff99..592c897c7dc6 100644 --- a/backend/go/crispasr/cpp/crispasr_shim.cpp +++ b/backend/go/crispasr/cpp/crispasr_shim.cpp @@ -23,6 +23,10 @@ const char *crispasr_session_result_segment_text(crispasr_session_result *r, int64_t crispasr_session_result_segment_t0(crispasr_session_result *r, int i); int64_t crispasr_session_result_segment_t1(crispasr_session_result *r, int i); void crispasr_session_result_free(crispasr_session_result *r); +float *crispasr_session_synthesize(crispasr_session *s, const char *text, + int *out_n_samples); +void crispasr_pcm_free(float *pcm); +int crispasr_session_set_speaker_name(crispasr_session *s, const char *name); } static crispasr_session *g_session = nullptr; @@ -199,3 +203,21 @@ int64_t get_segment_t1(int i) { const char *get_backend(void) { return g_session ? crispasr_session_backend(g_session) : ""; } + +// TTS uses the already-open session (crispasr_session_open auto-detects a TTS +// model). Output is 24 kHz mono float PCM (upstream CrispASR convention), +// malloc'd by the C API; the caller must release it via tts_free. +float *tts_synthesize(const char *text, int *out_n_samples) { + if (out_n_samples) *out_n_samples = 0; + if (!g_session || !text) return nullptr; + return crispasr_session_synthesize(g_session, text, out_n_samples); +} + +void tts_free(float *pcm) { + if (pcm) crispasr_pcm_free(pcm); +} + +int tts_set_voice(const char *name) { + if (!g_session || !name || !*name) return 0; + return crispasr_session_set_speaker_name(g_session, name); +} diff --git a/backend/go/crispasr/cpp/crispasr_shim.h b/backend/go/crispasr/cpp/crispasr_shim.h index fa01097423cb..c993a86401c3 100644 --- a/backend/go/crispasr/cpp/crispasr_shim.h +++ b/backend/go/crispasr/cpp/crispasr_shim.h @@ -14,4 +14,7 @@ int64_t get_segment_t0(int i); int64_t get_segment_t1(int i); const char *get_backend(void); void set_abort(int v); +float *tts_synthesize(const char *text, int *out_n_samples); // 24kHz mono float, malloc'd; NULL on failure +void tts_free(float *pcm); +int tts_set_voice(const char *name); // best-effort speaker selection; 0 ok } From c95a256ce0b1b69dfd13ab5688f6c32f55f9ef2a Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sun, 31 May 2026 07:11:44 +0000 Subject: [PATCH 19/29] feat(crispasr): implement TTS/TTSStream gRPC methods Bind the new shim functions via purego and implement TTS, TTSStream and a writeWAV24k helper. synthesize copies the C-owned PCM out before freeing it; TTS writes a 24 kHz mono 16-bit WAV to req.Dst via go-audio/wav. CrispASR has no progressive synth, so TTSStream synthesizes fully, encodes to WAV, and emits the bytes as a single chunk; it owns the results-channel close (the gRPC server wrapper ranges until close), mirroring vibevoice-cpp's TTSStream. Signed-off-by: Ettore Di Giacinto --- backend/go/crispasr/gocrispasr.go | 113 ++++++++++++++++++++++++++++++ backend/go/crispasr/main.go | 3 + 2 files changed, 116 insertions(+) diff --git a/backend/go/crispasr/gocrispasr.go b/backend/go/crispasr/gocrispasr.go index 17d93502aa1c..04dec9747733 100644 --- a/backend/go/crispasr/gocrispasr.go +++ b/backend/go/crispasr/gocrispasr.go @@ -9,6 +9,7 @@ import ( "sync" "unsafe" + "github.com/go-audio/audio" "github.com/go-audio/wav" "github.com/mudler/LocalAI/pkg/grpc/base" pb "github.com/mudler/LocalAI/pkg/grpc/proto" @@ -27,6 +28,9 @@ var ( CppGetSegmentEnd func(i int) int64 CppGetBackend func() string CppSetAbort func(v int) + CppTTSSynthesize func(text string, outNSamples unsafe.Pointer) uintptr + CppTTSFree func(ptr uintptr) + CppTTSSetVoice func(name string) int ) type CrispASR struct { @@ -304,3 +308,112 @@ func (w *CrispASR) AudioTranscriptionStream(ctx context.Context, opts *pb.Transc results <- &pb.TranscriptStreamResponse{FinalResult: final} return nil } + +// synthesize returns 24 kHz mono float32 PCM for text via the open session. +func (w *CrispASR) synthesize(text string) ([]float32, error) { + if text == "" { + return nil, fmt.Errorf("crispasr: TTS requires non-empty text") + } + var n int32 + ptr := CppTTSSynthesize(text, unsafe.Pointer(&n)) + if ptr == 0 || n <= 0 { + return nil, fmt.Errorf("crispasr: synthesis failed (the loaded model may not be a supported TTS backend, or needs extra config e.g. orpheus SNAC codec)") + } + defer CppTTSFree(ptr) + src := unsafe.Slice((*float32)(unsafe.Pointer(ptr)), int(n)) + out := make([]float32, int(n)) // copy out of C memory before free + copy(out, src) + return out, nil +} + +func (w *CrispASR) TTS(req *pb.TTSRequest) error { + if req.Dst == "" { + return fmt.Errorf("crispasr: TTS requires a destination path") + } + if v := strings.TrimSpace(req.Voice); v != "" { + CppTTSSetVoice(v) + } + pcm, err := w.synthesize(req.Text) + if err != nil { + return err + } + return writeWAV24k(req.Dst, pcm) +} + +// TTSStream is the streaming counterpart to TTS. CrispASR has no progressive +// (native streaming) synth, so we synthesize the whole utterance, encode it to +// a 24 kHz WAV, and emit the encoded bytes as a single chunk. The gRPC server +// wrapper (pkg/grpc/server.go:TTSStream) ranges over the channel until it is +// closed, so this method owns the close - mirrors vibevoice-cpp's TTSStream. +func (w *CrispASR) TTSStream(req *pb.TTSRequest, results chan []byte) error { + defer close(results) + + if req.Text == "" { + return fmt.Errorf("crispasr: TTSStream requires text") + } + if v := strings.TrimSpace(req.Voice); v != "" { + CppTTSSetVoice(v) + } + pcm, err := w.synthesize(req.Text) + if err != nil { + return err + } + + tmp, err := os.CreateTemp("", "crispasr-tts-stream-*.wav") + if err != nil { + return fmt.Errorf("crispasr: tempfile: %w", err) + } + dst := tmp.Name() + if err := tmp.Close(); err != nil { + return fmt.Errorf("crispasr: close tempfile: %w", err) + } + defer func() { _ = os.Remove(dst) }() + + if err := writeWAV24k(dst, pcm); err != nil { + return err + } + + encoded, err := os.ReadFile(dst) + if err != nil { + return fmt.Errorf("crispasr: read tempfile: %w", err) + } + results <- encoded + return nil +} + +// writeWAV24k writes pcm as a 24000 Hz, mono, 16-bit PCM WAV at dst. +func writeWAV24k(dst string, pcm []float32) error { + f, err := os.Create(dst) + if err != nil { + return fmt.Errorf("crispasr: create %q: %w", dst, err) + } + + enc := wav.NewEncoder(f, 24000, 16, 1, 1) + ints := make([]int, len(pcm)) + for i, s := range pcm { + if s > 1 { + s = 1 + } else if s < -1 { + s = -1 + } + ints[i] = int(s * 32767) + } + buf := &audio.IntBuffer{ + Format: &audio.Format{NumChannels: 1, SampleRate: 24000}, + Data: ints, + SourceBitDepth: 16, + } + if err := enc.Write(buf); err != nil { + _ = enc.Close() + _ = f.Close() + return fmt.Errorf("crispasr: encode WAV: %w", err) + } + if err := enc.Close(); err != nil { + _ = f.Close() + return fmt.Errorf("crispasr: finalize WAV: %w", err) + } + if err := f.Close(); err != nil { + return fmt.Errorf("crispasr: close %q: %w", dst, err) + } + return nil +} diff --git a/backend/go/crispasr/main.go b/backend/go/crispasr/main.go index 4d0f2b92b2ab..e178b48be5b3 100644 --- a/backend/go/crispasr/main.go +++ b/backend/go/crispasr/main.go @@ -39,6 +39,9 @@ func main() { {&CppGetSegmentEnd, "get_segment_t1"}, {&CppGetBackend, "get_backend"}, {&CppSetAbort, "set_abort"}, + {&CppTTSSynthesize, "tts_synthesize"}, + {&CppTTSFree, "tts_free"}, + {&CppTTSSetVoice, "tts_set_voice"}, } for _, lf := range libFuncs { From 4f66ecf6e3fec2f6aeb41f593d226fbc67747f6a Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sun, 31 May 2026 07:15:30 +0000 Subject: [PATCH 20/29] feat(crispasr): log when a TTS voice override is not honored Signed-off-by: Ettore Di Giacinto --- backend/go/crispasr/gocrispasr.go | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/backend/go/crispasr/gocrispasr.go b/backend/go/crispasr/gocrispasr.go index 04dec9747733..22a9375de5b4 100644 --- a/backend/go/crispasr/gocrispasr.go +++ b/backend/go/crispasr/gocrispasr.go @@ -326,13 +326,24 @@ func (w *CrispASR) synthesize(text string) ([]float32, error) { return out, nil } +// setVoice applies a per-call speaker/voice override (best effort). CrispASR +// returns a negative code when the active backend can't honor the name; we log +// it rather than fail, so an unknown voice falls back to the default speaker. +func setVoice(voice string) { + v := strings.TrimSpace(voice) + if v == "" { + return + } + if rc := CppTTSSetVoice(v); rc != 0 { + fmt.Fprintf(os.Stderr, "crispasr: voice %q not applied by the active TTS backend (rc=%d); using default\n", v, rc) + } +} + func (w *CrispASR) TTS(req *pb.TTSRequest) error { if req.Dst == "" { return fmt.Errorf("crispasr: TTS requires a destination path") } - if v := strings.TrimSpace(req.Voice); v != "" { - CppTTSSetVoice(v) - } + setVoice(req.Voice) pcm, err := w.synthesize(req.Text) if err != nil { return err @@ -351,9 +362,7 @@ func (w *CrispASR) TTSStream(req *pb.TTSRequest, results chan []byte) error { if req.Text == "" { return fmt.Errorf("crispasr: TTSStream requires text") } - if v := strings.TrimSpace(req.Voice); v != "" { - CppTTSSetVoice(v) - } + setVoice(req.Voice) pcm, err := w.synthesize(req.Text) if err != nil { return err From 39dcbb0088f952b05d60a3e6c689e57e5f52c2a1 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sun, 31 May 2026 07:20:02 +0000 Subject: [PATCH 21/29] feat(gallery): add CrispASR vibevoice-tts model entry Only vibevoice-tts works through the current shim: qwen3-tts, chatterbox, and orpheus require companion codec/s3gen/SNAC paths (set_codec_path / set_s3gen_path) that the shim doesn't wire yet, and kokoro/indextts/voxcpm2 aren't in the session auto-detect map. Those are follow-ups. Signed-off-by: Ettore Di Giacinto --- gallery/index.yaml | 21 +++++++++++++++++++++ gallery/vibevoice-tts-crispasr.yaml | 7 +++++++ 2 files changed, 28 insertions(+) create mode 100644 gallery/vibevoice-tts-crispasr.yaml diff --git a/gallery/index.yaml b/gallery/index.yaml index 9da2a1c3d891..061e8b64384a 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -32256,3 +32256,24 @@ files: - filename: vibevoice-asr-q4_k.gguf uri: huggingface://cstr/vibevoice-asr-GGUF/vibevoice-asr-q4_k.gguf +- name: vibevoice-tts-crispasr + url: github:mudler/LocalAI/gallery/vibevoice-tts-crispasr.yaml@master + urls: + - https://huggingface.co/cstr/vibevoice-realtime-0.5b-GGUF + description: | + VibeVoice Realtime 0.5B text-to-speech (TTS) model, synthesized through the CrispASR backend. Produces 24 kHz mono audio; runs end-to-end on CPU with a built-in default voice. Default GGUF size ~636 MB. + tags: + - crispasr + - tts + - text-to-speech + - gguf + overrides: + backend: crispasr + known_usecases: + - tts + name: vibevoice-tts-crispasr + parameters: + model: vibevoice-realtime-0.5b-q4_k.gguf + files: + - filename: vibevoice-realtime-0.5b-q4_k.gguf + uri: huggingface://cstr/vibevoice-realtime-0.5b-GGUF/vibevoice-realtime-0.5b-q4_k.gguf diff --git a/gallery/vibevoice-tts-crispasr.yaml b/gallery/vibevoice-tts-crispasr.yaml new file mode 100644 index 000000000000..dae60fe4f419 --- /dev/null +++ b/gallery/vibevoice-tts-crispasr.yaml @@ -0,0 +1,7 @@ +--- +name: "vibevoice-tts-crispasr" + +config_file: | + backend: crispasr + parameters: + model: vibevoice-realtime-0.5b-q4_k.gguf From 10e8100b7fb771609c57c5972d262adfc0854a6f Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sun, 31 May 2026 07:23:47 +0000 Subject: [PATCH 22/29] test(crispasr): gated TTS synthesis spec Signed-off-by: Ettore Di Giacinto --- backend/go/crispasr/gocrispasr_test.go | 35 ++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/backend/go/crispasr/gocrispasr_test.go b/backend/go/crispasr/gocrispasr_test.go index 658d4081dffb..5d1c2175ac2d 100644 --- a/backend/go/crispasr/gocrispasr_test.go +++ b/backend/go/crispasr/gocrispasr_test.go @@ -3,6 +3,7 @@ package main import ( "context" "os" + "path/filepath" "strings" "sync" "testing" @@ -50,6 +51,9 @@ func ensureLibLoaded() { purego.RegisterLibFunc(&CppGetSegmentEnd, gosd, "get_segment_t1") purego.RegisterLibFunc(&CppGetBackend, gosd, "get_backend") purego.RegisterLibFunc(&CppSetAbort, gosd, "set_abort") + purego.RegisterLibFunc(&CppTTSSynthesize, gosd, "tts_synthesize") + purego.RegisterLibFunc(&CppTTSFree, gosd, "tts_free") + purego.RegisterLibFunc(&CppTTSSetVoice, gosd, "tts_set_voice") }) if libLoadErr != nil { Skip("whisper library not loadable: " + libLoadErr.Error()) @@ -68,6 +72,17 @@ func fixturesOrSkip() (string, string) { return modelPath, audioPath } +// ttsModelOrSkip returns the TTS model path or skips the spec when the env var +// is unset. Like the transcription fixtures, this never runs in default CI — it +// needs a real TTS model (e.g. a vibevoice GGUF) on disk. +func ttsModelOrSkip() string { + modelPath := os.Getenv("CRISPASR_TTS_MODEL_PATH") + if modelPath == "" { + Skip("set CRISPASR_TTS_MODEL_PATH to run this spec") + } + return modelPath +} + var _ = Describe("CrispASR", func() { Context("AudioTranscription cancellation", func() { It("returns codes.Canceled on a pre-cancelled context and still succeeds afterwards", func() { @@ -153,4 +168,24 @@ var _ = Describe("CrispASR", func() { "concat(deltas) must equal final.Text") }) }) + + Context("TTS", func() { + It("synthesizes a non-empty WAV", func() { + ttsModel := ttsModelOrSkip() + ensureLibLoaded() + + w := &CrispASR{} + Expect(w.Load(&pb.ModelOptions{ModelFile: ttsModel})).To(Succeed()) + + dst := filepath.Join(GinkgoT().TempDir(), "out.wav") + Expect(w.TTS(&pb.TTSRequest{Text: "Hello from CrispASR.", Dst: dst})).To(Succeed()) + + info, err := os.Stat(dst) + Expect(err).ToNot(HaveOccurred(), "synthesized WAV should exist at %q", dst) + // A real 24 kHz mono WAV is a 44-byte header plus samples; anything + // this small would mean an empty/failed synth. + Expect(info.Size()).To(BeNumerically(">", 1024), + "expected a non-trivial WAV, got %d bytes", info.Size()) + }) + }) }) From 0084dc8721c9a106cbb59b0fef8884e3ef76fc83 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sun, 31 May 2026 07:28:30 +0000 Subject: [PATCH 23/29] fix(crispasr): satisfy golangci-lint (errcheck defers + unsafeptr nolint) The crispasr Go file is entirely new, so new-from-merge-base lints every line (unlike the grandfathered whisper backend it was forked from): - handle os.RemoveAll / fh.Close return values in AudioTranscription - annotate the two intentional C-pointer unsafe.Slice sites with //nolint:govet Signed-off-by: Ettore Di Giacinto --- backend/go/crispasr/gocrispasr.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/backend/go/crispasr/gocrispasr.go b/backend/go/crispasr/gocrispasr.go index 22a9375de5b4..bb8df5e6fa6e 100644 --- a/backend/go/crispasr/gocrispasr.go +++ b/backend/go/crispasr/gocrispasr.go @@ -84,7 +84,7 @@ func (w *CrispASR) VAD(req *pb.VADRequest) (pb.VADResponse, error) { // unsafeptr warning is caused by segsPtr being on the stack and therefor being subject to stack copying AFAICT // however the stack shouldn't have grown between setting segsPtr and now, also the memory pointed to is allocated by C++ - segs := unsafe.Slice((*float32)(unsafe.Pointer(segsPtr)), segsLen) + segs := unsafe.Slice((*float32)(unsafe.Pointer(segsPtr)), segsLen) //nolint:govet // segsPtr addresses C++-owned heap memory passed back through the cgo-free purego boundary; the uintptr->Pointer round-trip is intentional and the buffer outlives this read. vadSegments := []*pb.VADSegment{} for i := range len(segs) >> 1 { @@ -110,7 +110,7 @@ func (w *CrispASR) AudioTranscription(ctx context.Context, opts *pb.TranscriptRe if err != nil { return pb.TranscriptResult{}, err } - defer os.RemoveAll(dir) + defer func() { _ = os.RemoveAll(dir) }() convertedPath := filepath.Join(dir, "converted.wav") @@ -122,7 +122,7 @@ func (w *CrispASR) AudioTranscription(ctx context.Context, opts *pb.TranscriptRe if err != nil { return pb.TranscriptResult{}, err } - defer fh.Close() + defer func() { _ = fh.Close() }() d := wav.NewDecoder(fh) buf, err := d.FullPCMBuffer() @@ -320,7 +320,7 @@ func (w *CrispASR) synthesize(text string) ([]float32, error) { return nil, fmt.Errorf("crispasr: synthesis failed (the loaded model may not be a supported TTS backend, or needs extra config e.g. orpheus SNAC codec)") } defer CppTTSFree(ptr) - src := unsafe.Slice((*float32)(unsafe.Pointer(ptr)), int(n)) + src := unsafe.Slice((*float32)(unsafe.Pointer(ptr)), int(n)) //nolint:govet // ptr addresses C-allocated PCM returned across the purego boundary; copied out immediately below, before tts_free. out := make([]float32, int(n)) // copy out of C memory before free copy(out, src) return out, nil From eb5cbf98032d7756773caa9fa5492fb4f1bfedbd Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sun, 31 May 2026 07:38:09 +0000 Subject: [PATCH 24/29] feat(crispasr): backend: and codec: model options (explicit arch + companion files) Add two model-config options to the CrispASR backend via opts.Options: - backend: selects an explicit CrispASR backend (bypassing auto-detect) by routing load_model through crispasr_session_open_explicit, unlocking architectures the detector won't pick on its own (qwen3, cohere, granite, voxtral, moonshine, mimo-asr, orpheus, kokoro, chatterbox, etc.). - codec: loads a companion file (qwen3-tts codec, orpheus SNAC, chatterbox s3gen, or mimo-asr tokenizer) via the universal crispasr_session_set_codec_path setter after the session opens. A relative path resolves against the model directory. rc==0 means success or not-applicable; only a negative rc is fatal. The C shim load_model gains a backend_name argument and a new set_codec_path entry point; the Go bridge parses the prefix:value options and registers the new symbol. The vad_only path is unchanged. Signed-off-by: Ettore Di Giacinto --- backend/go/crispasr/cpp/crispasr_shim.cpp | 22 ++++++++++-- backend/go/crispasr/cpp/crispasr_shim.h | 4 ++- backend/go/crispasr/gocrispasr.go | 43 +++++++++++++++++++++-- backend/go/crispasr/gocrispasr_test.go | 1 + backend/go/crispasr/main.go | 1 + 5 files changed, 65 insertions(+), 6 deletions(-) diff --git a/backend/go/crispasr/cpp/crispasr_shim.cpp b/backend/go/crispasr/cpp/crispasr_shim.cpp index 592c897c7dc6..11e897b0f6b3 100644 --- a/backend/go/crispasr/cpp/crispasr_shim.cpp +++ b/backend/go/crispasr/cpp/crispasr_shim.cpp @@ -12,6 +12,10 @@ struct crispasr_session_result; extern "C" { crispasr_session *crispasr_session_open(const char *model_path, int n_threads); +crispasr_session *crispasr_session_open_explicit(const char *model_path, + const char *backend_name, + int n_threads); +int crispasr_session_set_codec_path(crispasr_session *s, const char *path); void crispasr_session_close(crispasr_session *s); const char *crispasr_session_backend(crispasr_session *s); int crispasr_session_set_translate(crispasr_session *s, int enable); @@ -72,11 +76,17 @@ static void ggml_log_cb(enum ggml_log_level level, const char *log, fflush(stderr); } -int load_model(const char *const model_path, int threads) { +int load_model(const char *const model_path, int threads, + const char *backend_name) { whisper_log_set(ggml_log_cb, nullptr); ggml_backend_load_all(); - g_session = crispasr_session_open(model_path, threads); + if (backend_name && *backend_name) { + g_session = + crispasr_session_open_explicit(model_path, backend_name, threads); + } else { + g_session = crispasr_session_open(model_path, threads); + } if (g_session == nullptr) { fprintf(stderr, "error: failed to open CrispASR session for model\n"); return 1; @@ -87,6 +97,14 @@ int load_model(const char *const model_path, int threads) { return 0; } +// set_codec_path forwards a companion file (qwen3-tts codec, orpheus SNAC, +// chatterbox s3gen, or mimo-asr tokenizer) to the active session. Returns 0 on +// success or when the active backend needs no companion, negative on failure, +// and -1 when no session is open. +int set_codec_path(const char *path) { + return g_session ? crispasr_session_set_codec_path(g_session, path) : -1; +} + int load_model_vad(const char *const model_path) { whisper_log_set(ggml_log_cb, nullptr); ggml_backend_load_all(); diff --git a/backend/go/crispasr/cpp/crispasr_shim.h b/backend/go/crispasr/cpp/crispasr_shim.h index c993a86401c3..40f1c31f378a 100644 --- a/backend/go/crispasr/cpp/crispasr_shim.h +++ b/backend/go/crispasr/cpp/crispasr_shim.h @@ -2,7 +2,9 @@ #include extern "C" { -int load_model(const char *const model_path, int threads); +int load_model(const char *const model_path, int threads, + const char *backend_name); +int set_codec_path(const char *path); int load_model_vad(const char *const model_path); int vad(float pcmf32[], size_t pcmf32_size, float **segs_out, size_t *segs_out_len); diff --git a/backend/go/crispasr/gocrispasr.go b/backend/go/crispasr/gocrispasr.go index bb8df5e6fa6e..9ecb5543ba54 100644 --- a/backend/go/crispasr/gocrispasr.go +++ b/backend/go/crispasr/gocrispasr.go @@ -19,7 +19,8 @@ import ( ) var ( - CppLoadModel func(modelPath string, threads int) int + CppLoadModel func(modelPath string, threads int, backendName string) int + CppSetCodecPath func(path string) int CppLoadModelVAD func(modelPath string) int CppVAD func(pcmf32 []float32, pcmf32Size uintptr, segsOut unsafe.Pointer, segsOutLen unsafe.Pointer) int CppTranscribe func(threads uint32, lang string, translate bool, diarize bool, pcmf32 []float32, pcmf32Len uintptr, segsOutLen unsafe.Pointer, prompt string) int @@ -37,13 +38,33 @@ type CrispASR struct { base.SingleThread } +// splitOption splits a "prefix:value" model option into its key and value, +// matching the convention used by other backends (see sherpa-onnx). It returns +// ok=false when the option carries no ':' separator. +func splitOption(oo string) (key, value string, ok bool) { + parts := strings.SplitN(oo, ":", 2) + if len(parts) != 2 { + return "", "", false + } + return parts[0], parts[1], true +} + func (w *CrispASR) Load(opts *pb.ModelOptions) error { vadOnly := false + backendName := "" + codecPath := "" for _, oo := range opts.Options { if oo == "vad_only" { vadOnly = true - } else { + continue + } + switch key, value, ok := splitOption(oo); { + case ok && key == "backend": + backendName = value + case ok && key == "codec": + codecPath = value + default: fmt.Fprintf(os.Stderr, "Unrecognized option: %v\n", oo) } } @@ -56,10 +77,26 @@ func (w *CrispASR) Load(opts *pb.ModelOptions) error { return nil } - if ret := CppLoadModel(opts.ModelFile, int(opts.Threads)); ret != 0 { + // Resolve a relative companion path against the model directory so a config + // can reference a sibling codec/tokenizer file by name alone. + if codecPath != "" && !filepath.IsAbs(codecPath) { + codecPath = filepath.Join(filepath.Dir(opts.ModelFile), codecPath) + } + + if ret := CppLoadModel(opts.ModelFile, int(opts.Threads), backendName); ret != 0 { return fmt.Errorf("Failed to load CrispASR transcription model") } + // Load the companion file (codec/tokenizer/s3gen) after the session is open. + // rc==0 means success or "not applicable" for the active backend; only a + // negative code is fatal. + if codecPath != "" { + if rc := CppSetCodecPath(codecPath); rc < 0 { + return fmt.Errorf("crispasr: failed to load companion file %q (rc=%d)", codecPath, rc) + } + fmt.Fprintf(os.Stderr, "CrispASR companion file loaded: %s\n", codecPath) + } + fmt.Fprintf(os.Stderr, "CrispASR backend selected: %s\n", CppGetBackend()) return nil diff --git a/backend/go/crispasr/gocrispasr_test.go b/backend/go/crispasr/gocrispasr_test.go index 5d1c2175ac2d..99417d33a333 100644 --- a/backend/go/crispasr/gocrispasr_test.go +++ b/backend/go/crispasr/gocrispasr_test.go @@ -45,6 +45,7 @@ func ensureLibLoaded() { return } purego.RegisterLibFunc(&CppLoadModel, gosd, "load_model") + purego.RegisterLibFunc(&CppSetCodecPath, gosd, "set_codec_path") purego.RegisterLibFunc(&CppTranscribe, gosd, "transcribe") purego.RegisterLibFunc(&CppGetSegmentText, gosd, "get_segment_text") purego.RegisterLibFunc(&CppGetSegmentStart, gosd, "get_segment_t0") diff --git a/backend/go/crispasr/main.go b/backend/go/crispasr/main.go index e178b48be5b3..09a90991f531 100644 --- a/backend/go/crispasr/main.go +++ b/backend/go/crispasr/main.go @@ -31,6 +31,7 @@ func main() { libFuncs := []LibFuncs{ {&CppLoadModel, "load_model"}, + {&CppSetCodecPath, "set_codec_path"}, {&CppLoadModelVAD, "load_model_vad"}, {&CppVAD, "vad"}, {&CppTranscribe, "transcribe"}, From 9d019ca7d8384dbc3d338b60d72ea35acc7f2725 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sun, 31 May 2026 07:45:02 +0000 Subject: [PATCH 25/29] feat(gallery): expand CrispASR models via backend:/codec: options (explicit arch + companions) Signed-off-by: Ettore Di Giacinto --- gallery/chatterbox-crispasr.yaml | 9 + gallery/data2vec-crispasr.yaml | 9 + gallery/firered-asr-crispasr.yaml | 9 + gallery/glm-asr-crispasr.yaml | 9 + gallery/hubert-crispasr.yaml | 9 + gallery/index.yaml | 332 ++++++++++++++++++++++ gallery/kyutai-stt-crispasr.yaml | 9 + gallery/mimo-asr-crispasr.yaml | 10 + gallery/moonshine-crispasr.yaml | 10 + gallery/moonshine-de-crispasr.yaml | 10 + gallery/moonshine-streaming-crispasr.yaml | 10 + gallery/moonshine-tiny-de-crispasr.yaml | 10 + gallery/orpheus-crispasr.yaml | 9 + gallery/qwen3-tts-crispasr.yaml | 9 + 14 files changed, 454 insertions(+) create mode 100644 gallery/chatterbox-crispasr.yaml create mode 100644 gallery/data2vec-crispasr.yaml create mode 100644 gallery/firered-asr-crispasr.yaml create mode 100644 gallery/glm-asr-crispasr.yaml create mode 100644 gallery/hubert-crispasr.yaml create mode 100644 gallery/kyutai-stt-crispasr.yaml create mode 100644 gallery/mimo-asr-crispasr.yaml create mode 100644 gallery/moonshine-crispasr.yaml create mode 100644 gallery/moonshine-de-crispasr.yaml create mode 100644 gallery/moonshine-streaming-crispasr.yaml create mode 100644 gallery/moonshine-tiny-de-crispasr.yaml create mode 100644 gallery/orpheus-crispasr.yaml create mode 100644 gallery/qwen3-tts-crispasr.yaml diff --git a/gallery/chatterbox-crispasr.yaml b/gallery/chatterbox-crispasr.yaml new file mode 100644 index 000000000000..21356a7b9775 --- /dev/null +++ b/gallery/chatterbox-crispasr.yaml @@ -0,0 +1,9 @@ +--- +name: "chatterbox-crispasr" + +config_file: | + backend: crispasr + options: + - "codec:chatterbox-s3gen-q8_0.gguf" + parameters: + model: chatterbox-t3-q8_0.gguf diff --git a/gallery/data2vec-crispasr.yaml b/gallery/data2vec-crispasr.yaml new file mode 100644 index 000000000000..064cd172c406 --- /dev/null +++ b/gallery/data2vec-crispasr.yaml @@ -0,0 +1,9 @@ +--- +name: "data2vec-crispasr" + +config_file: | + backend: crispasr + options: + - "backend:data2vec" + parameters: + model: data2vec-audio-base-960h-q4_k.gguf diff --git a/gallery/firered-asr-crispasr.yaml b/gallery/firered-asr-crispasr.yaml new file mode 100644 index 000000000000..79db1834fefe --- /dev/null +++ b/gallery/firered-asr-crispasr.yaml @@ -0,0 +1,9 @@ +--- +name: "firered-asr-crispasr" + +config_file: | + backend: crispasr + options: + - "backend:firered-asr" + parameters: + model: firered-asr2-aed-q4_k.gguf diff --git a/gallery/glm-asr-crispasr.yaml b/gallery/glm-asr-crispasr.yaml new file mode 100644 index 000000000000..97c246b883a4 --- /dev/null +++ b/gallery/glm-asr-crispasr.yaml @@ -0,0 +1,9 @@ +--- +name: "glm-asr-crispasr" + +config_file: | + backend: crispasr + options: + - "backend:glm-asr" + parameters: + model: glm-asr-nano-q4_k.gguf diff --git a/gallery/hubert-crispasr.yaml b/gallery/hubert-crispasr.yaml new file mode 100644 index 000000000000..df3f4321db7e --- /dev/null +++ b/gallery/hubert-crispasr.yaml @@ -0,0 +1,9 @@ +--- +name: "hubert-crispasr" + +config_file: | + backend: crispasr + options: + - "backend:hubert" + parameters: + model: hubert-large-ls960-ft-q4_k.gguf diff --git a/gallery/index.yaml b/gallery/index.yaml index 061e8b64384a..f5ecae8d74ff 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -32277,3 +32277,335 @@ files: - filename: vibevoice-realtime-0.5b-q4_k.gguf uri: huggingface://cstr/vibevoice-realtime-0.5b-GGUF/vibevoice-realtime-0.5b-q4_k.gguf +- name: hubert-crispasr + url: github:mudler/LocalAI/gallery/hubert-crispasr.yaml@master + urls: + - https://huggingface.co/cstr/hubert-large-ls960-ft-GGUF + description: | + HuBERT Large (LS960 fine-tune) CTC speech recognition, English. Runs via the CrispASR backend with an explicit backend selector. Default GGUF size ~200 MB. + tags: + - crispasr + - asr + - speech-recognition + - stt + - gguf + overrides: + backend: crispasr + known_usecases: + - transcript + name: hubert-crispasr + options: + - "backend:hubert" + parameters: + model: hubert-large-ls960-ft-q4_k.gguf + files: + - filename: hubert-large-ls960-ft-q4_k.gguf + uri: huggingface://cstr/hubert-large-ls960-ft-GGUF/hubert-large-ls960-ft-q4_k.gguf +- name: data2vec-crispasr + url: github:mudler/LocalAI/gallery/data2vec-crispasr.yaml@master + urls: + - https://huggingface.co/cstr/data2vec-audio-960h-GGUF + description: | + data2vec Audio Base (960h) CTC speech recognition, English. Runs via the CrispASR backend with an explicit backend selector. Default GGUF size ~60 MB. + tags: + - crispasr + - asr + - speech-recognition + - stt + - gguf + overrides: + backend: crispasr + known_usecases: + - transcript + name: data2vec-crispasr + options: + - "backend:data2vec" + parameters: + model: data2vec-audio-base-960h-q4_k.gguf + files: + - filename: data2vec-audio-base-960h-q4_k.gguf + uri: huggingface://cstr/data2vec-audio-960h-GGUF/data2vec-audio-base-960h-q4_k.gguf +- name: glm-asr-crispasr + url: github:mudler/LocalAI/gallery/glm-asr-crispasr.yaml@master + urls: + - https://huggingface.co/cstr/glm-asr-nano-GGUF + description: | + GLM-ASR Nano speech recognition. Runs via the CrispASR backend with an explicit backend selector. Default GGUF size ~1.2 GB. + tags: + - crispasr + - asr + - speech-recognition + - stt + - gguf + overrides: + backend: crispasr + known_usecases: + - transcript + name: glm-asr-crispasr + options: + - "backend:glm-asr" + parameters: + model: glm-asr-nano-q4_k.gguf + files: + - filename: glm-asr-nano-q4_k.gguf + uri: huggingface://cstr/glm-asr-nano-GGUF/glm-asr-nano-q4_k.gguf +- name: kyutai-stt-crispasr + url: github:mudler/LocalAI/gallery/kyutai-stt-crispasr.yaml@master + urls: + - https://huggingface.co/cstr/kyutai-stt-1b-GGUF + description: | + Kyutai STT 1B (Moshi-style) speech recognition. Runs via the CrispASR backend with an explicit backend selector. Default GGUF size ~636 MB. + tags: + - crispasr + - asr + - speech-recognition + - stt + - gguf + overrides: + backend: crispasr + known_usecases: + - transcript + name: kyutai-stt-crispasr + options: + - "backend:kyutai-stt" + parameters: + model: kyutai-stt-1b-q4_k.gguf + files: + - filename: kyutai-stt-1b-q4_k.gguf + uri: huggingface://cstr/kyutai-stt-1b-GGUF/kyutai-stt-1b-q4_k.gguf +- name: firered-asr-crispasr + url: github:mudler/LocalAI/gallery/firered-asr-crispasr.yaml@master + urls: + - https://huggingface.co/cstr/firered-asr2-aed-GGUF + description: | + FireRed-ASR2 AED speech recognition. Runs via the CrispASR backend with an explicit backend selector. Default GGUF size ~918 MB. + tags: + - crispasr + - asr + - speech-recognition + - stt + - gguf + overrides: + backend: crispasr + known_usecases: + - transcript + name: firered-asr-crispasr + options: + - "backend:firered-asr" + parameters: + model: firered-asr2-aed-q4_k.gguf + files: + - filename: firered-asr2-aed-q4_k.gguf + uri: huggingface://cstr/firered-asr2-aed-GGUF/firered-asr2-aed-q4_k.gguf +- name: moonshine-crispasr + url: github:mudler/LocalAI/gallery/moonshine-crispasr.yaml@master + urls: + - https://huggingface.co/cstr/moonshine-tiny-GGUF + description: | + Moonshine Tiny speech recognition, English. Runs via the CrispASR backend with an explicit backend selector and a companion tokenizer. Default GGUF size ~20 MB. + tags: + - crispasr + - asr + - speech-recognition + - stt + - gguf + overrides: + backend: crispasr + known_usecases: + - transcript + name: moonshine-crispasr + options: + - "backend:moonshine" + - "codec:tokenizer.bin" + parameters: + model: moonshine-tiny-q4_k.gguf + files: + - filename: moonshine-tiny-q4_k.gguf + uri: huggingface://cstr/moonshine-tiny-GGUF/moonshine-tiny-q4_k.gguf + - filename: tokenizer.bin + uri: huggingface://cstr/moonshine-tiny-GGUF/tokenizer.bin +- name: moonshine-de-crispasr + url: github:mudler/LocalAI/gallery/moonshine-de-crispasr.yaml@master + urls: + - https://huggingface.co/cstr/moonshine-base-de-fidoriel-GGUF + description: | + Moonshine Base German fine-tune (fidoriel), best-quality German Moonshine. Runs via the CrispASR backend with an explicit backend selector and a companion tokenizer. Default GGUF size ~39 MB. + license: CC-BY-NC-SA-4.0 + tags: + - crispasr + - asr + - speech-recognition + - stt + - gguf + overrides: + backend: crispasr + known_usecases: + - transcript + name: moonshine-de-crispasr + options: + - "backend:moonshine" + - "codec:tokenizer.bin" + parameters: + model: moonshine-base-de-fidoriel-q4_k.gguf + files: + - filename: moonshine-base-de-fidoriel-q4_k.gguf + uri: huggingface://cstr/moonshine-base-de-fidoriel-GGUF/moonshine-base-de-fidoriel-q4_k.gguf + - filename: tokenizer.bin + uri: huggingface://cstr/moonshine-base-de-fidoriel-GGUF/tokenizer.bin +- name: moonshine-tiny-de-crispasr + url: github:mudler/LocalAI/gallery/moonshine-tiny-de-crispasr.yaml@master + urls: + - https://huggingface.co/cstr/moonshine-tiny-de-fidoriel-GGUF + description: | + Moonshine Tiny German fine-tune (fidoriel), smaller/faster German Moonshine. Runs via the CrispASR backend with an explicit backend selector and a companion tokenizer. Default GGUF size ~17 MB. + license: CC-BY-NC-SA-4.0 + tags: + - crispasr + - asr + - speech-recognition + - stt + - gguf + overrides: + backend: crispasr + known_usecases: + - transcript + name: moonshine-tiny-de-crispasr + options: + - "backend:moonshine" + - "codec:tokenizer.bin" + parameters: + model: moonshine-tiny-de-fidoriel-q4_k.gguf + files: + - filename: moonshine-tiny-de-fidoriel-q4_k.gguf + uri: huggingface://cstr/moonshine-tiny-de-fidoriel-GGUF/moonshine-tiny-de-fidoriel-q4_k.gguf + - filename: tokenizer.bin + uri: huggingface://cstr/moonshine-tiny-de-fidoriel-GGUF/tokenizer.bin +- name: moonshine-streaming-crispasr + url: github:mudler/LocalAI/gallery/moonshine-streaming-crispasr.yaml@master + urls: + - https://huggingface.co/cstr/moonshine-streaming-tiny-GGUF + description: | + Moonshine Streaming Tiny speech recognition. Runs via the CrispASR backend with an explicit backend selector and a companion tokenizer. Default GGUF size ~31 MB. + tags: + - crispasr + - asr + - speech-recognition + - stt + - gguf + overrides: + backend: crispasr + known_usecases: + - transcript + name: moonshine-streaming-crispasr + options: + - "backend:moonshine-streaming" + - "codec:tokenizer.bin" + parameters: + model: moonshine-streaming-tiny-q4_k.gguf + files: + - filename: moonshine-streaming-tiny-q4_k.gguf + uri: huggingface://cstr/moonshine-streaming-tiny-GGUF/moonshine-streaming-tiny-q4_k.gguf + - filename: tokenizer.bin + uri: huggingface://cstr/moonshine-streaming-tiny-GGUF/tokenizer.bin +- name: mimo-asr-crispasr + url: github:mudler/LocalAI/gallery/mimo-asr-crispasr.yaml@master + urls: + - https://huggingface.co/cstr/mimo-asr-GGUF + description: | + MiMo-ASR speech recognition. Runs via the CrispASR backend with an explicit backend selector and a companion tokenizer GGUF. Default GGUF size ~4.2 GB. + tags: + - crispasr + - asr + - speech-recognition + - stt + - gguf + overrides: + backend: crispasr + known_usecases: + - transcript + name: mimo-asr-crispasr + options: + - "backend:mimo-asr" + - "codec:mimo-tokenizer-q4_k.gguf" + parameters: + model: mimo-asr-q4_k.gguf + files: + - filename: mimo-asr-q4_k.gguf + uri: huggingface://cstr/mimo-asr-GGUF/mimo-asr-q4_k.gguf + - filename: mimo-tokenizer-q4_k.gguf + uri: huggingface://cstr/mimo-tokenizer-GGUF/mimo-tokenizer-q4_k.gguf +- name: qwen3-tts-crispasr + url: github:mudler/LocalAI/gallery/qwen3-tts-crispasr.yaml@master + urls: + - https://huggingface.co/cstr/qwen3-tts-0.6b-base-GGUF + description: | + Qwen3-TTS 0.6B (12 Hz, Base) text-to-speech, synthesized through the CrispASR backend. Requires a companion 12 Hz tokenizer/codec GGUF. Default GGUF size ~986 MB. + tags: + - crispasr + - tts + - text-to-speech + - gguf + overrides: + backend: crispasr + known_usecases: + - tts + name: qwen3-tts-crispasr + options: + - "codec:qwen3-tts-tokenizer-12hz.gguf" + parameters: + model: qwen3-tts-12hz-0.6b-base-q8_0.gguf + files: + - filename: qwen3-tts-12hz-0.6b-base-q8_0.gguf + uri: huggingface://cstr/qwen3-tts-0.6b-base-GGUF/qwen3-tts-12hz-0.6b-base-q8_0.gguf + - filename: qwen3-tts-tokenizer-12hz.gguf + uri: huggingface://cstr/qwen3-tts-tokenizer-12hz-GGUF/qwen3-tts-tokenizer-12hz.gguf +- name: orpheus-crispasr + url: github:mudler/LocalAI/gallery/orpheus-crispasr.yaml@master + urls: + - https://huggingface.co/cstr/orpheus-3b-base-GGUF + description: | + Orpheus-3B text-to-speech (Llama-3.2-3B talker + SNAC 24 kHz codec), synthesized through the CrispASR backend. Requires the companion SNAC codec GGUF. Default GGUF size ~3.5 GB. + tags: + - crispasr + - tts + - text-to-speech + - gguf + overrides: + backend: crispasr + known_usecases: + - tts + name: orpheus-crispasr + options: + - "codec:snac-24khz.gguf" + parameters: + model: orpheus-3b-base-q8_0.gguf + files: + - filename: orpheus-3b-base-q8_0.gguf + uri: huggingface://cstr/orpheus-3b-base-GGUF/orpheus-3b-base-q8_0.gguf + - filename: snac-24khz.gguf + uri: huggingface://cstr/snac-24khz-GGUF/snac-24khz.gguf +- name: chatterbox-crispasr + url: github:mudler/LocalAI/gallery/chatterbox-crispasr.yaml@master + urls: + - https://huggingface.co/cstr/chatterbox-GGUF + description: | + Chatterbox (ResembleAI) text-to-speech: T3 token generator + S3Gen 24 kHz vocoder, synthesized through the CrispASR backend. Requires the companion S3Gen GGUF. Default GGUF size ~880 MB. + tags: + - crispasr + - tts + - text-to-speech + - gguf + overrides: + backend: crispasr + known_usecases: + - tts + name: chatterbox-crispasr + options: + - "codec:chatterbox-s3gen-q8_0.gguf" + parameters: + model: chatterbox-t3-q8_0.gguf + files: + - filename: chatterbox-t3-q8_0.gguf + uri: huggingface://cstr/chatterbox-GGUF/chatterbox-t3-q8_0.gguf + - filename: chatterbox-s3gen-q8_0.gguf + uri: huggingface://cstr/chatterbox-GGUF/chatterbox-s3gen-q8_0.gguf diff --git a/gallery/kyutai-stt-crispasr.yaml b/gallery/kyutai-stt-crispasr.yaml new file mode 100644 index 000000000000..da21b8a49d08 --- /dev/null +++ b/gallery/kyutai-stt-crispasr.yaml @@ -0,0 +1,9 @@ +--- +name: "kyutai-stt-crispasr" + +config_file: | + backend: crispasr + options: + - "backend:kyutai-stt" + parameters: + model: kyutai-stt-1b-q4_k.gguf diff --git a/gallery/mimo-asr-crispasr.yaml b/gallery/mimo-asr-crispasr.yaml new file mode 100644 index 000000000000..0617f4467bf4 --- /dev/null +++ b/gallery/mimo-asr-crispasr.yaml @@ -0,0 +1,10 @@ +--- +name: "mimo-asr-crispasr" + +config_file: | + backend: crispasr + options: + - "backend:mimo-asr" + - "codec:mimo-tokenizer-q4_k.gguf" + parameters: + model: mimo-asr-q4_k.gguf diff --git a/gallery/moonshine-crispasr.yaml b/gallery/moonshine-crispasr.yaml new file mode 100644 index 000000000000..571facd14a3e --- /dev/null +++ b/gallery/moonshine-crispasr.yaml @@ -0,0 +1,10 @@ +--- +name: "moonshine-crispasr" + +config_file: | + backend: crispasr + options: + - "backend:moonshine" + - "codec:tokenizer.bin" + parameters: + model: moonshine-tiny-q4_k.gguf diff --git a/gallery/moonshine-de-crispasr.yaml b/gallery/moonshine-de-crispasr.yaml new file mode 100644 index 000000000000..4c088676dfdf --- /dev/null +++ b/gallery/moonshine-de-crispasr.yaml @@ -0,0 +1,10 @@ +--- +name: "moonshine-de-crispasr" + +config_file: | + backend: crispasr + options: + - "backend:moonshine" + - "codec:tokenizer.bin" + parameters: + model: moonshine-base-de-fidoriel-q4_k.gguf diff --git a/gallery/moonshine-streaming-crispasr.yaml b/gallery/moonshine-streaming-crispasr.yaml new file mode 100644 index 000000000000..d40bf49d9807 --- /dev/null +++ b/gallery/moonshine-streaming-crispasr.yaml @@ -0,0 +1,10 @@ +--- +name: "moonshine-streaming-crispasr" + +config_file: | + backend: crispasr + options: + - "backend:moonshine-streaming" + - "codec:tokenizer.bin" + parameters: + model: moonshine-streaming-tiny-q4_k.gguf diff --git a/gallery/moonshine-tiny-de-crispasr.yaml b/gallery/moonshine-tiny-de-crispasr.yaml new file mode 100644 index 000000000000..24fa6344cdf0 --- /dev/null +++ b/gallery/moonshine-tiny-de-crispasr.yaml @@ -0,0 +1,10 @@ +--- +name: "moonshine-tiny-de-crispasr" + +config_file: | + backend: crispasr + options: + - "backend:moonshine" + - "codec:tokenizer.bin" + parameters: + model: moonshine-tiny-de-fidoriel-q4_k.gguf diff --git a/gallery/orpheus-crispasr.yaml b/gallery/orpheus-crispasr.yaml new file mode 100644 index 000000000000..e50f31c5aa63 --- /dev/null +++ b/gallery/orpheus-crispasr.yaml @@ -0,0 +1,9 @@ +--- +name: "orpheus-crispasr" + +config_file: | + backend: crispasr + options: + - "codec:snac-24khz.gguf" + parameters: + model: orpheus-3b-base-q8_0.gguf diff --git a/gallery/qwen3-tts-crispasr.yaml b/gallery/qwen3-tts-crispasr.yaml new file mode 100644 index 000000000000..34edb4abedad --- /dev/null +++ b/gallery/qwen3-tts-crispasr.yaml @@ -0,0 +1,9 @@ +--- +name: "qwen3-tts-crispasr" + +config_file: | + backend: crispasr + options: + - "codec:qwen3-tts-tokenizer-12hz.gguf" + parameters: + model: qwen3-tts-12hz-0.6b-base-q8_0.gguf From 6c43516d403dfe70e7b7beeb5dadd946993d3a41 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sun, 31 May 2026 08:06:08 +0000 Subject: [PATCH 26/29] refactor(gallery): use virtual.yaml base for crispasr models The crispasr entries are just backend + model + a couple options, fully expressed inline via overrides:/files: in gallery/index.yaml. Point each url: at the shared gallery/virtual.yaml (the established 'virtual' model trick) and drop the 36 redundant per-model gallery/*-crispasr.yaml files. Signed-off-by: Ettore Di Giacinto --- gallery/canary-crispasr.yaml | 7 -- gallery/chatterbox-crispasr.yaml | 9 --- gallery/cohere-crispasr.yaml | 7 -- gallery/data2vec-crispasr.yaml | 9 --- gallery/fastconformer-ctc-crispasr.yaml | 7 -- gallery/firered-asr-crispasr.yaml | 9 --- gallery/glm-asr-crispasr.yaml | 9 --- gallery/granite-4.1-crispasr.yaml | 7 -- gallery/granite-4.1-nar-crispasr.yaml | 7 -- gallery/granite-4.1-plus-crispasr.yaml | 7 -- gallery/granite-crispasr.yaml | 7 -- gallery/hubert-crispasr.yaml | 9 --- gallery/index.yaml | 72 ++++++++++----------- gallery/kyutai-stt-crispasr.yaml | 9 --- gallery/mimo-asr-crispasr.yaml | 10 --- gallery/moonshine-crispasr.yaml | 10 --- gallery/moonshine-de-crispasr.yaml | 10 --- gallery/moonshine-streaming-crispasr.yaml | 10 --- gallery/moonshine-tiny-de-crispasr.yaml | 10 --- gallery/orpheus-crispasr.yaml | 9 --- gallery/parakeet-crispasr.yaml | 7 -- gallery/parakeet-ja-crispasr.yaml | 7 -- gallery/parakeet-rnnt-0.6b-crispasr.yaml | 7 -- gallery/parakeet-rnnt-1.1b-crispasr.yaml | 7 -- gallery/parakeet-tdt-1.1b-crispasr.yaml | 7 -- gallery/parakeet-tdt_ctc-1.1b-crispasr.yaml | 7 -- gallery/parakeet-tdt_ctc-110m-crispasr.yaml | 7 -- gallery/parakeet-v2-crispasr.yaml | 7 -- gallery/qwen3-1.7b-crispasr.yaml | 7 -- gallery/qwen3-crispasr.yaml | 7 -- gallery/qwen3-tts-crispasr.yaml | 9 --- gallery/vibevoice-crispasr.yaml | 7 -- gallery/vibevoice-tts-crispasr.yaml | 7 -- gallery/voxtral-crispasr.yaml | 7 -- gallery/voxtral4b-crispasr.yaml | 7 -- gallery/wav2vec2-crispasr.yaml | 7 -- gallery/wav2vec2-de-crispasr.yaml | 7 -- 37 files changed, 36 insertions(+), 319 deletions(-) delete mode 100644 gallery/canary-crispasr.yaml delete mode 100644 gallery/chatterbox-crispasr.yaml delete mode 100644 gallery/cohere-crispasr.yaml delete mode 100644 gallery/data2vec-crispasr.yaml delete mode 100644 gallery/fastconformer-ctc-crispasr.yaml delete mode 100644 gallery/firered-asr-crispasr.yaml delete mode 100644 gallery/glm-asr-crispasr.yaml delete mode 100644 gallery/granite-4.1-crispasr.yaml delete mode 100644 gallery/granite-4.1-nar-crispasr.yaml delete mode 100644 gallery/granite-4.1-plus-crispasr.yaml delete mode 100644 gallery/granite-crispasr.yaml delete mode 100644 gallery/hubert-crispasr.yaml delete mode 100644 gallery/kyutai-stt-crispasr.yaml delete mode 100644 gallery/mimo-asr-crispasr.yaml delete mode 100644 gallery/moonshine-crispasr.yaml delete mode 100644 gallery/moonshine-de-crispasr.yaml delete mode 100644 gallery/moonshine-streaming-crispasr.yaml delete mode 100644 gallery/moonshine-tiny-de-crispasr.yaml delete mode 100644 gallery/orpheus-crispasr.yaml delete mode 100644 gallery/parakeet-crispasr.yaml delete mode 100644 gallery/parakeet-ja-crispasr.yaml delete mode 100644 gallery/parakeet-rnnt-0.6b-crispasr.yaml delete mode 100644 gallery/parakeet-rnnt-1.1b-crispasr.yaml delete mode 100644 gallery/parakeet-tdt-1.1b-crispasr.yaml delete mode 100644 gallery/parakeet-tdt_ctc-1.1b-crispasr.yaml delete mode 100644 gallery/parakeet-tdt_ctc-110m-crispasr.yaml delete mode 100644 gallery/parakeet-v2-crispasr.yaml delete mode 100644 gallery/qwen3-1.7b-crispasr.yaml delete mode 100644 gallery/qwen3-crispasr.yaml delete mode 100644 gallery/qwen3-tts-crispasr.yaml delete mode 100644 gallery/vibevoice-crispasr.yaml delete mode 100644 gallery/vibevoice-tts-crispasr.yaml delete mode 100644 gallery/voxtral-crispasr.yaml delete mode 100644 gallery/voxtral4b-crispasr.yaml delete mode 100644 gallery/wav2vec2-crispasr.yaml delete mode 100644 gallery/wav2vec2-de-crispasr.yaml diff --git a/gallery/canary-crispasr.yaml b/gallery/canary-crispasr.yaml deleted file mode 100644 index d7743d8c3eb0..000000000000 --- a/gallery/canary-crispasr.yaml +++ /dev/null @@ -1,7 +0,0 @@ ---- -name: "canary-crispasr" - -config_file: | - backend: crispasr - parameters: - model: canary-1b-v2-q4_k.gguf diff --git a/gallery/chatterbox-crispasr.yaml b/gallery/chatterbox-crispasr.yaml deleted file mode 100644 index 21356a7b9775..000000000000 --- a/gallery/chatterbox-crispasr.yaml +++ /dev/null @@ -1,9 +0,0 @@ ---- -name: "chatterbox-crispasr" - -config_file: | - backend: crispasr - options: - - "codec:chatterbox-s3gen-q8_0.gguf" - parameters: - model: chatterbox-t3-q8_0.gguf diff --git a/gallery/cohere-crispasr.yaml b/gallery/cohere-crispasr.yaml deleted file mode 100644 index 7f5a885409ec..000000000000 --- a/gallery/cohere-crispasr.yaml +++ /dev/null @@ -1,7 +0,0 @@ ---- -name: "cohere-crispasr" - -config_file: | - backend: crispasr - parameters: - model: cohere-transcribe-q4_k.gguf diff --git a/gallery/data2vec-crispasr.yaml b/gallery/data2vec-crispasr.yaml deleted file mode 100644 index 064cd172c406..000000000000 --- a/gallery/data2vec-crispasr.yaml +++ /dev/null @@ -1,9 +0,0 @@ ---- -name: "data2vec-crispasr" - -config_file: | - backend: crispasr - options: - - "backend:data2vec" - parameters: - model: data2vec-audio-base-960h-q4_k.gguf diff --git a/gallery/fastconformer-ctc-crispasr.yaml b/gallery/fastconformer-ctc-crispasr.yaml deleted file mode 100644 index a3ae51f83776..000000000000 --- a/gallery/fastconformer-ctc-crispasr.yaml +++ /dev/null @@ -1,7 +0,0 @@ ---- -name: "fastconformer-ctc-crispasr" - -config_file: | - backend: crispasr - parameters: - model: stt-en-fastconformer-ctc-large-q4_k.gguf diff --git a/gallery/firered-asr-crispasr.yaml b/gallery/firered-asr-crispasr.yaml deleted file mode 100644 index 79db1834fefe..000000000000 --- a/gallery/firered-asr-crispasr.yaml +++ /dev/null @@ -1,9 +0,0 @@ ---- -name: "firered-asr-crispasr" - -config_file: | - backend: crispasr - options: - - "backend:firered-asr" - parameters: - model: firered-asr2-aed-q4_k.gguf diff --git a/gallery/glm-asr-crispasr.yaml b/gallery/glm-asr-crispasr.yaml deleted file mode 100644 index 97c246b883a4..000000000000 --- a/gallery/glm-asr-crispasr.yaml +++ /dev/null @@ -1,9 +0,0 @@ ---- -name: "glm-asr-crispasr" - -config_file: | - backend: crispasr - options: - - "backend:glm-asr" - parameters: - model: glm-asr-nano-q4_k.gguf diff --git a/gallery/granite-4.1-crispasr.yaml b/gallery/granite-4.1-crispasr.yaml deleted file mode 100644 index fd9222f3bb9f..000000000000 --- a/gallery/granite-4.1-crispasr.yaml +++ /dev/null @@ -1,7 +0,0 @@ ---- -name: "granite-4.1-crispasr" - -config_file: | - backend: crispasr - parameters: - model: granite-speech-4.1-2b-q4_k.gguf diff --git a/gallery/granite-4.1-nar-crispasr.yaml b/gallery/granite-4.1-nar-crispasr.yaml deleted file mode 100644 index cf2ec73cff22..000000000000 --- a/gallery/granite-4.1-nar-crispasr.yaml +++ /dev/null @@ -1,7 +0,0 @@ ---- -name: "granite-4.1-nar-crispasr" - -config_file: | - backend: crispasr - parameters: - model: granite-speech-4.1-2b-nar-q4_k.gguf diff --git a/gallery/granite-4.1-plus-crispasr.yaml b/gallery/granite-4.1-plus-crispasr.yaml deleted file mode 100644 index 3be970fc87ca..000000000000 --- a/gallery/granite-4.1-plus-crispasr.yaml +++ /dev/null @@ -1,7 +0,0 @@ ---- -name: "granite-4.1-plus-crispasr" - -config_file: | - backend: crispasr - parameters: - model: granite-speech-4.1-2b-plus-q4_k.gguf diff --git a/gallery/granite-crispasr.yaml b/gallery/granite-crispasr.yaml deleted file mode 100644 index 7142ca86829f..000000000000 --- a/gallery/granite-crispasr.yaml +++ /dev/null @@ -1,7 +0,0 @@ ---- -name: "granite-crispasr" - -config_file: | - backend: crispasr - parameters: - model: granite-speech-4.0-1b-q4_k.gguf diff --git a/gallery/hubert-crispasr.yaml b/gallery/hubert-crispasr.yaml deleted file mode 100644 index df3f4321db7e..000000000000 --- a/gallery/hubert-crispasr.yaml +++ /dev/null @@ -1,9 +0,0 @@ ---- -name: "hubert-crispasr" - -config_file: | - backend: crispasr - options: - - "backend:hubert" - parameters: - model: hubert-large-ls960-ft-q4_k.gguf diff --git a/gallery/index.yaml b/gallery/index.yaml index f5ecae8d74ff..94526dbe03d3 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -31773,7 +31773,7 @@ sha256: cd53f64eefac2623a12f2f118ef50b56622dc3012f42c815c6adf0d08292f387 - name: parakeet-crispasr - url: github:mudler/LocalAI/gallery/parakeet-crispasr.yaml@master + url: github:mudler/LocalAI/gallery/virtual.yaml@master urls: - https://huggingface.co/cstr/parakeet-tdt-0.6b-v3-GGUF description: | @@ -31795,7 +31795,7 @@ - filename: parakeet-tdt-0.6b-v3-q4_k.gguf uri: huggingface://cstr/parakeet-tdt-0.6b-v3-GGUF/parakeet-tdt-0.6b-v3-q4_k.gguf - name: parakeet-v2-crispasr - url: github:mudler/LocalAI/gallery/parakeet-v2-crispasr.yaml@master + url: github:mudler/LocalAI/gallery/virtual.yaml@master urls: - https://huggingface.co/cstr/parakeet-tdt-0.6b-v2-GGUF description: | @@ -31817,7 +31817,7 @@ - filename: parakeet-tdt-0.6b-v2-q4_k.gguf uri: huggingface://cstr/parakeet-tdt-0.6b-v2-GGUF/parakeet-tdt-0.6b-v2-q4_k.gguf - name: parakeet-ja-crispasr - url: github:mudler/LocalAI/gallery/parakeet-ja-crispasr.yaml@master + url: github:mudler/LocalAI/gallery/virtual.yaml@master urls: - https://huggingface.co/cstr/parakeet-tdt-0.6b-ja-GGUF description: | @@ -31839,7 +31839,7 @@ - filename: parakeet-tdt-0.6b-ja.gguf uri: huggingface://cstr/parakeet-tdt-0.6b-ja-GGUF/parakeet-tdt-0.6b-ja.gguf - name: parakeet-tdt-1.1b-crispasr - url: github:mudler/LocalAI/gallery/parakeet-tdt-1.1b-crispasr.yaml@master + url: github:mudler/LocalAI/gallery/virtual.yaml@master urls: - https://huggingface.co/cstr/parakeet-tdt-1.1b-GGUF description: | @@ -31861,7 +31861,7 @@ - filename: parakeet-tdt-1.1b-q4_k.gguf uri: huggingface://cstr/parakeet-tdt-1.1b-GGUF/parakeet-tdt-1.1b-q4_k.gguf - name: parakeet-tdt_ctc-110m-crispasr - url: github:mudler/LocalAI/gallery/parakeet-tdt_ctc-110m-crispasr.yaml@master + url: github:mudler/LocalAI/gallery/virtual.yaml@master urls: - https://huggingface.co/cstr/parakeet-tdt_ctc-110m-GGUF description: | @@ -31883,7 +31883,7 @@ - filename: parakeet-tdt_ctc-110m-q4_k.gguf uri: huggingface://cstr/parakeet-tdt_ctc-110m-GGUF/parakeet-tdt_ctc-110m-q4_k.gguf - name: parakeet-tdt_ctc-1.1b-crispasr - url: github:mudler/LocalAI/gallery/parakeet-tdt_ctc-1.1b-crispasr.yaml@master + url: github:mudler/LocalAI/gallery/virtual.yaml@master urls: - https://huggingface.co/cstr/parakeet-tdt_ctc-1.1b-GGUF description: | @@ -31905,7 +31905,7 @@ - filename: parakeet-tdt_ctc-1.1b-q4_k.gguf uri: huggingface://cstr/parakeet-tdt_ctc-1.1b-GGUF/parakeet-tdt_ctc-1.1b-q4_k.gguf - name: parakeet-rnnt-0.6b-crispasr - url: github:mudler/LocalAI/gallery/parakeet-rnnt-0.6b-crispasr.yaml@master + url: github:mudler/LocalAI/gallery/virtual.yaml@master urls: - https://huggingface.co/cstr/parakeet-rnnt-0.6b-GGUF description: | @@ -31927,7 +31927,7 @@ - filename: parakeet-rnnt-0.6b-q4_k.gguf uri: huggingface://cstr/parakeet-rnnt-0.6b-GGUF/parakeet-rnnt-0.6b-q4_k.gguf - name: parakeet-rnnt-1.1b-crispasr - url: github:mudler/LocalAI/gallery/parakeet-rnnt-1.1b-crispasr.yaml@master + url: github:mudler/LocalAI/gallery/virtual.yaml@master urls: - https://huggingface.co/cstr/parakeet-rnnt-1.1b-GGUF description: | @@ -31949,7 +31949,7 @@ - filename: parakeet-rnnt-1.1b-q4_k.gguf uri: huggingface://cstr/parakeet-rnnt-1.1b-GGUF/parakeet-rnnt-1.1b-q4_k.gguf - name: fastconformer-ctc-crispasr - url: github:mudler/LocalAI/gallery/fastconformer-ctc-crispasr.yaml@master + url: github:mudler/LocalAI/gallery/virtual.yaml@master urls: - https://huggingface.co/cstr/stt-en-fastconformer-ctc-large-GGUF description: | @@ -31971,7 +31971,7 @@ - filename: stt-en-fastconformer-ctc-large-q4_k.gguf uri: huggingface://cstr/stt-en-fastconformer-ctc-large-GGUF/stt-en-fastconformer-ctc-large-q4_k.gguf - name: canary-crispasr - url: github:mudler/LocalAI/gallery/canary-crispasr.yaml@master + url: github:mudler/LocalAI/gallery/virtual.yaml@master urls: - https://huggingface.co/cstr/canary-1b-v2-GGUF description: | @@ -31993,7 +31993,7 @@ - filename: canary-1b-v2-q4_k.gguf uri: huggingface://cstr/canary-1b-v2-GGUF/canary-1b-v2-q4_k.gguf - name: voxtral-crispasr - url: github:mudler/LocalAI/gallery/voxtral-crispasr.yaml@master + url: github:mudler/LocalAI/gallery/virtual.yaml@master urls: - https://huggingface.co/cstr/voxtral-mini-3b-2507-GGUF description: | @@ -32015,7 +32015,7 @@ - filename: voxtral-mini-3b-2507-q4_k.gguf uri: huggingface://cstr/voxtral-mini-3b-2507-GGUF/voxtral-mini-3b-2507-q4_k.gguf - name: voxtral4b-crispasr - url: github:mudler/LocalAI/gallery/voxtral4b-crispasr.yaml@master + url: github:mudler/LocalAI/gallery/virtual.yaml@master urls: - https://huggingface.co/cstr/voxtral-mini-4b-realtime-GGUF description: | @@ -32037,7 +32037,7 @@ - filename: voxtral-mini-4b-realtime-q4_k.gguf uri: huggingface://cstr/voxtral-mini-4b-realtime-GGUF/voxtral-mini-4b-realtime-q4_k.gguf - name: granite-crispasr - url: github:mudler/LocalAI/gallery/granite-crispasr.yaml@master + url: github:mudler/LocalAI/gallery/virtual.yaml@master urls: - https://huggingface.co/cstr/granite-speech-4.0-1b-GGUF description: | @@ -32059,7 +32059,7 @@ - filename: granite-speech-4.0-1b-q4_k.gguf uri: huggingface://cstr/granite-speech-4.0-1b-GGUF/granite-speech-4.0-1b-q4_k.gguf - name: granite-4.1-crispasr - url: github:mudler/LocalAI/gallery/granite-4.1-crispasr.yaml@master + url: github:mudler/LocalAI/gallery/virtual.yaml@master urls: - https://huggingface.co/cstr/granite-speech-4.1-2b-GGUF description: | @@ -32081,7 +32081,7 @@ - filename: granite-speech-4.1-2b-q4_k.gguf uri: huggingface://cstr/granite-speech-4.1-2b-GGUF/granite-speech-4.1-2b-q4_k.gguf - name: granite-4.1-plus-crispasr - url: github:mudler/LocalAI/gallery/granite-4.1-plus-crispasr.yaml@master + url: github:mudler/LocalAI/gallery/virtual.yaml@master urls: - https://huggingface.co/cstr/granite-speech-4.1-2b-plus-GGUF description: | @@ -32103,7 +32103,7 @@ - filename: granite-speech-4.1-2b-plus-q4_k.gguf uri: huggingface://cstr/granite-speech-4.1-2b-plus-GGUF/granite-speech-4.1-2b-plus-q4_k.gguf - name: granite-4.1-nar-crispasr - url: github:mudler/LocalAI/gallery/granite-4.1-nar-crispasr.yaml@master + url: github:mudler/LocalAI/gallery/virtual.yaml@master urls: - https://huggingface.co/cstr/granite-speech-4.1-2b-nar-GGUF description: | @@ -32125,7 +32125,7 @@ - filename: granite-speech-4.1-2b-nar-q4_k.gguf uri: huggingface://cstr/granite-speech-4.1-2b-nar-GGUF/granite-speech-4.1-2b-nar-q4_k.gguf - name: qwen3-crispasr - url: github:mudler/LocalAI/gallery/qwen3-crispasr.yaml@master + url: github:mudler/LocalAI/gallery/virtual.yaml@master urls: - https://huggingface.co/cstr/qwen3-asr-0.6b-GGUF description: | @@ -32147,7 +32147,7 @@ - filename: qwen3-asr-0.6b-q4_k.gguf uri: huggingface://cstr/qwen3-asr-0.6b-GGUF/qwen3-asr-0.6b-q4_k.gguf - name: qwen3-1.7b-crispasr - url: github:mudler/LocalAI/gallery/qwen3-1.7b-crispasr.yaml@master + url: github:mudler/LocalAI/gallery/virtual.yaml@master urls: - https://huggingface.co/cstr/qwen3-asr-1.7b-GGUF description: | @@ -32169,7 +32169,7 @@ - filename: qwen3-asr-1.7b-q4_k.gguf uri: huggingface://cstr/qwen3-asr-1.7b-GGUF/qwen3-asr-1.7b-q4_k.gguf - name: cohere-crispasr - url: github:mudler/LocalAI/gallery/cohere-crispasr.yaml@master + url: github:mudler/LocalAI/gallery/virtual.yaml@master urls: - https://huggingface.co/cstr/cohere-transcribe-03-2026-GGUF description: | @@ -32191,7 +32191,7 @@ - filename: cohere-transcribe-q4_k.gguf uri: huggingface://cstr/cohere-transcribe-03-2026-GGUF/cohere-transcribe-q4_k.gguf - name: wav2vec2-crispasr - url: github:mudler/LocalAI/gallery/wav2vec2-crispasr.yaml@master + url: github:mudler/LocalAI/gallery/virtual.yaml@master urls: - https://huggingface.co/cstr/wav2vec2-large-xlsr-53-english-GGUF description: | @@ -32213,7 +32213,7 @@ - filename: wav2vec2-xlsr-en-q4_k.gguf uri: huggingface://cstr/wav2vec2-large-xlsr-53-english-GGUF/wav2vec2-xlsr-en-q4_k.gguf - name: wav2vec2-de-crispasr - url: github:mudler/LocalAI/gallery/wav2vec2-de-crispasr.yaml@master + url: github:mudler/LocalAI/gallery/virtual.yaml@master urls: - https://huggingface.co/cstr/wav2vec2-large-xlsr-53-german-GGUF description: | @@ -32235,7 +32235,7 @@ - filename: wav2vec2-large-xlsr-53-german-q4_k.gguf uri: huggingface://cstr/wav2vec2-large-xlsr-53-german-GGUF/wav2vec2-large-xlsr-53-german-q4_k.gguf - name: vibevoice-crispasr - url: github:mudler/LocalAI/gallery/vibevoice-crispasr.yaml@master + url: github:mudler/LocalAI/gallery/virtual.yaml@master urls: - https://huggingface.co/cstr/vibevoice-asr-GGUF description: | @@ -32257,7 +32257,7 @@ - filename: vibevoice-asr-q4_k.gguf uri: huggingface://cstr/vibevoice-asr-GGUF/vibevoice-asr-q4_k.gguf - name: vibevoice-tts-crispasr - url: github:mudler/LocalAI/gallery/vibevoice-tts-crispasr.yaml@master + url: github:mudler/LocalAI/gallery/virtual.yaml@master urls: - https://huggingface.co/cstr/vibevoice-realtime-0.5b-GGUF description: | @@ -32278,7 +32278,7 @@ - filename: vibevoice-realtime-0.5b-q4_k.gguf uri: huggingface://cstr/vibevoice-realtime-0.5b-GGUF/vibevoice-realtime-0.5b-q4_k.gguf - name: hubert-crispasr - url: github:mudler/LocalAI/gallery/hubert-crispasr.yaml@master + url: github:mudler/LocalAI/gallery/virtual.yaml@master urls: - https://huggingface.co/cstr/hubert-large-ls960-ft-GGUF description: | @@ -32302,7 +32302,7 @@ - filename: hubert-large-ls960-ft-q4_k.gguf uri: huggingface://cstr/hubert-large-ls960-ft-GGUF/hubert-large-ls960-ft-q4_k.gguf - name: data2vec-crispasr - url: github:mudler/LocalAI/gallery/data2vec-crispasr.yaml@master + url: github:mudler/LocalAI/gallery/virtual.yaml@master urls: - https://huggingface.co/cstr/data2vec-audio-960h-GGUF description: | @@ -32326,7 +32326,7 @@ - filename: data2vec-audio-base-960h-q4_k.gguf uri: huggingface://cstr/data2vec-audio-960h-GGUF/data2vec-audio-base-960h-q4_k.gguf - name: glm-asr-crispasr - url: github:mudler/LocalAI/gallery/glm-asr-crispasr.yaml@master + url: github:mudler/LocalAI/gallery/virtual.yaml@master urls: - https://huggingface.co/cstr/glm-asr-nano-GGUF description: | @@ -32350,7 +32350,7 @@ - filename: glm-asr-nano-q4_k.gguf uri: huggingface://cstr/glm-asr-nano-GGUF/glm-asr-nano-q4_k.gguf - name: kyutai-stt-crispasr - url: github:mudler/LocalAI/gallery/kyutai-stt-crispasr.yaml@master + url: github:mudler/LocalAI/gallery/virtual.yaml@master urls: - https://huggingface.co/cstr/kyutai-stt-1b-GGUF description: | @@ -32374,7 +32374,7 @@ - filename: kyutai-stt-1b-q4_k.gguf uri: huggingface://cstr/kyutai-stt-1b-GGUF/kyutai-stt-1b-q4_k.gguf - name: firered-asr-crispasr - url: github:mudler/LocalAI/gallery/firered-asr-crispasr.yaml@master + url: github:mudler/LocalAI/gallery/virtual.yaml@master urls: - https://huggingface.co/cstr/firered-asr2-aed-GGUF description: | @@ -32398,7 +32398,7 @@ - filename: firered-asr2-aed-q4_k.gguf uri: huggingface://cstr/firered-asr2-aed-GGUF/firered-asr2-aed-q4_k.gguf - name: moonshine-crispasr - url: github:mudler/LocalAI/gallery/moonshine-crispasr.yaml@master + url: github:mudler/LocalAI/gallery/virtual.yaml@master urls: - https://huggingface.co/cstr/moonshine-tiny-GGUF description: | @@ -32425,7 +32425,7 @@ - filename: tokenizer.bin uri: huggingface://cstr/moonshine-tiny-GGUF/tokenizer.bin - name: moonshine-de-crispasr - url: github:mudler/LocalAI/gallery/moonshine-de-crispasr.yaml@master + url: github:mudler/LocalAI/gallery/virtual.yaml@master urls: - https://huggingface.co/cstr/moonshine-base-de-fidoriel-GGUF description: | @@ -32453,7 +32453,7 @@ - filename: tokenizer.bin uri: huggingface://cstr/moonshine-base-de-fidoriel-GGUF/tokenizer.bin - name: moonshine-tiny-de-crispasr - url: github:mudler/LocalAI/gallery/moonshine-tiny-de-crispasr.yaml@master + url: github:mudler/LocalAI/gallery/virtual.yaml@master urls: - https://huggingface.co/cstr/moonshine-tiny-de-fidoriel-GGUF description: | @@ -32481,7 +32481,7 @@ - filename: tokenizer.bin uri: huggingface://cstr/moonshine-tiny-de-fidoriel-GGUF/tokenizer.bin - name: moonshine-streaming-crispasr - url: github:mudler/LocalAI/gallery/moonshine-streaming-crispasr.yaml@master + url: github:mudler/LocalAI/gallery/virtual.yaml@master urls: - https://huggingface.co/cstr/moonshine-streaming-tiny-GGUF description: | @@ -32508,7 +32508,7 @@ - filename: tokenizer.bin uri: huggingface://cstr/moonshine-streaming-tiny-GGUF/tokenizer.bin - name: mimo-asr-crispasr - url: github:mudler/LocalAI/gallery/mimo-asr-crispasr.yaml@master + url: github:mudler/LocalAI/gallery/virtual.yaml@master urls: - https://huggingface.co/cstr/mimo-asr-GGUF description: | @@ -32535,7 +32535,7 @@ - filename: mimo-tokenizer-q4_k.gguf uri: huggingface://cstr/mimo-tokenizer-GGUF/mimo-tokenizer-q4_k.gguf - name: qwen3-tts-crispasr - url: github:mudler/LocalAI/gallery/qwen3-tts-crispasr.yaml@master + url: github:mudler/LocalAI/gallery/virtual.yaml@master urls: - https://huggingface.co/cstr/qwen3-tts-0.6b-base-GGUF description: | @@ -32560,7 +32560,7 @@ - filename: qwen3-tts-tokenizer-12hz.gguf uri: huggingface://cstr/qwen3-tts-tokenizer-12hz-GGUF/qwen3-tts-tokenizer-12hz.gguf - name: orpheus-crispasr - url: github:mudler/LocalAI/gallery/orpheus-crispasr.yaml@master + url: github:mudler/LocalAI/gallery/virtual.yaml@master urls: - https://huggingface.co/cstr/orpheus-3b-base-GGUF description: | @@ -32585,7 +32585,7 @@ - filename: snac-24khz.gguf uri: huggingface://cstr/snac-24khz-GGUF/snac-24khz.gguf - name: chatterbox-crispasr - url: github:mudler/LocalAI/gallery/chatterbox-crispasr.yaml@master + url: github:mudler/LocalAI/gallery/virtual.yaml@master urls: - https://huggingface.co/cstr/chatterbox-GGUF description: | diff --git a/gallery/kyutai-stt-crispasr.yaml b/gallery/kyutai-stt-crispasr.yaml deleted file mode 100644 index da21b8a49d08..000000000000 --- a/gallery/kyutai-stt-crispasr.yaml +++ /dev/null @@ -1,9 +0,0 @@ ---- -name: "kyutai-stt-crispasr" - -config_file: | - backend: crispasr - options: - - "backend:kyutai-stt" - parameters: - model: kyutai-stt-1b-q4_k.gguf diff --git a/gallery/mimo-asr-crispasr.yaml b/gallery/mimo-asr-crispasr.yaml deleted file mode 100644 index 0617f4467bf4..000000000000 --- a/gallery/mimo-asr-crispasr.yaml +++ /dev/null @@ -1,10 +0,0 @@ ---- -name: "mimo-asr-crispasr" - -config_file: | - backend: crispasr - options: - - "backend:mimo-asr" - - "codec:mimo-tokenizer-q4_k.gguf" - parameters: - model: mimo-asr-q4_k.gguf diff --git a/gallery/moonshine-crispasr.yaml b/gallery/moonshine-crispasr.yaml deleted file mode 100644 index 571facd14a3e..000000000000 --- a/gallery/moonshine-crispasr.yaml +++ /dev/null @@ -1,10 +0,0 @@ ---- -name: "moonshine-crispasr" - -config_file: | - backend: crispasr - options: - - "backend:moonshine" - - "codec:tokenizer.bin" - parameters: - model: moonshine-tiny-q4_k.gguf diff --git a/gallery/moonshine-de-crispasr.yaml b/gallery/moonshine-de-crispasr.yaml deleted file mode 100644 index 4c088676dfdf..000000000000 --- a/gallery/moonshine-de-crispasr.yaml +++ /dev/null @@ -1,10 +0,0 @@ ---- -name: "moonshine-de-crispasr" - -config_file: | - backend: crispasr - options: - - "backend:moonshine" - - "codec:tokenizer.bin" - parameters: - model: moonshine-base-de-fidoriel-q4_k.gguf diff --git a/gallery/moonshine-streaming-crispasr.yaml b/gallery/moonshine-streaming-crispasr.yaml deleted file mode 100644 index d40bf49d9807..000000000000 --- a/gallery/moonshine-streaming-crispasr.yaml +++ /dev/null @@ -1,10 +0,0 @@ ---- -name: "moonshine-streaming-crispasr" - -config_file: | - backend: crispasr - options: - - "backend:moonshine-streaming" - - "codec:tokenizer.bin" - parameters: - model: moonshine-streaming-tiny-q4_k.gguf diff --git a/gallery/moonshine-tiny-de-crispasr.yaml b/gallery/moonshine-tiny-de-crispasr.yaml deleted file mode 100644 index 24fa6344cdf0..000000000000 --- a/gallery/moonshine-tiny-de-crispasr.yaml +++ /dev/null @@ -1,10 +0,0 @@ ---- -name: "moonshine-tiny-de-crispasr" - -config_file: | - backend: crispasr - options: - - "backend:moonshine" - - "codec:tokenizer.bin" - parameters: - model: moonshine-tiny-de-fidoriel-q4_k.gguf diff --git a/gallery/orpheus-crispasr.yaml b/gallery/orpheus-crispasr.yaml deleted file mode 100644 index e50f31c5aa63..000000000000 --- a/gallery/orpheus-crispasr.yaml +++ /dev/null @@ -1,9 +0,0 @@ ---- -name: "orpheus-crispasr" - -config_file: | - backend: crispasr - options: - - "codec:snac-24khz.gguf" - parameters: - model: orpheus-3b-base-q8_0.gguf diff --git a/gallery/parakeet-crispasr.yaml b/gallery/parakeet-crispasr.yaml deleted file mode 100644 index 6490290d4167..000000000000 --- a/gallery/parakeet-crispasr.yaml +++ /dev/null @@ -1,7 +0,0 @@ ---- -name: "parakeet-crispasr" - -config_file: | - backend: crispasr - parameters: - model: parakeet-tdt-0.6b-v3-q4_k.gguf diff --git a/gallery/parakeet-ja-crispasr.yaml b/gallery/parakeet-ja-crispasr.yaml deleted file mode 100644 index a441f9bb0000..000000000000 --- a/gallery/parakeet-ja-crispasr.yaml +++ /dev/null @@ -1,7 +0,0 @@ ---- -name: "parakeet-ja-crispasr" - -config_file: | - backend: crispasr - parameters: - model: parakeet-tdt-0.6b-ja.gguf diff --git a/gallery/parakeet-rnnt-0.6b-crispasr.yaml b/gallery/parakeet-rnnt-0.6b-crispasr.yaml deleted file mode 100644 index 67203508de7a..000000000000 --- a/gallery/parakeet-rnnt-0.6b-crispasr.yaml +++ /dev/null @@ -1,7 +0,0 @@ ---- -name: "parakeet-rnnt-0.6b-crispasr" - -config_file: | - backend: crispasr - parameters: - model: parakeet-rnnt-0.6b-q4_k.gguf diff --git a/gallery/parakeet-rnnt-1.1b-crispasr.yaml b/gallery/parakeet-rnnt-1.1b-crispasr.yaml deleted file mode 100644 index 800d683c9dee..000000000000 --- a/gallery/parakeet-rnnt-1.1b-crispasr.yaml +++ /dev/null @@ -1,7 +0,0 @@ ---- -name: "parakeet-rnnt-1.1b-crispasr" - -config_file: | - backend: crispasr - parameters: - model: parakeet-rnnt-1.1b-q4_k.gguf diff --git a/gallery/parakeet-tdt-1.1b-crispasr.yaml b/gallery/parakeet-tdt-1.1b-crispasr.yaml deleted file mode 100644 index 92cad7967fb3..000000000000 --- a/gallery/parakeet-tdt-1.1b-crispasr.yaml +++ /dev/null @@ -1,7 +0,0 @@ ---- -name: "parakeet-tdt-1.1b-crispasr" - -config_file: | - backend: crispasr - parameters: - model: parakeet-tdt-1.1b-q4_k.gguf diff --git a/gallery/parakeet-tdt_ctc-1.1b-crispasr.yaml b/gallery/parakeet-tdt_ctc-1.1b-crispasr.yaml deleted file mode 100644 index b71c5e95eb7a..000000000000 --- a/gallery/parakeet-tdt_ctc-1.1b-crispasr.yaml +++ /dev/null @@ -1,7 +0,0 @@ ---- -name: "parakeet-tdt_ctc-1.1b-crispasr" - -config_file: | - backend: crispasr - parameters: - model: parakeet-tdt_ctc-1.1b-q4_k.gguf diff --git a/gallery/parakeet-tdt_ctc-110m-crispasr.yaml b/gallery/parakeet-tdt_ctc-110m-crispasr.yaml deleted file mode 100644 index 4bf39a1c3225..000000000000 --- a/gallery/parakeet-tdt_ctc-110m-crispasr.yaml +++ /dev/null @@ -1,7 +0,0 @@ ---- -name: "parakeet-tdt_ctc-110m-crispasr" - -config_file: | - backend: crispasr - parameters: - model: parakeet-tdt_ctc-110m-q4_k.gguf diff --git a/gallery/parakeet-v2-crispasr.yaml b/gallery/parakeet-v2-crispasr.yaml deleted file mode 100644 index 4094b420ca3a..000000000000 --- a/gallery/parakeet-v2-crispasr.yaml +++ /dev/null @@ -1,7 +0,0 @@ ---- -name: "parakeet-v2-crispasr" - -config_file: | - backend: crispasr - parameters: - model: parakeet-tdt-0.6b-v2-q4_k.gguf diff --git a/gallery/qwen3-1.7b-crispasr.yaml b/gallery/qwen3-1.7b-crispasr.yaml deleted file mode 100644 index 0458badf242a..000000000000 --- a/gallery/qwen3-1.7b-crispasr.yaml +++ /dev/null @@ -1,7 +0,0 @@ ---- -name: "qwen3-1.7b-crispasr" - -config_file: | - backend: crispasr - parameters: - model: qwen3-asr-1.7b-q4_k.gguf diff --git a/gallery/qwen3-crispasr.yaml b/gallery/qwen3-crispasr.yaml deleted file mode 100644 index 5bb037538bbf..000000000000 --- a/gallery/qwen3-crispasr.yaml +++ /dev/null @@ -1,7 +0,0 @@ ---- -name: "qwen3-crispasr" - -config_file: | - backend: crispasr - parameters: - model: qwen3-asr-0.6b-q4_k.gguf diff --git a/gallery/qwen3-tts-crispasr.yaml b/gallery/qwen3-tts-crispasr.yaml deleted file mode 100644 index 34edb4abedad..000000000000 --- a/gallery/qwen3-tts-crispasr.yaml +++ /dev/null @@ -1,9 +0,0 @@ ---- -name: "qwen3-tts-crispasr" - -config_file: | - backend: crispasr - options: - - "codec:qwen3-tts-tokenizer-12hz.gguf" - parameters: - model: qwen3-tts-12hz-0.6b-base-q8_0.gguf diff --git a/gallery/vibevoice-crispasr.yaml b/gallery/vibevoice-crispasr.yaml deleted file mode 100644 index ed3817ed973c..000000000000 --- a/gallery/vibevoice-crispasr.yaml +++ /dev/null @@ -1,7 +0,0 @@ ---- -name: "vibevoice-crispasr" - -config_file: | - backend: crispasr - parameters: - model: vibevoice-asr-q4_k.gguf diff --git a/gallery/vibevoice-tts-crispasr.yaml b/gallery/vibevoice-tts-crispasr.yaml deleted file mode 100644 index dae60fe4f419..000000000000 --- a/gallery/vibevoice-tts-crispasr.yaml +++ /dev/null @@ -1,7 +0,0 @@ ---- -name: "vibevoice-tts-crispasr" - -config_file: | - backend: crispasr - parameters: - model: vibevoice-realtime-0.5b-q4_k.gguf diff --git a/gallery/voxtral-crispasr.yaml b/gallery/voxtral-crispasr.yaml deleted file mode 100644 index 319d7f50b1dc..000000000000 --- a/gallery/voxtral-crispasr.yaml +++ /dev/null @@ -1,7 +0,0 @@ ---- -name: "voxtral-crispasr" - -config_file: | - backend: crispasr - parameters: - model: voxtral-mini-3b-2507-q4_k.gguf diff --git a/gallery/voxtral4b-crispasr.yaml b/gallery/voxtral4b-crispasr.yaml deleted file mode 100644 index 64f601e7c2b8..000000000000 --- a/gallery/voxtral4b-crispasr.yaml +++ /dev/null @@ -1,7 +0,0 @@ ---- -name: "voxtral4b-crispasr" - -config_file: | - backend: crispasr - parameters: - model: voxtral-mini-4b-realtime-q4_k.gguf diff --git a/gallery/wav2vec2-crispasr.yaml b/gallery/wav2vec2-crispasr.yaml deleted file mode 100644 index 66afe60bd720..000000000000 --- a/gallery/wav2vec2-crispasr.yaml +++ /dev/null @@ -1,7 +0,0 @@ ---- -name: "wav2vec2-crispasr" - -config_file: | - backend: crispasr - parameters: - model: wav2vec2-xlsr-en-q4_k.gguf diff --git a/gallery/wav2vec2-de-crispasr.yaml b/gallery/wav2vec2-de-crispasr.yaml deleted file mode 100644 index e4ac5999160b..000000000000 --- a/gallery/wav2vec2-de-crispasr.yaml +++ /dev/null @@ -1,7 +0,0 @@ ---- -name: "wav2vec2-de-crispasr" - -config_file: | - backend: crispasr - parameters: - model: wav2vec2-large-xlsr-53-german-q4_k.gguf From fdd0a85e13a069d9d8d68ba5637545f5b412c09f Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sun, 31 May 2026 08:10:53 +0000 Subject: [PATCH 27/29] fix(gallery): drop voice-requiring TTS entries (keep vibevoice-tts) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Real e2e showed qwen3-tts/orpheus/chatterbox don't synthesize through the current shim: the codec: companion loads fine, but these engines additionally need a voice pack / voice prompt / reference clip (qwen3-tts base errors 'no voice'; chatterbox is zero-shot cloning; orpheus uses named voices) that the backend doesn't wire. (qwen3-tts also can't auto-detect: its GGUF arch is 'qwen3tts', unmapped by the detector — would need backend:qwen3-tts.) Removed to avoid shipping non-working gallery entries; vibevoice-tts (built-in voice, e2e-verified) remains the working TTS. Voice-pack wiring is a follow-up. Signed-off-by: Ettore Di Giacinto --- gallery/index.yaml | 75 ---------------------------------------------- 1 file changed, 75 deletions(-) diff --git a/gallery/index.yaml b/gallery/index.yaml index 94526dbe03d3..4105fa62ac9b 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -32534,78 +32534,3 @@ uri: huggingface://cstr/mimo-asr-GGUF/mimo-asr-q4_k.gguf - filename: mimo-tokenizer-q4_k.gguf uri: huggingface://cstr/mimo-tokenizer-GGUF/mimo-tokenizer-q4_k.gguf -- name: qwen3-tts-crispasr - url: github:mudler/LocalAI/gallery/virtual.yaml@master - urls: - - https://huggingface.co/cstr/qwen3-tts-0.6b-base-GGUF - description: | - Qwen3-TTS 0.6B (12 Hz, Base) text-to-speech, synthesized through the CrispASR backend. Requires a companion 12 Hz tokenizer/codec GGUF. Default GGUF size ~986 MB. - tags: - - crispasr - - tts - - text-to-speech - - gguf - overrides: - backend: crispasr - known_usecases: - - tts - name: qwen3-tts-crispasr - options: - - "codec:qwen3-tts-tokenizer-12hz.gguf" - parameters: - model: qwen3-tts-12hz-0.6b-base-q8_0.gguf - files: - - filename: qwen3-tts-12hz-0.6b-base-q8_0.gguf - uri: huggingface://cstr/qwen3-tts-0.6b-base-GGUF/qwen3-tts-12hz-0.6b-base-q8_0.gguf - - filename: qwen3-tts-tokenizer-12hz.gguf - uri: huggingface://cstr/qwen3-tts-tokenizer-12hz-GGUF/qwen3-tts-tokenizer-12hz.gguf -- name: orpheus-crispasr - url: github:mudler/LocalAI/gallery/virtual.yaml@master - urls: - - https://huggingface.co/cstr/orpheus-3b-base-GGUF - description: | - Orpheus-3B text-to-speech (Llama-3.2-3B talker + SNAC 24 kHz codec), synthesized through the CrispASR backend. Requires the companion SNAC codec GGUF. Default GGUF size ~3.5 GB. - tags: - - crispasr - - tts - - text-to-speech - - gguf - overrides: - backend: crispasr - known_usecases: - - tts - name: orpheus-crispasr - options: - - "codec:snac-24khz.gguf" - parameters: - model: orpheus-3b-base-q8_0.gguf - files: - - filename: orpheus-3b-base-q8_0.gguf - uri: huggingface://cstr/orpheus-3b-base-GGUF/orpheus-3b-base-q8_0.gguf - - filename: snac-24khz.gguf - uri: huggingface://cstr/snac-24khz-GGUF/snac-24khz.gguf -- name: chatterbox-crispasr - url: github:mudler/LocalAI/gallery/virtual.yaml@master - urls: - - https://huggingface.co/cstr/chatterbox-GGUF - description: | - Chatterbox (ResembleAI) text-to-speech: T3 token generator + S3Gen 24 kHz vocoder, synthesized through the CrispASR backend. Requires the companion S3Gen GGUF. Default GGUF size ~880 MB. - tags: - - crispasr - - tts - - text-to-speech - - gguf - overrides: - backend: crispasr - known_usecases: - - tts - name: chatterbox-crispasr - options: - - "codec:chatterbox-s3gen-q8_0.gguf" - parameters: - model: chatterbox-t3-q8_0.gguf - files: - - filename: chatterbox-t3-q8_0.gguf - uri: huggingface://cstr/chatterbox-GGUF/chatterbox-t3-q8_0.gguf - - filename: chatterbox-s3gen-q8_0.gguf - uri: huggingface://cstr/chatterbox-GGUF/chatterbox-s3gen-q8_0.gguf From b7b3f7bbf066e150399fdf2b9b92473adc4e22d7 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sun, 31 May 2026 08:22:36 +0000 Subject: [PATCH 28/29] feat(crispasr): speaker: and voice: TTS options (baked speakers + voice packs/prompts) speaker: -> crispasr_session_set_speaker_name (baked speakers: qwen3-tts CustomVoice, orpheus). voice:(+voice_text:) -> crispasr_session_set_voice (voice-pack GGUF, or WAV zero-shot clone with ref text). Applied at Load as the default voice; req.Voice still overrides the speaker per request. Signed-off-by: Ettore Di Giacinto --- backend/go/crispasr/cpp/crispasr_shim.cpp | 12 +++++++++ backend/go/crispasr/cpp/crispasr_shim.h | 1 + backend/go/crispasr/gocrispasr.go | 32 +++++++++++++++++++++++ backend/go/crispasr/gocrispasr_test.go | 1 + backend/go/crispasr/main.go | 1 + 5 files changed, 47 insertions(+) diff --git a/backend/go/crispasr/cpp/crispasr_shim.cpp b/backend/go/crispasr/cpp/crispasr_shim.cpp index 11e897b0f6b3..bf6151ae1384 100644 --- a/backend/go/crispasr/cpp/crispasr_shim.cpp +++ b/backend/go/crispasr/cpp/crispasr_shim.cpp @@ -31,6 +31,8 @@ float *crispasr_session_synthesize(crispasr_session *s, const char *text, int *out_n_samples); void crispasr_pcm_free(float *pcm); int crispasr_session_set_speaker_name(crispasr_session *s, const char *name); +int crispasr_session_set_voice(crispasr_session *s, const char *path, + const char *ref_text_or_null); } static crispasr_session *g_session = nullptr; @@ -239,3 +241,13 @@ int tts_set_voice(const char *name) { if (!g_session || !name || !*name) return 0; return crispasr_session_set_speaker_name(g_session, name); } + +// tts_set_voice_file loads a voice from a file: a .gguf path selects a voice +// pack, a .wav path with a non-empty ref_text performs zero-shot voice cloning +// (the C API returns -2 when ref_text is required but missing). Returns -1 when +// no session is open or path is null. +int tts_set_voice_file(const char *path, const char *ref_text) { + if (!g_session || !path) return -1; + const char *ref = (ref_text && *ref_text) ? ref_text : nullptr; + return crispasr_session_set_voice(g_session, path, ref); +} diff --git a/backend/go/crispasr/cpp/crispasr_shim.h b/backend/go/crispasr/cpp/crispasr_shim.h index 40f1c31f378a..7c593951a08f 100644 --- a/backend/go/crispasr/cpp/crispasr_shim.h +++ b/backend/go/crispasr/cpp/crispasr_shim.h @@ -19,4 +19,5 @@ void set_abort(int v); float *tts_synthesize(const char *text, int *out_n_samples); // 24kHz mono float, malloc'd; NULL on failure void tts_free(float *pcm); int tts_set_voice(const char *name); // best-effort speaker selection; 0 ok +int tts_set_voice_file(const char *path, const char *ref_text); // load voice pack (.gguf) or zero-shot clone (.wav + ref_text) } diff --git a/backend/go/crispasr/gocrispasr.go b/backend/go/crispasr/gocrispasr.go index 9ecb5543ba54..dc21a28fd56a 100644 --- a/backend/go/crispasr/gocrispasr.go +++ b/backend/go/crispasr/gocrispasr.go @@ -32,6 +32,7 @@ var ( CppTTSSynthesize func(text string, outNSamples unsafe.Pointer) uintptr CppTTSFree func(ptr uintptr) CppTTSSetVoice func(name string) int + CppTTSSetVoiceFile func(path string, refText string) int ) type CrispASR struct { @@ -53,6 +54,9 @@ func (w *CrispASR) Load(opts *pb.ModelOptions) error { vadOnly := false backendName := "" codecPath := "" + speakerName := "" + voicePath := "" + voiceRefText := "" for _, oo := range opts.Options { if oo == "vad_only" { @@ -64,6 +68,12 @@ func (w *CrispASR) Load(opts *pb.ModelOptions) error { backendName = value case ok && key == "codec": codecPath = value + case ok && key == "speaker": + speakerName = value + case ok && key == "voice": + voicePath = value + case ok && key == "voice_text": + voiceRefText = value default: fmt.Fprintf(os.Stderr, "Unrecognized option: %v\n", oo) } @@ -83,6 +93,12 @@ func (w *CrispASR) Load(opts *pb.ModelOptions) error { codecPath = filepath.Join(filepath.Dir(opts.ModelFile), codecPath) } + // A voice file (.gguf pack or .wav prompt) is resolved against the model + // directory just like the codec, so a config can reference a sibling file. + if voicePath != "" && !filepath.IsAbs(voicePath) { + voicePath = filepath.Join(filepath.Dir(opts.ModelFile), voicePath) + } + if ret := CppLoadModel(opts.ModelFile, int(opts.Threads), backendName); ret != 0 { return fmt.Errorf("Failed to load CrispASR transcription model") } @@ -97,6 +113,22 @@ func (w *CrispASR) Load(opts *pb.ModelOptions) error { fmt.Fprintf(os.Stderr, "CrispASR companion file loaded: %s\n", codecPath) } + // Apply the Load-time default voice. A baked speaker (speaker:) is selected + // by name and is best-effort: a backend that can't honor it is logged, not + // fatal. A voice file (voice:) is a hard requirement once configured, so a + // negative rc fails Load. + if speakerName != "" { + if rc := CppTTSSetVoice(speakerName); rc != 0 { + fmt.Fprintf(os.Stderr, "crispasr: speaker %q not applied (rc=%d)\n", speakerName, rc) + } + } + if voicePath != "" { + if rc := CppTTSSetVoiceFile(voicePath, voiceRefText); rc < 0 { + return fmt.Errorf("crispasr: failed to load voice %q (rc=%d)", voicePath, rc) + } + fmt.Fprintf(os.Stderr, "CrispASR voice loaded: %s\n", voicePath) + } + fmt.Fprintf(os.Stderr, "CrispASR backend selected: %s\n", CppGetBackend()) return nil diff --git a/backend/go/crispasr/gocrispasr_test.go b/backend/go/crispasr/gocrispasr_test.go index 99417d33a333..63ea47907b09 100644 --- a/backend/go/crispasr/gocrispasr_test.go +++ b/backend/go/crispasr/gocrispasr_test.go @@ -55,6 +55,7 @@ func ensureLibLoaded() { purego.RegisterLibFunc(&CppTTSSynthesize, gosd, "tts_synthesize") purego.RegisterLibFunc(&CppTTSFree, gosd, "tts_free") purego.RegisterLibFunc(&CppTTSSetVoice, gosd, "tts_set_voice") + purego.RegisterLibFunc(&CppTTSSetVoiceFile, gosd, "tts_set_voice_file") }) if libLoadErr != nil { Skip("whisper library not loadable: " + libLoadErr.Error()) diff --git a/backend/go/crispasr/main.go b/backend/go/crispasr/main.go index 09a90991f531..c2069bd851f5 100644 --- a/backend/go/crispasr/main.go +++ b/backend/go/crispasr/main.go @@ -43,6 +43,7 @@ func main() { {&CppTTSSynthesize, "tts_synthesize"}, {&CppTTSFree, "tts_free"}, {&CppTTSSetVoice, "tts_set_voice"}, + {&CppTTSSetVoiceFile, "tts_set_voice_file"}, } for _, lf := range libFuncs { From 5f208a454a89f2dc585b800b6873eb5b1e50e2f4 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sun, 31 May 2026 08:34:00 +0000 Subject: [PATCH 29/29] feat(gallery): re-add e2e-verified TTS engines (chatterbox, qwen3-tts-customvoice, orpheus) Signed-off-by: Ettore Di Giacinto --- gallery/index.yaml | 78 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/gallery/index.yaml b/gallery/index.yaml index 4105fa62ac9b..865eec9f0482 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -32277,6 +32277,84 @@ files: - filename: vibevoice-realtime-0.5b-q4_k.gguf uri: huggingface://cstr/vibevoice-realtime-0.5b-GGUF/vibevoice-realtime-0.5b-q4_k.gguf +- name: chatterbox-tts-crispasr + url: github:mudler/LocalAI/gallery/virtual.yaml@master + urls: + - https://huggingface.co/cstr/chatterbox-GGUF + description: | + Chatterbox (ResembleAI, MIT) text-to-speech synthesized through the CrispASR backend. Two-GGUF runtime: a Llama T3 token model plus an S3Gen codec companion (tokens to 24 kHz waveform). Auto-detected by CrispASR and ships with a built-in default voice; runs end-to-end on CPU and produces 24 kHz mono audio. Default GGUF sizes ~630 MB (T3) + ~358 MB (S3Gen). + tags: + - crispasr + - tts + - text-to-speech + - gguf + overrides: + backend: crispasr + known_usecases: + - tts + name: chatterbox-tts-crispasr + options: + - "codec:chatterbox-s3gen-q8_0.gguf" + parameters: + model: chatterbox-t3-q8_0.gguf + files: + - filename: chatterbox-t3-q8_0.gguf + uri: huggingface://cstr/chatterbox-GGUF/chatterbox-t3-q8_0.gguf + - filename: chatterbox-s3gen-q8_0.gguf + uri: huggingface://cstr/chatterbox-GGUF/chatterbox-s3gen-q8_0.gguf +- name: qwen3-tts-customvoice-crispasr + url: github:mudler/LocalAI/gallery/virtual.yaml@master + urls: + - https://huggingface.co/cstr/qwen3-tts-0.6b-customvoice-GGUF + description: | + Qwen3-TTS CustomVoice 0.6B (12 Hz) text-to-speech synthesized through the CrispASR backend. Fixed-speaker fine-tune driven via an explicit backend selector plus a tokenizer codec companion. Ships baked speakers (vivian, aiden, dylan, eric, ono_anna, ryan, serena, sohee, uncle_fu); the default config selects vivian. Runs end-to-end on CPU and produces 24 kHz mono audio. Default GGUF sizes ~968 MB (talker) + ~358 MB (tokenizer). + tags: + - crispasr + - tts + - text-to-speech + - gguf + overrides: + backend: crispasr + known_usecases: + - tts + name: qwen3-tts-customvoice-crispasr + options: + - "backend:qwen3-tts" + - "codec:qwen3-tts-tokenizer-12hz.gguf" + - "speaker:vivian" + parameters: + model: qwen3-tts-12hz-0.6b-customvoice-q8_0.gguf + files: + - filename: qwen3-tts-12hz-0.6b-customvoice-q8_0.gguf + uri: huggingface://cstr/qwen3-tts-0.6b-customvoice-GGUF/qwen3-tts-12hz-0.6b-customvoice-q8_0.gguf + - filename: qwen3-tts-tokenizer-12hz.gguf + uri: huggingface://cstr/qwen3-tts-tokenizer-12hz-GGUF/qwen3-tts-tokenizer-12hz.gguf +- name: orpheus-tts-crispasr + url: github:mudler/LocalAI/gallery/virtual.yaml@master + urls: + - https://huggingface.co/cstr/orpheus-3b-base-GGUF + description: | + Orpheus-3B (Llama-3.2 base) text-to-speech synthesized through the CrispASR backend. Auto-detected by CrispASR; needs a SNAC 24 kHz codec companion and a baked speaker. Ships speaker tara (selected by the default config). Runs end-to-end on CPU and produces 24 kHz mono audio. Default GGUF sizes ~3.5 GB (model) + ~26 MB (SNAC codec). + tags: + - crispasr + - tts + - text-to-speech + - gguf + overrides: + backend: crispasr + known_usecases: + - tts + name: orpheus-tts-crispasr + options: + - "codec:snac-24khz.gguf" + - "speaker:tara" + parameters: + model: orpheus-3b-base-q8_0.gguf + files: + - filename: orpheus-3b-base-q8_0.gguf + uri: huggingface://cstr/orpheus-3b-base-GGUF/orpheus-3b-base-q8_0.gguf + - filename: snac-24khz.gguf + uri: huggingface://cstr/snac-24khz-GGUF/snac-24khz.gguf - name: hubert-crispasr url: github:mudler/LocalAI/gallery/virtual.yaml@master urls: