diff --git a/backend/go/crispasr/cpp/crispasr_shim.cpp b/backend/go/crispasr/cpp/crispasr_shim.cpp index b1b23231ff99..592c897c7dc6 100644 --- a/backend/go/crispasr/cpp/crispasr_shim.cpp +++ b/backend/go/crispasr/cpp/crispasr_shim.cpp @@ -23,6 +23,10 @@ const char *crispasr_session_result_segment_text(crispasr_session_result *r, int64_t crispasr_session_result_segment_t0(crispasr_session_result *r, int i); int64_t crispasr_session_result_segment_t1(crispasr_session_result *r, int i); void crispasr_session_result_free(crispasr_session_result *r); +float *crispasr_session_synthesize(crispasr_session *s, const char *text, + int *out_n_samples); +void crispasr_pcm_free(float *pcm); +int crispasr_session_set_speaker_name(crispasr_session *s, const char *name); } static crispasr_session *g_session = nullptr; @@ -199,3 +203,21 @@ int64_t get_segment_t1(int i) { const char *get_backend(void) { return g_session ? crispasr_session_backend(g_session) : ""; } + +// TTS uses the already-open session (crispasr_session_open auto-detects a TTS +// model). Output is 24 kHz mono float PCM (upstream CrispASR convention), +// malloc'd by the C API; the caller must release it via tts_free. +float *tts_synthesize(const char *text, int *out_n_samples) { + if (out_n_samples) *out_n_samples = 0; + if (!g_session || !text) return nullptr; + return crispasr_session_synthesize(g_session, text, out_n_samples); +} + +void tts_free(float *pcm) { + if (pcm) crispasr_pcm_free(pcm); +} + +int tts_set_voice(const char *name) { + if (!g_session || !name || !*name) return 0; + return crispasr_session_set_speaker_name(g_session, name); +} diff --git a/backend/go/crispasr/cpp/crispasr_shim.h b/backend/go/crispasr/cpp/crispasr_shim.h index fa01097423cb..c993a86401c3 100644 --- a/backend/go/crispasr/cpp/crispasr_shim.h +++ b/backend/go/crispasr/cpp/crispasr_shim.h @@ -14,4 +14,7 @@ int64_t get_segment_t0(int i); int64_t get_segment_t1(int i); const char *get_backend(void); void set_abort(int v); +float *tts_synthesize(const char *text, int *out_n_samples); // 24kHz mono float, malloc'd; NULL on failure +void tts_free(float *pcm); +int tts_set_voice(const char *name); // best-effort speaker selection; 0 ok } diff --git a/backend/go/crispasr/gocrispasr.go b/backend/go/crispasr/gocrispasr.go index 17d93502aa1c..22a9375de5b4 100644 --- a/backend/go/crispasr/gocrispasr.go +++ b/backend/go/crispasr/gocrispasr.go @@ -9,6 +9,7 @@ import ( "sync" "unsafe" + "github.com/go-audio/audio" "github.com/go-audio/wav" "github.com/mudler/LocalAI/pkg/grpc/base" pb "github.com/mudler/LocalAI/pkg/grpc/proto" @@ -27,6 +28,9 @@ var ( CppGetSegmentEnd func(i int) int64 CppGetBackend func() string CppSetAbort func(v int) + CppTTSSynthesize func(text string, outNSamples unsafe.Pointer) uintptr + CppTTSFree func(ptr uintptr) + CppTTSSetVoice func(name string) int ) type CrispASR struct { @@ -304,3 +308,121 @@ func (w *CrispASR) AudioTranscriptionStream(ctx context.Context, opts *pb.Transc results <- &pb.TranscriptStreamResponse{FinalResult: final} return nil } + +// synthesize returns 24 kHz mono float32 PCM for text via the open session. +func (w *CrispASR) synthesize(text string) ([]float32, error) { + if text == "" { + return nil, fmt.Errorf("crispasr: TTS requires non-empty text") + } + var n int32 + ptr := CppTTSSynthesize(text, unsafe.Pointer(&n)) + if ptr == 0 || n <= 0 { + return nil, fmt.Errorf("crispasr: synthesis failed (the loaded model may not be a supported TTS backend, or needs extra config e.g. orpheus SNAC codec)") + } + defer CppTTSFree(ptr) + src := unsafe.Slice((*float32)(unsafe.Pointer(ptr)), int(n)) + out := make([]float32, int(n)) // copy out of C memory before free + copy(out, src) + return out, nil +} + +// setVoice applies a per-call speaker/voice override (best effort). CrispASR +// returns a negative code when the active backend can't honor the name; we log +// it rather than fail, so an unknown voice falls back to the default speaker. +func setVoice(voice string) { + v := strings.TrimSpace(voice) + if v == "" { + return + } + if rc := CppTTSSetVoice(v); rc != 0 { + fmt.Fprintf(os.Stderr, "crispasr: voice %q not applied by the active TTS backend (rc=%d); using default\n", v, rc) + } +} + +func (w *CrispASR) TTS(req *pb.TTSRequest) error { + if req.Dst == "" { + return fmt.Errorf("crispasr: TTS requires a destination path") + } + setVoice(req.Voice) + pcm, err := w.synthesize(req.Text) + if err != nil { + return err + } + return writeWAV24k(req.Dst, pcm) +} + +// TTSStream is the streaming counterpart to TTS. CrispASR has no progressive +// (native streaming) synth, so we synthesize the whole utterance, encode it to +// a 24 kHz WAV, and emit the encoded bytes as a single chunk. The gRPC server +// wrapper (pkg/grpc/server.go:TTSStream) ranges over the channel until it is +// closed, so this method owns the close - mirrors vibevoice-cpp's TTSStream. +func (w *CrispASR) TTSStream(req *pb.TTSRequest, results chan []byte) error { + defer close(results) + + if req.Text == "" { + return fmt.Errorf("crispasr: TTSStream requires text") + } + setVoice(req.Voice) + pcm, err := w.synthesize(req.Text) + if err != nil { + return err + } + + tmp, err := os.CreateTemp("", "crispasr-tts-stream-*.wav") + if err != nil { + return fmt.Errorf("crispasr: tempfile: %w", err) + } + dst := tmp.Name() + if err := tmp.Close(); err != nil { + return fmt.Errorf("crispasr: close tempfile: %w", err) + } + defer func() { _ = os.Remove(dst) }() + + if err := writeWAV24k(dst, pcm); err != nil { + return err + } + + encoded, err := os.ReadFile(dst) + if err != nil { + return fmt.Errorf("crispasr: read tempfile: %w", err) + } + results <- encoded + return nil +} + +// writeWAV24k writes pcm as a 24000 Hz, mono, 16-bit PCM WAV at dst. +func writeWAV24k(dst string, pcm []float32) error { + f, err := os.Create(dst) + if err != nil { + return fmt.Errorf("crispasr: create %q: %w", dst, err) + } + + enc := wav.NewEncoder(f, 24000, 16, 1, 1) + ints := make([]int, len(pcm)) + for i, s := range pcm { + if s > 1 { + s = 1 + } else if s < -1 { + s = -1 + } + ints[i] = int(s * 32767) + } + buf := &audio.IntBuffer{ + Format: &audio.Format{NumChannels: 1, SampleRate: 24000}, + Data: ints, + SourceBitDepth: 16, + } + if err := enc.Write(buf); err != nil { + _ = enc.Close() + _ = f.Close() + return fmt.Errorf("crispasr: encode WAV: %w", err) + } + if err := enc.Close(); err != nil { + _ = f.Close() + return fmt.Errorf("crispasr: finalize WAV: %w", err) + } + if err := f.Close(); err != nil { + return fmt.Errorf("crispasr: close %q: %w", dst, err) + } + return nil +} diff --git a/backend/go/crispasr/gocrispasr_test.go b/backend/go/crispasr/gocrispasr_test.go index 658d4081dffb..5d1c2175ac2d 100644 --- a/backend/go/crispasr/gocrispasr_test.go +++ b/backend/go/crispasr/gocrispasr_test.go @@ -3,6 +3,7 @@ package main import ( "context" "os" + "path/filepath" "strings" "sync" "testing" @@ -50,6 +51,9 @@ func ensureLibLoaded() { purego.RegisterLibFunc(&CppGetSegmentEnd, gosd, "get_segment_t1") purego.RegisterLibFunc(&CppGetBackend, gosd, "get_backend") purego.RegisterLibFunc(&CppSetAbort, gosd, "set_abort") + purego.RegisterLibFunc(&CppTTSSynthesize, gosd, "tts_synthesize") + purego.RegisterLibFunc(&CppTTSFree, gosd, "tts_free") + purego.RegisterLibFunc(&CppTTSSetVoice, gosd, "tts_set_voice") }) if libLoadErr != nil { Skip("whisper library not loadable: " + libLoadErr.Error()) @@ -68,6 +72,17 @@ func fixturesOrSkip() (string, string) { return modelPath, audioPath } +// ttsModelOrSkip returns the TTS model path or skips the spec when the env var +// is unset. Like the transcription fixtures, this never runs in default CI — it +// needs a real TTS model (e.g. a vibevoice GGUF) on disk. +func ttsModelOrSkip() string { + modelPath := os.Getenv("CRISPASR_TTS_MODEL_PATH") + if modelPath == "" { + Skip("set CRISPASR_TTS_MODEL_PATH to run this spec") + } + return modelPath +} + var _ = Describe("CrispASR", func() { Context("AudioTranscription cancellation", func() { It("returns codes.Canceled on a pre-cancelled context and still succeeds afterwards", func() { @@ -153,4 +168,24 @@ var _ = Describe("CrispASR", func() { "concat(deltas) must equal final.Text") }) }) + + Context("TTS", func() { + It("synthesizes a non-empty WAV", func() { + ttsModel := ttsModelOrSkip() + ensureLibLoaded() + + w := &CrispASR{} + Expect(w.Load(&pb.ModelOptions{ModelFile: ttsModel})).To(Succeed()) + + dst := filepath.Join(GinkgoT().TempDir(), "out.wav") + Expect(w.TTS(&pb.TTSRequest{Text: "Hello from CrispASR.", Dst: dst})).To(Succeed()) + + info, err := os.Stat(dst) + Expect(err).ToNot(HaveOccurred(), "synthesized WAV should exist at %q", dst) + // A real 24 kHz mono WAV is a 44-byte header plus samples; anything + // this small would mean an empty/failed synth. + Expect(info.Size()).To(BeNumerically(">", 1024), + "expected a non-trivial WAV, got %d bytes", info.Size()) + }) + }) }) diff --git a/backend/go/crispasr/main.go b/backend/go/crispasr/main.go index 4d0f2b92b2ab..e178b48be5b3 100644 --- a/backend/go/crispasr/main.go +++ b/backend/go/crispasr/main.go @@ -39,6 +39,9 @@ func main() { {&CppGetSegmentEnd, "get_segment_t1"}, {&CppGetBackend, "get_backend"}, {&CppSetAbort, "set_abort"}, + {&CppTTSSynthesize, "tts_synthesize"}, + {&CppTTSFree, "tts_free"}, + {&CppTTSSetVoice, "tts_set_voice"}, } for _, lf := range libFuncs { diff --git a/gallery/index.yaml b/gallery/index.yaml index 9da2a1c3d891..061e8b64384a 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -32256,3 +32256,24 @@ files: - filename: vibevoice-asr-q4_k.gguf uri: huggingface://cstr/vibevoice-asr-GGUF/vibevoice-asr-q4_k.gguf +- name: vibevoice-tts-crispasr + url: github:mudler/LocalAI/gallery/vibevoice-tts-crispasr.yaml@master + urls: + - https://huggingface.co/cstr/vibevoice-realtime-0.5b-GGUF + description: | + VibeVoice Realtime 0.5B text-to-speech (TTS) model, synthesized through the CrispASR backend. Produces 24 kHz mono audio; runs end-to-end on CPU with a built-in default voice. Default GGUF size ~636 MB. + tags: + - crispasr + - tts + - text-to-speech + - gguf + overrides: + backend: crispasr + known_usecases: + - tts + name: vibevoice-tts-crispasr + parameters: + model: vibevoice-realtime-0.5b-q4_k.gguf + files: + - filename: vibevoice-realtime-0.5b-q4_k.gguf + uri: huggingface://cstr/vibevoice-realtime-0.5b-GGUF/vibevoice-realtime-0.5b-q4_k.gguf diff --git a/gallery/vibevoice-tts-crispasr.yaml b/gallery/vibevoice-tts-crispasr.yaml new file mode 100644 index 000000000000..dae60fe4f419 --- /dev/null +++ b/gallery/vibevoice-tts-crispasr.yaml @@ -0,0 +1,7 @@ +--- +name: "vibevoice-tts-crispasr" + +config_file: | + backend: crispasr + parameters: + model: vibevoice-realtime-0.5b-q4_k.gguf