From c7f5e8b611752644f34cb7519437667b2e8fe5c5 Mon Sep 17 00:00:00 2001 From: Cynthia J Date: Tue, 9 Jun 2026 10:34:37 -0700 Subject: [PATCH 1/4] speech config init --- .../example/lib/pages/chat_page.dart | 63 +++++++++- .../example/lib/utils/audio_output.dart | 7 ++ .../firebase_ai/lib/firebase_ai.dart | 3 +- .../firebase_ai/firebase_ai/lib/src/api.dart | 9 ++ .../firebase_ai/lib/src/live_api.dart | 73 +----------- .../firebase_ai/lib/src/speech_config.dart | 111 ++++++++++++++++++ .../firebase_ai/test/live_test.dart | 73 ++++++++++++ 7 files changed, 263 insertions(+), 76 deletions(-) create mode 100644 packages/firebase_ai/firebase_ai/lib/src/speech_config.dart diff --git a/packages/firebase_ai/firebase_ai/example/lib/pages/chat_page.dart b/packages/firebase_ai/firebase_ai/example/lib/pages/chat_page.dart index 5eb1ee91a4a8..e65bc03081d5 100644 --- a/packages/firebase_ai/firebase_ai/example/lib/pages/chat_page.dart +++ b/packages/firebase_ai/firebase_ai/example/lib/pages/chat_page.dart @@ -15,6 +15,7 @@ import 'package:flutter/material.dart'; import 'package:firebase_ai/firebase_ai.dart'; import '../widgets/message_widget.dart'; +import '../utils/audio_output.dart'; class ChatPage extends StatefulWidget { const ChatPage({ @@ -39,13 +40,25 @@ class _ChatPageState extends State { final List _messages = []; bool _loading = false; bool _enableThinking = false; + bool _enableTTS = false; + final AudioOutput _audioOutput = AudioOutput(); @override void initState() { super.initState(); + _audioOutput.init(); _initializeChat(); } + @override + void dispose() { + _audioOutput.dispose(); + _scrollController.dispose(); + _textController.dispose(); + _textFieldFocus.dispose(); + super.dispose(); + } + void _initializeChat() { final generationConfig = GenerationConfig( thinkingConfig: _enableThinking @@ -54,15 +67,22 @@ class _ChatPageState extends State { includeThoughts: true, ) // Using thinkingBudget since we are testing with gemini 2.5 : null, + responseModalities: _enableTTS + ? [ResponseModalities.text, ResponseModalities.audio] + : null, + speechConfig: _enableTTS + ? SpeechConfig(voiceName: 'Kore', languageCode: 'en-US') + : null, ); + final modelName = _enableTTS ? 'gemini-3.1-flash-tts-preview' : 'gemini-3.1-flash-lite'; if (widget.useVertexBackend) { _model = FirebaseAI.vertexAI(location: 'global').generativeModel( - model: 'gemini-3.1-flash-lite', + model: modelName, generationConfig: generationConfig, ); } else { _model = FirebaseAI.googleAI().generativeModel( - model: 'gemini-3.1-flash-lite', + model: modelName, generationConfig: generationConfig, ); } @@ -103,6 +123,16 @@ class _ChatPageState extends State { }); }, ), + SwitchListTile( + title: const Text('Enable TTS (gemini-3.1-flash-tts-preview)'), + value: _enableTTS, + onChanged: (bool value) { + setState(() { + _enableTTS = value; + _initializeChat(); + }); + }, + ), Expanded( child: ListView.builder( controller: _scrollController, @@ -195,6 +225,9 @@ class _ChatPageState extends State { final textBuffer = StringBuffer(); _messages.add(MessageData(text: textBuffer.toString(), fromUser: false)); + if (_enableTTS) { + await _audioOutput.playStream(); + } await for (final response in responseStream) { final thought = response.thoughtSummary; if (thought != null) { @@ -208,8 +241,20 @@ class _ChatPageState extends State { _messages.last = MessageData(text: textBuffer.toString(), fromUser: false); }); + if (_enableTTS) { + for (final candidate in response.candidates) { + for (final part in candidate.content.parts) { + if (part is InlineDataPart && part.mimeType.startsWith('audio/')) { + _audioOutput.addDataToAudioStream(part.bytes); + } + } + } + } _scrollDown(); } + if (_enableTTS) { + _audioOutput.finishStream(); + } if (textBuffer.isEmpty) { _showError('No response from API.'); @@ -244,7 +289,19 @@ class _ChatPageState extends State { var text = response?.text; _messages.add(MessageData(text: text, fromUser: false)); - if (text == null) { + if (_enableTTS && response != null) { + await _audioOutput.playStream(); + for (final candidate in response.candidates) { + for (final part in candidate.content.parts) { + if (part is InlineDataPart && part.mimeType.startsWith('audio/')) { + _audioOutput.addDataToAudioStream(part.bytes); + } + } + } + _audioOutput.finishStream(); + } + + if (text == null && !_enableTTS) { _showError('No response from API.'); return; } else { diff --git a/packages/firebase_ai/firebase_ai/example/lib/utils/audio_output.dart b/packages/firebase_ai/firebase_ai/example/lib/utils/audio_output.dart index 4ec2a1a4911d..d1cfc988b453 100644 --- a/packages/firebase_ai/firebase_ai/example/lib/utils/audio_output.dart +++ b/packages/firebase_ai/firebase_ai/example/lib/utils/audio_output.dart @@ -94,4 +94,11 @@ class AudioOutput { SoLoud.instance.setDataIsEnded(currentStream); await SoLoud.instance.stop(currentHandle); } + + void finishStream() { + var currentStream = stream; + if (currentStream != null) { + SoLoud.instance.setDataIsEnded(currentStream); + } + } } diff --git a/packages/firebase_ai/firebase_ai/lib/firebase_ai.dart b/packages/firebase_ai/firebase_ai/lib/firebase_ai.dart index 460fe5cb79e9..440f00bf2f5d 100644 --- a/packages/firebase_ai/firebase_ai/lib/firebase_ai.dart +++ b/packages/firebase_ai/firebase_ai/lib/firebase_ai.dart @@ -116,8 +116,9 @@ export 'src/live_api.dart' SessionResumptionConfig, SessionResumptionUpdate, SlidingWindow, - SpeechConfig, Transcription; +export 'src/speech_config.dart' + show SpeechConfig, MultiSpeakerVoiceConfig, SpeakerVoiceConfig; export 'src/live_session.dart' show LiveSession; export 'src/schema.dart' show JSONSchema, Schema, SchemaType; export 'src/server_template/template_chat.dart' diff --git a/packages/firebase_ai/firebase_ai/lib/src/api.dart b/packages/firebase_ai/firebase_ai/lib/src/api.dart index 1d2ee8947954..aee9c108159c 100644 --- a/packages/firebase_ai/firebase_ai/lib/src/api.dart +++ b/packages/firebase_ai/firebase_ai/lib/src/api.dart @@ -16,8 +16,10 @@ import 'content.dart'; import 'error.dart'; import 'image_config.dart'; import 'schema.dart'; +import 'speech_config.dart'; import 'tool.dart' show Tool, ToolConfig; + /// Response for Count Tokens final class CountTokensResponse { // ignore: public_member_api_docs @@ -1206,6 +1208,7 @@ abstract class BaseGenerationConfig { this.frequencyPenalty, this.responseModalities, this.mediaResolution, + this.speechConfig, }) : assert(mediaResolution != MediaResolution.ultraHigh, 'MediaResolution.ultraHigh is only supported on individual media parts.'); @@ -1294,6 +1297,9 @@ abstract class BaseGenerationConfig { /// [MediaResolution.ultraHigh] is only supported on individual media parts. final MediaResolution? mediaResolution; + /// The configuration parameters controlling the model's speech and audio generation. + final SpeechConfig? speechConfig; + // ignore: public_member_api_docs Map toJson() => { if (candidateCount case final candidateCount?) @@ -1312,6 +1318,8 @@ abstract class BaseGenerationConfig { responseModalities.map((modality) => modality.toJson()).toList(), if (mediaResolution case final mediaResolution?) 'mediaResolution': mediaResolution.toJson(), + if (speechConfig case final speechConfig?) + 'speechConfig': speechConfig.toJson(), }; } @@ -1329,6 +1337,7 @@ final class GenerationConfig extends BaseGenerationConfig { super.frequencyPenalty, super.responseModalities, super.mediaResolution, + super.speechConfig, this.responseMimeType, this.responseSchema, this.responseJsonSchema, diff --git a/packages/firebase_ai/firebase_ai/lib/src/live_api.dart b/packages/firebase_ai/firebase_ai/lib/src/live_api.dart index 7061bc94c694..b961d5348786 100644 --- a/packages/firebase_ai/firebase_ai/lib/src/live_api.dart +++ b/packages/firebase_ai/firebase_ai/lib/src/live_api.dart @@ -15,72 +15,6 @@ import 'api.dart'; import 'content.dart'; import 'error.dart'; -/// Configuration for a prebuilt voice. -/// -/// This class allows specifying a voice by its name. -class PrebuiltVoiceConfig { - // ignore: public_member_api_docs - const PrebuiltVoiceConfig({this.voiceName}); - - /// The voice name to use for speech synthesis. - /// - /// See https://cloud.google.com/text-to-speech/docs/chirp3-hd for names and - /// sound demos. - final String? voiceName; - // ignore: public_member_api_docs - Map toJson() => - {if (voiceName case final voiceName?) 'voice_name': voiceName}; -} - -/// Configuration for the voice to be used in speech synthesis. -/// -/// This class currently supports using a prebuilt voice configuration. -class VoiceConfig { - // ignore: public_member_api_docs - VoiceConfig({this.prebuiltVoiceConfig}); - - // ignore: public_member_api_docs - final PrebuiltVoiceConfig? prebuiltVoiceConfig; - // ignore: public_member_api_docs - Map toJson() => { - if (prebuiltVoiceConfig case final prebuiltVoiceConfig?) - 'prebuilt_voice_config': prebuiltVoiceConfig.toJson() - }; -} - -/// Configures speech synthesis settings. -/// -/// Allows specifying the desired voice and language for speech synthesis. -class SpeechConfig { - /// Creates a [SpeechConfig] instance. - /// - /// [voiceName] See https://cloud.google.com/text-to-speech/docs/chirp3-hd - /// for names and sound demos. - /// - /// [languageCode] The language code (BCP-47) for speech synthesis, - /// e.g. "en-US", "fr-FR", "de-DE". - SpeechConfig({String? voiceName, this.languageCode}) - : voiceConfig = voiceName != null - ? VoiceConfig( - prebuiltVoiceConfig: PrebuiltVoiceConfig(voiceName: voiceName)) - : null; - - /// The voice config to use for speech synthesis. - final VoiceConfig? voiceConfig; - - /// The language code (BCP-47) for speech synthesis, - /// e.g. "en-US", "fr-FR", "de-DE". - final String? languageCode; - - // ignore: public_member_api_docs - Map toJson() => { - if (voiceConfig case final voiceConfig?) - 'voice_config': voiceConfig.toJson(), - if (languageCode case final languageCode?) - 'language_code': languageCode, - }; -} - /// The audio transcription configuration. class AudioTranscriptionConfig { // ignore: public_member_api_docs @@ -168,7 +102,7 @@ class SessionResumptionConfig { final class LiveGenerationConfig extends BaseGenerationConfig { // ignore: public_member_api_docs LiveGenerationConfig( - {this.speechConfig, + {super.speechConfig, this.inputAudioTranscription, this.outputAudioTranscription, this.contextWindowCompression, @@ -181,9 +115,6 @@ final class LiveGenerationConfig extends BaseGenerationConfig { super.frequencyPenalty, super.mediaResolution}); - /// The speech configuration. - final SpeechConfig? speechConfig; - /// The transcription of the input aligns with the input audio language. final AudioTranscriptionConfig? inputAudioTranscription; @@ -197,8 +128,6 @@ final class LiveGenerationConfig extends BaseGenerationConfig { @override Map toJson() => { ...super.toJson(), - if (speechConfig case final speechConfig?) - 'speechConfig': speechConfig.toJson(), }; } diff --git a/packages/firebase_ai/firebase_ai/lib/src/speech_config.dart b/packages/firebase_ai/firebase_ai/lib/src/speech_config.dart new file mode 100644 index 000000000000..21606ff03a83 --- /dev/null +++ b/packages/firebase_ai/firebase_ai/lib/src/speech_config.dart @@ -0,0 +1,111 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import 'package:meta/meta.dart'; + +/// Speech configuration class for controlling the model's speech and audio generation behaviors. +class SpeechConfig { + /// The voice name to use for a single-speaker setup. + final String? voiceName; + + /// The multi-speaker configuration. + final MultiSpeakerVoiceConfig? multiSpeakerVoiceConfig; + + /// The optional IETF BCP-47 language code. + final String? languageCode; + + /// Constructs a [SpeechConfig] for a single-speaker setup. + SpeechConfig({this.voiceName, this.languageCode}) + : multiSpeakerVoiceConfig = null; + + /// Constructs a [SpeechConfig] for a multi-speaker setup. + SpeechConfig.multiSpeaker({required this.multiSpeakerVoiceConfig, this.languageCode}) + : voiceName = null; + + /// Convert to json format. + @internal + Map toJson() => { + if (voiceName != null) + 'voice_config': VoiceConfig( + prebuiltVoiceConfig: PrebuiltVoiceConfig(voiceName: voiceName), + ).toJson(), + if (multiSpeakerVoiceConfig case final multiSpeakerVoiceConfig?) + 'multi_speaker_voice_config': multiSpeakerVoiceConfig.toJson(), + if (languageCode case final languageCode?) + 'language_code': languageCode, + }; +} + +/// Configuration for a multi-speaker audio generation setup. +class MultiSpeakerVoiceConfig { + /// A list of voice configurations for the participating speakers. + final List speakerVoiceConfigs; + + /// Constructor + MultiSpeakerVoiceConfig({required this.speakerVoiceConfigs}); + + /// Convert to json format. + Map toJson() => { + 'speaker_voice_configs': + speakerVoiceConfigs.map((e) => e.toJson()).toList(), + }; +} + +/// Configures a participating speaker within a multi-speaker setup. +class SpeakerVoiceConfig { + /// The unique name/identifier of the speaker. + final String speaker; + + /// The specific voice assigned to this speaker. + final String voiceName; + + /// Constructor + SpeakerVoiceConfig({required this.speaker, required this.voiceName}); + + /// Convert to json format. + Map toJson() => { + 'speaker': speaker, + 'voice_config': VoiceConfig( + prebuiltVoiceConfig: PrebuiltVoiceConfig(voiceName: voiceName), + ).toJson(), + }; +} + +/// Configuration for a prebuilt voice. +class PrebuiltVoiceConfig { + /// Constructor + const PrebuiltVoiceConfig({this.voiceName}); + + /// The voice name to use for speech synthesis. + final String? voiceName; + + /// Convert to json format. + Map toJson() => + {if (voiceName case final voiceName?) 'voice_name': voiceName}; +} + +/// Configuration for the voice to be used in speech synthesis. +class VoiceConfig { + /// Constructor + VoiceConfig({this.prebuiltVoiceConfig}); + + /// The prebuilt voice configuration. + final PrebuiltVoiceConfig? prebuiltVoiceConfig; + + /// Convert to json format. + Map toJson() => { + if (prebuiltVoiceConfig case final prebuiltVoiceConfig?) + 'prebuilt_voice_config': prebuiltVoiceConfig.toJson() + }; +} diff --git a/packages/firebase_ai/firebase_ai/test/live_test.dart b/packages/firebase_ai/firebase_ai/test/live_test.dart index ed6f088aed4a..e7345ce575ff 100644 --- a/packages/firebase_ai/firebase_ai/test/live_test.dart +++ b/packages/firebase_ai/firebase_ai/test/live_test.dart @@ -18,6 +18,7 @@ import 'package:firebase_ai/src/api.dart'; import 'package:firebase_ai/src/content.dart'; import 'package:firebase_ai/src/error.dart'; import 'package:firebase_ai/src/live_api.dart'; +import 'package:firebase_ai/src/speech_config.dart'; import 'package:flutter_test/flutter_test.dart'; void main() { @@ -50,6 +51,38 @@ void main() { }); }); + test('SpeechConfig.multiSpeaker toJson() returns correct JSON', () { + final multiSpeechConfig = SpeechConfig.multiSpeaker( + multiSpeakerVoiceConfig: MultiSpeakerVoiceConfig( + speakerVoiceConfigs: [ + SpeakerVoiceConfig(speaker: 'Joe', voiceName: 'Kore'), + SpeakerVoiceConfig(speaker: 'Jane', voiceName: 'Puck'), + ], + ), + languageCode: 'en-US', + ); + + expect(multiSpeechConfig.toJson(), { + 'multi_speaker_voice_config': { + 'speaker_voice_configs': [ + { + 'speaker': 'Joe', + 'voice_config': { + 'prebuilt_voice_config': {'voice_name': 'Kore'} + } + }, + { + 'speaker': 'Jane', + 'voice_config': { + 'prebuilt_voice_config': {'voice_name': 'Puck'} + } + } + ] + }, + 'language_code': 'en-US', + }); + }); + test('ResponseModalities enum toJson() returns correct value', () { expect(ResponseModalities.text.toJson(), 'TEXT'); expect(ResponseModalities.image.toJson(), 'IMAGE'); @@ -85,6 +118,46 @@ void main() { expect(liveGenerationConfigWithoutOptionals.toJson(), {}); }); + test('GenerationConfig with SpeechConfig toJson() returns correct JSON', () { + final config = GenerationConfig( + speechConfig: SpeechConfig(voiceName: 'Aoede', languageCode: 'en-US'), + ); + + expect(config.toJson(), { + 'speechConfig': { + 'voice_config': { + 'prebuilt_voice_config': {'voice_name': 'Aoede'} + }, + 'language_code': 'en-US', + } + }); + + final multiConfig = GenerationConfig( + speechConfig: SpeechConfig.multiSpeaker( + multiSpeakerVoiceConfig: MultiSpeakerVoiceConfig( + speakerVoiceConfigs: [ + SpeakerVoiceConfig(speaker: 'Joe', voiceName: 'Kore'), + ], + ), + ), + ); + + expect(multiConfig.toJson(), { + 'speechConfig': { + 'multi_speaker_voice_config': { + 'speaker_voice_configs': [ + { + 'speaker': 'Joe', + 'voice_config': { + 'prebuilt_voice_config': {'voice_name': 'Kore'} + } + } + ] + } + } + }); + }); + test('SessionResumptionConfig toJson() returns correct JSON', () { final resumableConfig = SessionResumptionConfig(); expect(resumableConfig.toJson(), {}); From d0e7f15ea49cd2b25622e28456abb6809a2b067f Mon Sep 17 00:00:00 2001 From: Cynthia J Date: Tue, 9 Jun 2026 11:18:41 -0700 Subject: [PATCH 2/4] add test page for TTS --- .../firebase_ai/example/lib/main.dart | 13 + .../example/lib/pages/chat_page.dart | 68 +-- .../example/lib/pages/tts_page.dart | 448 ++++++++++++++++++ 3 files changed, 465 insertions(+), 64 deletions(-) create mode 100644 packages/firebase_ai/firebase_ai/example/lib/pages/tts_page.dart diff --git a/packages/firebase_ai/firebase_ai/example/lib/main.dart b/packages/firebase_ai/firebase_ai/example/lib/main.dart index 296fbbc79a91..01a68ff9506a 100644 --- a/packages/firebase_ai/firebase_ai/example/lib/main.dart +++ b/packages/firebase_ai/firebase_ai/example/lib/main.dart @@ -32,6 +32,7 @@ import 'pages/schema_page.dart'; import 'pages/server_template_page.dart'; import 'pages/grounding_page.dart'; import 'pages/token_count_page.dart'; +import 'pages/tts_page.dart'; void main() async { WidgetsFlutterBinding.ensureInitialized(); @@ -180,6 +181,11 @@ class _HomeScreenState extends State { title: 'Grounding', useVertexBackend: useVertexBackend, ); + case 11: + return TTSPage( + title: 'TTS Test', + useVertexBackend: useVertexBackend, + ); default: // Fallback to the first page in case of an unexpected index @@ -314,6 +320,13 @@ class _HomeScreenState extends State { label: 'Grounding', tooltip: 'Search & Maps Grounding', ), + BottomNavigationBarItem( + icon: Icon( + Icons.record_voice_over, + ), + label: 'TTS', + tooltip: 'Text to Speech', + ), ], currentIndex: _selectedIndex, onTap: _onItemTapped, diff --git a/packages/firebase_ai/firebase_ai/example/lib/pages/chat_page.dart b/packages/firebase_ai/firebase_ai/example/lib/pages/chat_page.dart index e65bc03081d5..6562420bbdc4 100644 --- a/packages/firebase_ai/firebase_ai/example/lib/pages/chat_page.dart +++ b/packages/firebase_ai/firebase_ai/example/lib/pages/chat_page.dart @@ -15,7 +15,6 @@ import 'package:flutter/material.dart'; import 'package:firebase_ai/firebase_ai.dart'; import '../widgets/message_widget.dart'; -import '../utils/audio_output.dart'; class ChatPage extends StatefulWidget { const ChatPage({ @@ -40,49 +39,27 @@ class _ChatPageState extends State { final List _messages = []; bool _loading = false; bool _enableThinking = false; - bool _enableTTS = false; - final AudioOutput _audioOutput = AudioOutput(); @override void initState() { super.initState(); - _audioOutput.init(); _initializeChat(); } - @override - void dispose() { - _audioOutput.dispose(); - _scrollController.dispose(); - _textController.dispose(); - _textFieldFocus.dispose(); - super.dispose(); - } - void _initializeChat() { final generationConfig = GenerationConfig( thinkingConfig: _enableThinking - ? ThinkingConfig.withThinkingBudget( - null, - includeThoughts: true, - ) // Using thinkingBudget since we are testing with gemini 2.5 - : null, - responseModalities: _enableTTS - ? [ResponseModalities.text, ResponseModalities.audio] - : null, - speechConfig: _enableTTS - ? SpeechConfig(voiceName: 'Kore', languageCode: 'en-US') + ? ThinkingConfig.withThinkingLevel(ThinkingLevel.medium) : null, ); - final modelName = _enableTTS ? 'gemini-3.1-flash-tts-preview' : 'gemini-3.1-flash-lite'; if (widget.useVertexBackend) { _model = FirebaseAI.vertexAI(location: 'global').generativeModel( - model: modelName, + model: 'gemini-3.1-flash-lite', generationConfig: generationConfig, ); } else { _model = FirebaseAI.googleAI().generativeModel( - model: modelName, + model: 'gemini-3.1-flash-lite', generationConfig: generationConfig, ); } @@ -123,16 +100,6 @@ class _ChatPageState extends State { }); }, ), - SwitchListTile( - title: const Text('Enable TTS (gemini-3.1-flash-tts-preview)'), - value: _enableTTS, - onChanged: (bool value) { - setState(() { - _enableTTS = value; - _initializeChat(); - }); - }, - ), Expanded( child: ListView.builder( controller: _scrollController, @@ -225,9 +192,6 @@ class _ChatPageState extends State { final textBuffer = StringBuffer(); _messages.add(MessageData(text: textBuffer.toString(), fromUser: false)); - if (_enableTTS) { - await _audioOutput.playStream(); - } await for (final response in responseStream) { final thought = response.thoughtSummary; if (thought != null) { @@ -241,20 +205,8 @@ class _ChatPageState extends State { _messages.last = MessageData(text: textBuffer.toString(), fromUser: false); }); - if (_enableTTS) { - for (final candidate in response.candidates) { - for (final part in candidate.content.parts) { - if (part is InlineDataPart && part.mimeType.startsWith('audio/')) { - _audioOutput.addDataToAudioStream(part.bytes); - } - } - } - } _scrollDown(); } - if (_enableTTS) { - _audioOutput.finishStream(); - } if (textBuffer.isEmpty) { _showError('No response from API.'); @@ -289,19 +241,7 @@ class _ChatPageState extends State { var text = response?.text; _messages.add(MessageData(text: text, fromUser: false)); - if (_enableTTS && response != null) { - await _audioOutput.playStream(); - for (final candidate in response.candidates) { - for (final part in candidate.content.parts) { - if (part is InlineDataPart && part.mimeType.startsWith('audio/')) { - _audioOutput.addDataToAudioStream(part.bytes); - } - } - } - _audioOutput.finishStream(); - } - - if (text == null && !_enableTTS) { + if (text == null) { _showError('No response from API.'); return; } else { diff --git a/packages/firebase_ai/firebase_ai/example/lib/pages/tts_page.dart b/packages/firebase_ai/firebase_ai/example/lib/pages/tts_page.dart new file mode 100644 index 000000000000..1afb9678e27c --- /dev/null +++ b/packages/firebase_ai/firebase_ai/example/lib/pages/tts_page.dart @@ -0,0 +1,448 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import 'dart:async'; +import 'dart:math'; +import 'dart:typed_data'; +import 'package:flutter/material.dart'; +import 'package:firebase_ai/firebase_ai.dart'; +import 'package:waveform_flutter/waveform_flutter.dart'; +import '../utils/audio_output.dart'; +import '../widgets/audio_visualizer.dart'; + +class TTSPage extends StatefulWidget { + const TTSPage({ + super.key, + required this.title, + required this.useVertexBackend, + }); + + final String title; + final bool useVertexBackend; + + @override + State createState() => _TTSPageState(); +} + +class _TTSPageState extends State { + final AudioOutput _audioOutput = AudioOutput(); + final MockAmplitudeGenerator _mockAmpGen = MockAmplitudeGenerator(); + + bool _isMultiSpeaker = false; + bool _loading = false; + bool _isPlaying = false; + String? _responseText; + + // Single Speaker Controller + final TextEditingController _singlePromptController = TextEditingController( + text: 'Say cheerfully: Have a wonderful day!', + ); + String _selectedVoice = 'Kore'; + + // Multi Speaker Controllers + final TextEditingController _multiScriptController = TextEditingController( + text: "Joe: How's it going today Jane?\nJane: Not too bad, how about you?", + ); + final TextEditingController _speaker1NameController = + TextEditingController(text: 'Joe'); + String _speaker1Voice = 'Kore'; + final TextEditingController _speaker2NameController = + TextEditingController(text: 'Jane'); + String _speaker2Voice = 'Puck'; + + final List _availableVoices = [ + 'Kore', + 'Puck', + 'Fenrir', + 'Aoede', + 'Charon', + 'Leda', + ]; + + Stream? _amplitudeStream; + Timer? _playbackTimer; + + @override + void initState() { + super.initState(); + _audioOutput.init(); + } + + @override + void dispose() { + _audioOutput.dispose(); + _mockAmpGen.stop(); + _playbackTimer?.cancel(); + _singlePromptController.dispose(); + _multiScriptController.dispose(); + _speaker1NameController.dispose(); + _speaker2NameController.dispose(); + super.dispose(); + } + + void _showError(String message) { + showDialog( + context: context, + builder: (context) { + return AlertDialog( + title: const Text('Something went wrong'), + content: SingleChildScrollView( + child: SelectableText(message), + ), + actions: [ + TextButton( + onPressed: () { + Navigator.of(context).pop(); + }, + child: const Text('OK'), + ), + ], + ); + }, + ); + } + + Future _generateAndPlay() async { + setState(() { + _loading = true; + _responseText = null; + }); + + try { + final GenerationConfig config; + final String prompt; + + if (_isMultiSpeaker) { + prompt = _multiScriptController.text; + config = GenerationConfig( + responseModalities: [ResponseModalities.audio], + speechConfig: SpeechConfig.multiSpeaker( + multiSpeakerVoiceConfig: MultiSpeakerVoiceConfig( + speakerVoiceConfigs: [ + SpeakerVoiceConfig( + speaker: _speaker1NameController.text, + voiceName: _speaker1Voice, + ), + SpeakerVoiceConfig( + speaker: _speaker2NameController.text, + voiceName: _speaker2Voice, + ), + ], + ), + ), + ); + } else { + prompt = _singlePromptController.text; + config = GenerationConfig( + responseModalities: [ResponseModalities.audio], + speechConfig: SpeechConfig( + voiceName: _selectedVoice, + languageCode: 'en-US', + ), + ); + } + + // Use the preview model for TTS + final modelName = widget.useVertexBackend + ? 'gemini-2.5-flash-tts' + : 'gemini-3.1-flash-tts-preview'; + final GenerativeModel model; + if (widget.useVertexBackend) { + model = FirebaseAI.vertexAI(location: 'global').generativeModel( + model: modelName, + generationConfig: config, + ); + } else { + model = FirebaseAI.googleAI().generativeModel( + model: modelName, + generationConfig: config, + ); + } + + final response = await model.generateContent([Content.text(prompt)]); + + // Extract text response + _responseText = response.text; + + // Find audio bytes + Uint8List? audioBytes; + for (final candidate in response.candidates) { + for (final part in candidate.content.parts) { + if (part is InlineDataPart && part.mimeType.startsWith('audio/')) { + audioBytes = part.bytes; + break; + } + } + if (audioBytes != null) break; + } + + if (audioBytes == null || audioBytes.isEmpty) { + throw Exception('No audio received from the model.'); + } + + // Play audio and start visualizer + await _audioOutput.playStream(); + _audioOutput.addDataToAudioStream(audioBytes); + _audioOutput.finishStream(); + + // Calculate duration: 24000 Hz, 1 channel, 16-bit (2 bytes) = 48000 bytes/sec + final durationMs = (audioBytes.length / 48.0).round(); + final duration = Duration(milliseconds: durationMs); + + setState(() { + _loading = false; + _isPlaying = true; + _amplitudeStream = _mockAmpGen.start(duration); + }); + + _playbackTimer = Timer(duration, () { + setState(() { + _isPlaying = false; + _amplitudeStream = null; + }); + }); + } catch (e) { + setState(() { + _loading = false; + }); + _showError(e.toString()); + } + } + + void _stopPlayback() { + _audioOutput.stopStream(); + _mockAmpGen.stop(); + _playbackTimer?.cancel(); + setState(() { + _isPlaying = false; + _amplitudeStream = null; + }); + } + + Widget _buildSingleSpeakerForm() { + return Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + TextField( + controller: _singlePromptController, + decoration: const InputDecoration( + labelText: 'Prompt', + hintText: 'Enter text to generate speech from', + border: OutlineInputBorder(), + ), + maxLines: 3, + ), + const SizedBox(height: 16), + DropdownButtonFormField( + value: _selectedVoice, + decoration: const InputDecoration( + labelText: 'Voice Name', + border: OutlineInputBorder(), + ), + items: _availableVoices.map((voice) { + return DropdownMenuItem( + value: voice, + child: Text(voice), + ); + }).toList(), + onChanged: (value) { + if (value != null) { + setState(() { + _selectedVoice = value; + }); + } + }, + ), + ], + ); + } + + Widget _buildMultiSpeakerForm() { + return Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + TextField( + controller: _multiScriptController, + decoration: const InputDecoration( + labelText: 'Script', + hintText: 'Format: Name: text to say', + border: OutlineInputBorder(), + ), + maxLines: 5, + ), + const SizedBox(height: 16), + Row( + children: [ + Expanded( + child: TextField( + controller: _speaker1NameController, + decoration: const InputDecoration( + labelText: 'Speaker 1 Name', + border: OutlineInputBorder(), + ), + ), + ), + const SizedBox(width: 8), + Expanded( + child: DropdownButtonFormField( + value: _speaker1Voice, + decoration: const InputDecoration( + labelText: 'Speaker 1 Voice', + border: OutlineInputBorder(), + ), + items: _availableVoices.map((voice) { + return DropdownMenuItem( + value: voice, + child: Text(voice), + ); + }).toList(), + onChanged: (value) { + if (value != null) { + setState(() { + _speaker1Voice = value; + }); + } + }, + ), + ), + ], + ), + const SizedBox(height: 16), + Row( + children: [ + Expanded( + child: TextField( + controller: _speaker2NameController, + decoration: const InputDecoration( + labelText: 'Speaker 2 Name', + border: OutlineInputBorder(), + ), + ), + ), + const SizedBox(width: 8), + Expanded( + child: DropdownButtonFormField( + value: _speaker2Voice, + decoration: const InputDecoration( + labelText: 'Speaker 2 Voice', + border: OutlineInputBorder(), + ), + items: _availableVoices.map((voice) { + return DropdownMenuItem( + value: voice, + child: Text(voice), + ); + }).toList(), + onChanged: (value) { + if (value != null) { + setState(() { + _speaker2Voice = value; + }); + } + }, + ), + ), + ], + ), + ], + ); + } + + @override + Widget build(BuildContext context) { + return Padding( + padding: const EdgeInsets.all(16.0), + child: Column( + children: [ + SegmentedButton( + segments: const [ + ButtonSegment(value: false, label: Text('Single Speaker')), + ButtonSegment(value: true, label: Text('Multi Speaker')), + ], + selected: {_isMultiSpeaker}, + onSelectionChanged: (value) { + setState(() { + _isMultiSpeaker = value.first; + }); + }, + ), + const SizedBox(height: 16), + Expanded( + child: SingleChildScrollView( + child: _isMultiSpeaker + ? _buildMultiSpeakerForm() + : _buildSingleSpeakerForm(), + ), + ), + const Divider(), + Padding( + padding: const EdgeInsets.symmetric(vertical: 10, horizontal: 8), + child: Row( + children: [ + if (_loading) + const CircularProgressIndicator() + else + IconButton( + icon: Icon(_isPlaying ? Icons.stop : Icons.play_arrow), + iconSize: 36, + color: Theme.of(context).colorScheme.primary, + onPressed: _isPlaying ? _stopPlayback : _generateAndPlay, + ), + const SizedBox(width: 16), + AudioVisualizer( + audioStreamIsActive: _isPlaying, + amplitudeStream: _amplitudeStream, + ), + if (!_isPlaying && !_loading && _responseText != null) + Expanded( + child: Text( + _responseText!, + maxLines: 2, + overflow: TextOverflow.ellipsis, + style: Theme.of(context).textTheme.bodyMedium, + ), + ), + ], + ), + ), + ], + ), + ); + } +} + +class MockAmplitudeGenerator { + StreamController? _controller; + Timer? _timer; + final Random _random = Random(); + + Stream start(Duration duration) { + stop(); + _controller = StreamController.broadcast(); + + _timer = Timer.periodic(const Duration(milliseconds: 100), (timer) { + double current = -60.0 + _random.nextDouble() * 60.0; + _controller?.add(Amplitude(current: current, max: 0.0)); + }); + + return _controller!.stream; + } + + void stop() { + _timer?.cancel(); + _timer = null; + _controller?.close(); + _controller = null; + } +} From 5edf9f6024a4ba3157d9cc7a6d69de5aa09decf7 Mon Sep 17 00:00:00 2001 From: Cynthia J Date: Tue, 9 Jun 2026 11:55:07 -0700 Subject: [PATCH 3/4] fix analyzer --- .../example/lib/pages/tts_page.dart | 10 +++---- .../firebase_ai/lib/firebase_ai.dart | 4 +-- .../firebase_ai/lib/src/speech_config.dart | 29 ++++++++++--------- 3 files changed, 22 insertions(+), 21 deletions(-) diff --git a/packages/firebase_ai/firebase_ai/example/lib/pages/tts_page.dart b/packages/firebase_ai/firebase_ai/example/lib/pages/tts_page.dart index 1afb9678e27c..06ba3a63a9a0 100644 --- a/packages/firebase_ai/firebase_ai/example/lib/pages/tts_page.dart +++ b/packages/firebase_ai/firebase_ai/example/lib/pages/tts_page.dart @@ -245,7 +245,7 @@ class _TTSPageState extends State { ), const SizedBox(height: 16), DropdownButtonFormField( - value: _selectedVoice, + initialValue: _selectedVoice, decoration: const InputDecoration( labelText: 'Voice Name', border: OutlineInputBorder(), @@ -296,7 +296,7 @@ class _TTSPageState extends State { const SizedBox(width: 8), Expanded( child: DropdownButtonFormField( - value: _speaker1Voice, + initialValue: _speaker1Voice, decoration: const InputDecoration( labelText: 'Speaker 1 Voice', border: OutlineInputBorder(), @@ -333,7 +333,7 @@ class _TTSPageState extends State { const SizedBox(width: 8), Expanded( child: DropdownButtonFormField( - value: _speaker2Voice, + initialValue: _speaker2Voice, decoration: const InputDecoration( labelText: 'Speaker 2 Voice', border: OutlineInputBorder(), @@ -362,7 +362,7 @@ class _TTSPageState extends State { @override Widget build(BuildContext context) { return Padding( - padding: const EdgeInsets.all(16.0), + padding: const EdgeInsets.all(16), child: Column( children: [ SegmentedButton( @@ -433,7 +433,7 @@ class MockAmplitudeGenerator { _timer = Timer.periodic(const Duration(milliseconds: 100), (timer) { double current = -60.0 + _random.nextDouble() * 60.0; - _controller?.add(Amplitude(current: current, max: 0.0)); + _controller?.add(Amplitude(current: current, max: 0)); }); return _controller!.stream; diff --git a/packages/firebase_ai/firebase_ai/lib/firebase_ai.dart b/packages/firebase_ai/firebase_ai/lib/firebase_ai.dart index 440f00bf2f5d..06eeaaa1195f 100644 --- a/packages/firebase_ai/firebase_ai/lib/firebase_ai.dart +++ b/packages/firebase_ai/firebase_ai/lib/firebase_ai.dart @@ -117,8 +117,6 @@ export 'src/live_api.dart' SessionResumptionUpdate, SlidingWindow, Transcription; -export 'src/speech_config.dart' - show SpeechConfig, MultiSpeakerVoiceConfig, SpeakerVoiceConfig; export 'src/live_session.dart' show LiveSession; export 'src/schema.dart' show JSONSchema, Schema, SchemaType; export 'src/server_template/template_chat.dart' @@ -129,6 +127,8 @@ export 'src/server_template/template_tool.dart' TemplateFunctionDeclaration, TemplateTool, TemplateToolConfig; +export 'src/speech_config.dart' + show SpeechConfig, MultiSpeakerVoiceConfig, SpeakerVoiceConfig; export 'src/tool.dart' show AutoFunctionDeclaration, diff --git a/packages/firebase_ai/firebase_ai/lib/src/speech_config.dart b/packages/firebase_ai/firebase_ai/lib/src/speech_config.dart index 21606ff03a83..1196ac05b92a 100644 --- a/packages/firebase_ai/firebase_ai/lib/src/speech_config.dart +++ b/packages/firebase_ai/firebase_ai/lib/src/speech_config.dart @@ -16,6 +16,15 @@ import 'package:meta/meta.dart'; /// Speech configuration class for controlling the model's speech and audio generation behaviors. class SpeechConfig { + /// Constructs a [SpeechConfig] for a single-speaker setup. + SpeechConfig({this.voiceName, this.languageCode}) + : multiSpeakerVoiceConfig = null; + + /// Constructs a [SpeechConfig] for a multi-speaker setup. + SpeechConfig.multiSpeaker( + {required this.multiSpeakerVoiceConfig, this.languageCode}) + : voiceName = null; + /// The voice name to use for a single-speaker setup. final String? voiceName; @@ -25,14 +34,6 @@ class SpeechConfig { /// The optional IETF BCP-47 language code. final String? languageCode; - /// Constructs a [SpeechConfig] for a single-speaker setup. - SpeechConfig({this.voiceName, this.languageCode}) - : multiSpeakerVoiceConfig = null; - - /// Constructs a [SpeechConfig] for a multi-speaker setup. - SpeechConfig.multiSpeaker({required this.multiSpeakerVoiceConfig, this.languageCode}) - : voiceName = null; - /// Convert to json format. @internal Map toJson() => { @@ -49,12 +50,12 @@ class SpeechConfig { /// Configuration for a multi-speaker audio generation setup. class MultiSpeakerVoiceConfig { - /// A list of voice configurations for the participating speakers. - final List speakerVoiceConfigs; - /// Constructor MultiSpeakerVoiceConfig({required this.speakerVoiceConfigs}); + /// A list of voice configurations for the participating speakers. + final List speakerVoiceConfigs; + /// Convert to json format. Map toJson() => { 'speaker_voice_configs': @@ -64,15 +65,15 @@ class MultiSpeakerVoiceConfig { /// Configures a participating speaker within a multi-speaker setup. class SpeakerVoiceConfig { + /// Constructor + SpeakerVoiceConfig({required this.speaker, required this.voiceName}); + /// The unique name/identifier of the speaker. final String speaker; /// The specific voice assigned to this speaker. final String voiceName; - /// Constructor - SpeakerVoiceConfig({required this.speaker, required this.voiceName}); - /// Convert to json format. Map toJson() => { 'speaker': speaker, From fd4f5d0e9d42c478f6677516f2169ac9294550fa Mon Sep 17 00:00:00 2001 From: Cynthia J Date: Tue, 9 Jun 2026 16:54:39 -0700 Subject: [PATCH 4/4] format --- packages/firebase_ai/firebase_ai/lib/src/api.dart | 1 - packages/firebase_ai/firebase_ai/test/live_test.dart | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/firebase_ai/firebase_ai/lib/src/api.dart b/packages/firebase_ai/firebase_ai/lib/src/api.dart index aee9c108159c..d2c2c7c9b3e9 100644 --- a/packages/firebase_ai/firebase_ai/lib/src/api.dart +++ b/packages/firebase_ai/firebase_ai/lib/src/api.dart @@ -19,7 +19,6 @@ import 'schema.dart'; import 'speech_config.dart'; import 'tool.dart' show Tool, ToolConfig; - /// Response for Count Tokens final class CountTokensResponse { // ignore: public_member_api_docs diff --git a/packages/firebase_ai/firebase_ai/test/live_test.dart b/packages/firebase_ai/firebase_ai/test/live_test.dart index e7345ce575ff..9965c59ca46b 100644 --- a/packages/firebase_ai/firebase_ai/test/live_test.dart +++ b/packages/firebase_ai/firebase_ai/test/live_test.dart @@ -118,7 +118,8 @@ void main() { expect(liveGenerationConfigWithoutOptionals.toJson(), {}); }); - test('GenerationConfig with SpeechConfig toJson() returns correct JSON', () { + test('GenerationConfig with SpeechConfig toJson() returns correct JSON', + () { final config = GenerationConfig( speechConfig: SpeechConfig(voiceName: 'Aoede', languageCode: 'en-US'), );