interim results on transformers.js implementation

Author: Roy3838

app/src/utils/whisper/WhisperModelManager.ts

@@ -19,6 +19,7 @@ export class WhisperModelManager {
   private currentConfig: WhisperModelConfig | null = null;
   private stateChangeListeners: Array<(state: WhisperModelState) => void> = [];
   private pendingTranscriptions = new Map<number, PendingTranscription>();
+  private interimResultListeners: Array<(text: string, chunkId: number) => void> = [];
 
   private constructor() {}
 
@@ -40,6 +41,17 @@ export class WhisperModelManager {
     };
   }
 
+  public onInterimResult(listener: (text: string, chunkId: number) => void): () => void {
+    this.interimResultListeners.push(listener);
+    return () => {
+      this.interimResultListeners = this.interimResultListeners.filter(l => l !== listener);
+    };
+  }
+
+  private notifyInterimResult(text: string, chunkId: number): void {
+    this.interimResultListeners.forEach(listener => listener(text, chunkId));
+  }
+
   private setState(updates: Partial<WhisperModelState>): void {
     this.state = { ...this.state, ...updates };
     this.stateChangeListeners.forEach(listener => listener(this.getState()));
@@ -189,6 +201,10 @@ export class WhisperModelManager {
         });
         break;
 
+      case 'transcription-interim':
+        this.handleInterimResult(data);
+        break;
+
       case 'transcription-complete':
         this.handleTranscriptionComplete(data);
         break;
@@ -202,6 +218,12 @@ export class WhisperModelManager {
     }
   }
 
+  private handleInterimResult(data: { text: string; chunkId: number }): void {
+    const { text, chunkId } = data;
+    Logger.debug('WhisperModelManager', `Interim result for chunk ${chunkId}: "${text.slice(0, 50)}..."`);
+    this.notifyInterimResult(text, chunkId);
+  }
+
   private handleProgressMessage(progress: ProgressItem): void {
     this.setState({
       progress: this.updateProgressItems(this.state.progress, progress)

app/src/utils/whisper/WhisperTranscriptionService.ts

@@ -19,6 +19,9 @@ export class WhisperTranscriptionService {
   // Subscriber management - services push to all subscribers
   private subscribers = new Set<TranscriptionSubscriber>();
 
+  // Cleanup function for interim result listener
+  private unsubscribeInterimResults: (() => void) | null = null;
+
   public async start(stream: MediaStream, streamType?: AudioStreamType): Promise<void> {
     if (this.isRunning) {
       Logger.warn('WhisperTranscriptionService', 'Service already running');
@@ -39,6 +42,14 @@ export class WhisperTranscriptionService {
     const settings = SensorSettings.getWhisperSettings();
     this.chunkDurationMs = settings.chunkDurationMs;
 
+    // Subscribe to interim results from the model manager
+    this.unsubscribeInterimResults = modelManager.onInterimResult((text, _chunkId) => {
+      if (this.isRunning) {
+        this.setInterimToSubscribers(text);
+        TranscriptionStateManager.setInterimText(this.streamType, text);
+      }
+    });
+
     Logger.info('WhisperTranscriptionService', `Starting transcription for ${this.streamType} with ${this.chunkDurationMs}ms chunks`);
 
     this.transcribeLoop();
@@ -51,6 +62,12 @@ export class WhisperTranscriptionService {
     this.isRunning = false;
     this.currentStream = null;
 
+    // Unsubscribe from interim results
+    if (this.unsubscribeInterimResults) {
+      this.unsubscribeInterimResults();
+      this.unsubscribeInterimResults = null;
+    }
+
     // Note: We don't clear subscribers here - they are managed by StreamManager
     // and may persist across service restarts
 
@@ -71,11 +88,20 @@ export class WhisperTranscriptionService {
   }
 
   /**
-   * Push transcribed text to all registered subscribers
+   * Set interim text on all registered subscribers (replaces previous interim)
+   */
+  private setInterimToSubscribers(text: string): void {
+    for (const subscriber of this.subscribers) {
+      subscriber.setInterimText(text);
+    }
+  }
+
+  /**
+   * Commit final text to all registered subscribers (clears interim, adds to committed)
    */
-  private pushToSubscribers(text: string): void {
+  private commitToSubscribers(text: string): void {
     for (const subscriber of this.subscribers) {
-      subscriber.appendText(text);
+      subscriber.commitText(text);
     }
   }
 
@@ -121,8 +147,8 @@ export class WhisperTranscriptionService {
         .transcribe(await audioBlob.arrayBuffer(), chunkId);
 
       if (result?.text && this.isRunning) {
-        // Push to all registered subscribers (agent-specific accumulators)
-        this.pushToSubscribers(result.text);
+        // Commit final text to all registered subscribers (clears interim, adds to committed)
+        this.commitToSubscribers(result.text);
 
         // Notify state manager for UI updates (maintains its own rolling window)
         TranscriptionStateManager.chunkTranscriptionEnded(

app/src/utils/whisper/whisper.worker.ts

@@ -2,6 +2,9 @@ import { WhisperModelConfig } from './types';
 
 const TASK = 'automatic-speech-recognition';
 
+// WhisperTextStreamer will be loaded dynamically with transformers module
+let WhisperTextStreamer: any = null;
+
 class WhisperPipelineFactory {
   static task = TASK;
   static model: string | null = null;
@@ -20,6 +23,8 @@ class WhisperPipelineFactory {
       // Dynamic import - only loads when actually needed!
       this.transformersModule = await import('@huggingface/transformers');
       this.transformersModule.env.allowLocalModels = false;
+      // Load WhisperTextStreamer for streaming interim results
+      WhisperTextStreamer = this.transformersModule.WhisperTextStreamer;
     }
     return this.transformersModule;
   }
@@ -79,35 +84,56 @@ self.onmessage = async (event) => {
 
       case 'transcribe':
         const { audio, chunkId } = data;
-        
+
         const instance = await WhisperPipelineFactory.getInstance();
         const currentConfig = WhisperPipelineFactory.config;
-        
+
         if (!instance || !currentConfig) {
           throw new Error('Pipeline not initialized');
         }
 
         // Build transcription options - only for multilingual models
         const isEnglishOnlyModel = currentConfig.modelId.endsWith('.en');
         const transcribeOptions: any = {};
-        
+
         if (!isEnglishOnlyModel) {
           // Only add parameters for multilingual models
           if (currentConfig.task) {
             transcribeOptions.task = currentConfig.task;
           }
-          
+
           if (currentConfig.language && currentConfig.language !== 'auto') {
             transcribeOptions.language = currentConfig.language;
           }
-          
+
           // Set default chunking for different model types
           const isDistilWhisper = currentConfig.modelId.startsWith('distil-whisper/');
           transcribeOptions.chunk_length_s = isDistilWhisper ? 20 : 30;
           transcribeOptions.stride_length_s = isDistilWhisper ? 3 : 5;
         }
 
-        const output = Object.keys(transcribeOptions).length > 0 
+        // Create streamer for interim results if WhisperTextStreamer is available
+        if (WhisperTextStreamer && instance.tokenizer) {
+          let lastInterimText = '';
+          const streamer = new WhisperTextStreamer(instance.tokenizer, {
+            skip_prompt: true,
+            skip_special_tokens: true,
+            callback_function: (text: string) => {
+              // Only send if text has changed to reduce message spam
+              const trimmedText = text.trim();
+              if (trimmedText && trimmedText !== lastInterimText) {
+                lastInterimText = trimmedText;
+                self.postMessage({
+                  type: 'transcription-interim',
+                  data: { text: trimmedText, chunkId }
+                });
+              }
+            }
+          });
+          transcribeOptions.streamer = streamer;
+        }
+
+        const output = Object.keys(transcribeOptions).length > 0
           ? await instance(audio, transcribeOptions)
           : await instance(audio);
         const newText = (output.text as string).trim();