From 5917718f2517b406abcfc8b707e1f9b9639ce4a2 Mon Sep 17 00:00:00 2001 From: Urjit Chakraborty <135136842+urjitc@users.noreply.github.com> Date: Mon, 9 Feb 2026 17:10:08 -0500 Subject: [PATCH 1/9] feat: first iteration of audio recordings --- src/app/api/audio/process/route.ts | 171 ++++++++++ src/components/modals/AudioRecorderDialog.tsx | 310 ++++++++++++++++++ src/components/modals/MoveToDialog.tsx | 4 +- .../workspace-canvas/AudioCardContent.tsx | 164 +++++++++ .../AudioRecordingIndicator.tsx | 94 ++++++ .../workspace-canvas/CardRenderer.tsx | 5 + .../WorkspaceCanvasDropzone.tsx | 71 +++- .../workspace-canvas/WorkspaceCard.tsx | 13 + .../workspace-canvas/WorkspaceContent.tsx | 35 +- .../workspace-canvas/WorkspaceHeader.tsx | 102 +++++- .../workspace-canvas/WorkspaceSection.tsx | 4 + src/hooks/audio/use-audio-recorder.ts | 171 ++++++++++ .../workspace/use-workspace-operations.ts | 4 +- src/lib/ai/workers/workspace-worker.ts | 24 +- .../supabase-attachment-adapter.ts | 2 +- src/lib/stores/audio-recording-store.ts | 187 +++++++++++ .../workspace-state/grid-layout-helpers.ts | 1 + src/lib/workspace-state/item-helpers.ts | 4 +- src/lib/workspace-state/types.ts | 28 +- 19 files changed, 1380 insertions(+), 14 deletions(-) create mode 100644 src/app/api/audio/process/route.ts create mode 100644 src/components/modals/AudioRecorderDialog.tsx create mode 100644 src/components/workspace-canvas/AudioCardContent.tsx create mode 100644 src/components/workspace-canvas/AudioRecordingIndicator.tsx create mode 100644 src/hooks/audio/use-audio-recorder.ts create mode 100644 src/lib/stores/audio-recording-store.ts diff --git a/src/app/api/audio/process/route.ts b/src/app/api/audio/process/route.ts new file mode 100644 index 00000000..30805fc6 --- /dev/null +++ b/src/app/api/audio/process/route.ts @@ -0,0 +1,171 @@ +import { GoogleGenAI, Type } from "@google/genai"; +import { NextRequest, NextResponse } from "next/server"; +import { auth } from "@/lib/auth"; +import { headers } from "next/headers"; + +export const maxDuration = 60; +export const dynamic = "force-dynamic"; + +const apiKey = process.env.GOOGLE_GENERATIVE_AI_API_KEY; + +/** + * POST /api/audio/process + * Receives an audio file URL, downloads it, uploads to Gemini Files API, + * and returns a structured transcript + summary. + */ +export async function POST(req: NextRequest) { + try { + // Auth check + const session = await auth.api.getSession({ + headers: await headers(), + }); + if (!session) { + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + } + + if (!apiKey) { + return NextResponse.json( + { error: "GOOGLE_GENERATIVE_AI_API_KEY is not set" }, + { status: 500 } + ); + } + + const body = await req.json(); + const { fileUrl, filename, mimeType } = body; + + if (!fileUrl) { + return NextResponse.json( + { error: "fileUrl is required" }, + { status: 400 } + ); + } + + // Determine MIME type + const audioMimeType = mimeType || guessMimeType(filename || fileUrl); + + // Download the audio file from storage + const audioResponse = await fetch(fileUrl); + if (!audioResponse.ok) { + return NextResponse.json( + { error: `Failed to download audio: ${audioResponse.statusText}` }, + { status: 500 } + ); + } + + const audioBuffer = await audioResponse.arrayBuffer(); + const base64Audio = Buffer.from(audioBuffer).toString("base64"); + + const client = new GoogleGenAI({ apiKey }); + + const prompt = `Process this audio file and generate a detailed transcription and summary. + +Requirements: +1. Provide a brief summary of the entire audio content. +2. Identify distinct speakers (e.g., Speaker 1, Speaker 2, or names if context allows). +3. Provide accurate timestamps for each segment (Format: MM:SS). +4. Detect the primary language of each segment. +5. If the segment is in a language different than English, also provide the English translation. +6. Identify the primary emotion of the speaker in this segment. You MUST choose exactly one of the following: happy, sad, angry, neutral. +7. Provide the full plain-text transcript combining all segments.`; + + const response = await client.models.generateContent({ + model: "gemini-2.5-flash", + contents: [ + { + role: "user", + parts: [ + { + inlineData: { + mimeType: audioMimeType, + data: base64Audio, + }, + }, + { text: prompt }, + ], + }, + ], + config: { + responseMimeType: "application/json", + responseSchema: { + type: Type.OBJECT, + properties: { + summary: { + type: Type.STRING, + description: "A concise summary of the audio content.", + }, + transcript: { + type: Type.STRING, + description: + "Full plain-text transcript of the audio, combining all segments.", + }, + segments: { + type: Type.ARRAY, + description: + "List of transcribed segments with speaker and timestamp.", + items: { + type: Type.OBJECT, + properties: { + speaker: { type: Type.STRING }, + timestamp: { type: Type.STRING }, + content: { type: Type.STRING }, + language: { type: Type.STRING }, + language_code: { type: Type.STRING }, + translation: { type: Type.STRING }, + emotion: { + type: Type.STRING, + enum: ["happy", "sad", "angry", "neutral"], + }, + }, + required: [ + "speaker", + "timestamp", + "content", + "language", + "language_code", + "emotion", + ], + }, + }, + }, + required: ["summary", "transcript", "segments"], + }, + }, + }); + + const resultText = response.text; + if (!resultText) { + return NextResponse.json( + { error: "No response from Gemini" }, + { status: 500 } + ); + } + + const result = JSON.parse(resultText); + + return NextResponse.json({ + success: true, + summary: result.summary, + transcript: result.transcript, + segments: result.segments, + }); + } catch (error: any) { + console.error("[AUDIO_PROCESS] Error:", error); + return NextResponse.json( + { error: error?.message || "Failed to process audio" }, + { status: 500 } + ); + } +} + +function guessMimeType(filenameOrUrl: string): string { + const lower = filenameOrUrl.toLowerCase(); + if (lower.endsWith(".mp3")) return "audio/mp3"; + if (lower.endsWith(".wav")) return "audio/wav"; + if (lower.endsWith(".ogg")) return "audio/ogg"; + if (lower.endsWith(".aac")) return "audio/aac"; + if (lower.endsWith(".flac")) return "audio/flac"; + if (lower.endsWith(".aiff")) return "audio/aiff"; + if (lower.endsWith(".webm")) return "audio/webm"; + if (lower.endsWith(".m4a")) return "audio/mp4"; + return "audio/mp3"; // Default fallback +} diff --git a/src/components/modals/AudioRecorderDialog.tsx b/src/components/modals/AudioRecorderDialog.tsx new file mode 100644 index 00000000..8cd58109 --- /dev/null +++ b/src/components/modals/AudioRecorderDialog.tsx @@ -0,0 +1,310 @@ +"use client"; + +import { useState, useRef, useCallback, useMemo } from "react"; +import { Mic, Square, Upload, Loader2, Pause, Play } from "lucide-react"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog"; +import { Button } from "@/components/ui/button"; +import { useAudioRecordingStore } from "@/lib/stores/audio-recording-store"; +import { cn } from "@/lib/utils"; + +const ACCEPTED_AUDIO_TYPES = [ + "audio/mp3", + "audio/mpeg", + "audio/wav", + "audio/ogg", + "audio/aac", + "audio/flac", + "audio/aiff", + "audio/webm", + "audio/mp4", + "audio/x-m4a", +]; + +const ACCEPTED_EXTENSIONS = ".mp3,.wav,.ogg,.aac,.flac,.aiff,.webm,.m4a"; + +interface AudioRecorderDialogProps { + open: boolean; + onOpenChange: (open: boolean) => void; + onAudioReady: (file: File) => void; +} + +function formatDuration(seconds: number): string { + const mins = Math.floor(seconds / 60); + const secs = seconds % 60; + return `${mins.toString().padStart(2, "0")}:${secs.toString().padStart(2, "0")}`; +} + +export function AudioRecorderDialog({ + open, + onOpenChange, + onAudioReady, +}: AudioRecorderDialogProps) { + const isRecording = useAudioRecordingStore((s) => s.isRecording); + const isPaused = useAudioRecordingStore((s) => s.isPaused); + const duration = useAudioRecordingStore((s) => s.duration); + const audioBlob = useAudioRecordingStore((s) => s.audioBlob); + const error = useAudioRecordingStore((s) => s.error); + const startRecording = useAudioRecordingStore((s) => s.startRecording); + const stopRecording = useAudioRecordingStore((s) => s.stopRecording); + const pauseRecording = useAudioRecordingStore((s) => s.pauseRecording); + const resumeRecording = useAudioRecordingStore((s) => s.resumeRecording); + const resetRecording = useAudioRecordingStore((s) => s.resetRecording); + + const fileInputRef = useRef(null); + const [isSubmitting, setIsSubmitting] = useState(false); + + // Stable blob URL for audio preview (avoids creating new URLs on every render) + const audioBlobUrl = useMemo(() => { + if (audioBlob) return URL.createObjectURL(audioBlob); + return null; + }, [audioBlob]); + + // Close dialog — if recording, just hide; if idle/done, fully close + const handleClose = useCallback(() => { + if (isRecording) { + // Just hide the dialog — recording continues in the background + onOpenChange(false); + return; + } + // Not recording — fully reset and close + resetRecording(); + setIsSubmitting(false); + onOpenChange(false); + }, [isRecording, resetRecording, onOpenChange]); + + const handleFileSelect = useCallback( + (e: React.ChangeEvent) => { + const file = e.target.files?.[0]; + if (!file) return; + + if ( + !ACCEPTED_AUDIO_TYPES.includes(file.type) && + !file.name.match(/\.(mp3|wav|ogg|aac|flac|aiff|webm|m4a)$/i) + ) { + return; + } + + setIsSubmitting(true); + onAudioReady(file); + resetRecording(); + setIsSubmitting(false); + onOpenChange(false); + }, + [onAudioReady, resetRecording, onOpenChange] + ); + + const handleSubmitRecording = useCallback(() => { + if (!audioBlob) return; + + const mimeType = audioBlob.type; + let ext = "webm"; + if (mimeType.includes("mp4")) ext = "m4a"; + else if (mimeType.includes("ogg")) ext = "ogg"; + else if (mimeType.includes("wav")) ext = "wav"; + + const filename = `recording-${Date.now()}.${ext}`; + const file = new File([audioBlob], filename, { type: mimeType }); + + setIsSubmitting(true); + onAudioReady(file); + resetRecording(); + setIsSubmitting(false); + onOpenChange(false); + }, [audioBlob, onAudioReady, resetRecording, onOpenChange]); + + return ( + + e.stopPropagation()}> + + Audio + + {isRecording + ? "Recording in progress. You can close this dialog — recording continues in the background." + : "Record audio or upload an existing audio file."} + + + +
+ {/* Recording Visualizer */} +
+ {isRecording && !isPaused && ( +
+ )} + + {isRecording ? ( +
+ + + {formatDuration(duration)} + +
+ ) : audioBlob ? ( +
+ + + {formatDuration(duration)} + +
+ ) : ( + + )} +
+ + {/* Controls */} +
+ {!isRecording && !audioBlob && ( + + )} + + {isRecording && ( + <> + + + + )} + + {audioBlob && !isRecording && ( + <> + + + + )} +
+ + {/* Preview audio playback */} + {audioBlobUrl && !isRecording && ( +
+ ); +} + +export default AudioRecorderDialog; diff --git a/src/components/modals/MoveToDialog.tsx b/src/components/modals/MoveToDialog.tsx index 67b9ca0f..b8259a1c 100644 --- a/src/components/modals/MoveToDialog.tsx +++ b/src/components/modals/MoveToDialog.tsx @@ -1,7 +1,7 @@ "use client"; import { useState, useMemo, useCallback, useEffect } from "react"; -import { ChevronRight, Folder as FolderIcon, FolderOpen, Check, FileText, File, Play, Brain, ImageIcon } from "lucide-react"; +import { ChevronRight, Folder as FolderIcon, FolderOpen, Check, FileText, File, Play, Brain, ImageIcon, Mic } from "lucide-react"; import { PiCardsThreeBold } from "react-icons/pi"; import { Dialog, @@ -36,6 +36,8 @@ function getCardTypeIcon(type: CardType) { return ; case "image": return ; + case "audio": + return ; default: return ; } diff --git a/src/components/workspace-canvas/AudioCardContent.tsx b/src/components/workspace-canvas/AudioCardContent.tsx new file mode 100644 index 00000000..3ad659f4 --- /dev/null +++ b/src/components/workspace-canvas/AudioCardContent.tsx @@ -0,0 +1,164 @@ +"use client"; + +import { useState } from "react"; +import { Mic, ChevronDown, ChevronUp, AlertCircle, Loader2, Clock, User, Globe } from "lucide-react"; +import type { Item, AudioData, AudioSegment } from "@/lib/workspace-state/types"; +import { cn } from "@/lib/utils"; + +interface AudioCardContentProps { + item: Item; + isCompact?: boolean; +} + +export function AudioCardContent({ item, isCompact = false }: AudioCardContentProps) { + const audioData = item.data as AudioData; + const [showTranscript, setShowTranscript] = useState(false); + const [showSegments, setShowSegments] = useState(false); + + // Uploading state + if (audioData.processingStatus === "uploading") { + return ( +
+ +

Uploading audio...

+
+ ); + } + + // Processing state + if (audioData.processingStatus === "processing") { + return ( +
+
+ + +
+
+

Analyzing audio...

+

+ Generating transcript and summary with Gemini +

+
+
+ ); + } + + // Failed state + if (audioData.processingStatus === "failed") { + return ( +
+ +
+

Processing failed

+

+ {audioData.error || "An error occurred while processing the audio."} +

+
+
+ ); + } + + // Complete state + return ( +
+ {/* Summary */} + {audioData.summary && ( +
+

+ Summary +

+

+ {audioData.summary} +

+
+ )} + + {/* Full Transcript Toggle */} + {audioData.transcript && ( +
+ + {showTranscript && ( +
+ {audioData.transcript} +
+ )} +
+ )} + + {/* Segments Toggle */} + {audioData.segments && audioData.segments.length > 0 && ( +
+ + {showSegments && ( +
+ {audioData.segments.map((segment, idx) => ( + + ))} +
+ )} +
+ )} +
+ ); +} + +function SegmentRow({ segment }: { segment: AudioSegment }) { + return ( +
+
+ + + {segment.timestamp} + + + + {segment.speaker} + + {segment.language && segment.language !== "English" && ( + + + {segment.language} + + )} + {segment.emotion && segment.emotion !== "neutral" && ( + + {segment.emotion} + + )} +
+

{segment.content}

+ {segment.translation && ( +

+ Translation: {segment.translation} +

+ )} +
+ ); +} + +export default AudioCardContent; diff --git a/src/components/workspace-canvas/AudioRecordingIndicator.tsx b/src/components/workspace-canvas/AudioRecordingIndicator.tsx new file mode 100644 index 00000000..6b646667 --- /dev/null +++ b/src/components/workspace-canvas/AudioRecordingIndicator.tsx @@ -0,0 +1,94 @@ +"use client"; + +import { Mic, Square, Pause, Play } from "lucide-react"; +import { useAudioRecordingStore } from "@/lib/stores/audio-recording-store"; +import { cn } from "@/lib/utils"; + +function formatDuration(seconds: number): string { + const mins = Math.floor(seconds / 60); + const secs = seconds % 60; + return `${mins.toString().padStart(2, "0")}:${secs.toString().padStart(2, "0")}`; +} + +/** + * Floating pill that appears at the bottom of the screen when audio is recording + * and the dialog is closed. Lets the user see the timer, pause/resume, stop, + * or re-open the dialog. + */ +export function AudioRecordingIndicator() { + const isRecording = useAudioRecordingStore((s) => s.isRecording); + const isPaused = useAudioRecordingStore((s) => s.isPaused); + const duration = useAudioRecordingStore((s) => s.duration); + const isDialogOpen = useAudioRecordingStore((s) => s.isDialogOpen); + const stopRecording = useAudioRecordingStore((s) => s.stopRecording); + const pauseRecording = useAudioRecordingStore((s) => s.pauseRecording); + const resumeRecording = useAudioRecordingStore((s) => s.resumeRecording); + const openDialog = useAudioRecordingStore((s) => s.openDialog); + + // Only show when recording is active AND the dialog is closed + if (!isRecording || isDialogOpen) return null; + + return ( +
+
+ {/* Pulsing dot */} + + {!isPaused && ( + + )} + + + + {/* Timer */} + + {formatDuration(duration)} + + + {/* Pause / Resume */} + + + {/* Stop */} + + + {/* Expand to dialog */} + +
+
+ ); +} diff --git a/src/components/workspace-canvas/CardRenderer.tsx b/src/components/workspace-canvas/CardRenderer.tsx index 5328a5c3..a1ec50f5 100644 --- a/src/components/workspace-canvas/CardRenderer.tsx +++ b/src/components/workspace-canvas/CardRenderer.tsx @@ -9,6 +9,7 @@ import FlashcardContent from "./FlashcardContent"; import YouTubeCardContent from "./YouTubeCardContent"; import { SourcesDisplay } from "./SourcesDisplay"; import ImageCardContent from "./ImageCardContent"; +import { AudioCardContent } from "./AudioCardContent"; import { QuizContent } from "./QuizContent"; @@ -114,6 +115,10 @@ export function CardRenderer(props: { return ; } + if (item.type === "audio") { + return ; + } + return (

diff --git a/src/components/workspace-canvas/WorkspaceCanvasDropzone.tsx b/src/components/workspace-canvas/WorkspaceCanvasDropzone.tsx index 24af2b8b..723820bb 100644 --- a/src/components/workspace-canvas/WorkspaceCanvasDropzone.tsx +++ b/src/components/workspace-canvas/WorkspaceCanvasDropzone.tsx @@ -7,7 +7,7 @@ import { useWorkspaceOperations } from "@/hooks/workspace/use-workspace-operatio import { FileText } from "lucide-react"; import { useCallback, useState, useRef, useEffect } from "react"; import { toast } from "sonner"; -import type { PdfData, ImageData } from "@/lib/workspace-state/types"; +import type { PdfData, ImageData, AudioData } from "@/lib/workspace-state/types"; import { getBestFrameForRatio, type GridFrame } from "@/lib/workspace-state/aspect-ratios"; import { useReactiveNavigation } from "@/hooks/ui/use-reactive-navigation"; import { uploadFileDirect } from "@/lib/uploads/client-upload"; @@ -170,11 +170,14 @@ export function WorkspaceCanvasDropzone({ children }: WorkspaceCanvasDropzonePro // Separate files by type const pdfResults: typeof validResults = []; const imageResults: typeof validResults = []; + const audioResults: typeof validResults = []; validResults.forEach((result, index) => { const file = filteredFiles[index]; if (file.type === 'application/pdf') { pdfResults.push(result); + } else if (file.type.startsWith('audio/')) { + audioResults.push(result); } else { imageResults.push(result); } @@ -294,6 +297,60 @@ export function WorkspaceCanvasDropzone({ children }: WorkspaceCanvasDropzonePro handleCreatedItems(imageCreatedIds); } + // Create audio cards and trigger Gemini processing + if (audioResults.length > 0) { + const audioCardDefinitions = audioResults.map((result, index) => { + const file = filteredFiles.find(f => f.name === result.filename) || filteredFiles[validResults.indexOf(result)]; + const audioData: Partial = { + fileUrl: result.fileUrl, + filename: result.filename, + fileSize: result.fileSize, + mimeType: file?.type || 'audio/mpeg', + processingStatus: 'processing', + }; + return { + type: 'audio' as const, + name: result.name.replace(/\.[^/.]+$/, ''), + initialData: audioData, + }; + }); + + const audioCreatedIds = operations.createItems(audioCardDefinitions); + handleCreatedItems(audioCreatedIds); + + // Trigger Gemini processing for each audio file + audioResults.forEach((result, index) => { + const itemId = audioCreatedIds[index]; + const file = filteredFiles.find(f => f.name === result.filename) || filteredFiles[validResults.indexOf(result)]; + fetch('/api/audio/process', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + fileUrl: result.fileUrl, + filename: result.filename, + mimeType: file?.type || 'audio/mpeg', + }), + }) + .then(res => res.json()) + .then(data => { + window.dispatchEvent( + new CustomEvent('audio-processing-complete', { + detail: data.success + ? { itemId, summary: data.summary, transcript: data.transcript, segments: data.segments } + : { itemId, error: data.error || 'Processing failed' }, + }) + ); + }) + .catch(err => { + window.dispatchEvent( + new CustomEvent('audio-processing-complete', { + detail: { itemId, error: err.message || 'Processing failed' }, + }) + ); + }); + }); + } + // Show success toast const totalCreated = validResults.length; toast.success( @@ -345,6 +402,14 @@ export function WorkspaceCanvasDropzone({ children }: WorkspaceCanvasDropzonePro 'image/jpeg': ['.jpg', '.jpeg'], 'image/gif': ['.gif'], 'image/webp': ['.webp'], + 'audio/mpeg': ['.mp3'], + 'audio/wav': ['.wav'], + 'audio/ogg': ['.ogg'], + 'audio/aac': ['.aac'], + 'audio/flac': ['.flac'], + 'audio/aiff': ['.aiff'], + 'audio/webm': ['.webm'], + 'audio/mp4': ['.m4a'], }, onDragEnter: () => setIsDragging(true), onDragLeave: () => { @@ -362,7 +427,7 @@ export function WorkspaceCanvasDropzone({ children }: WorkspaceCanvasDropzonePro if (fileRejections.length > 0) { const rejectedFileNames = fileRejections.map(rejection => rejection.file.name); toast.error( - `Only PDF and image files (PNG, JPG, GIF, WebP) can be dropped.\nRejected: ${rejectedFileNames.join(', ')}`, + `Only PDF, image, and audio files can be dropped.\nRejected: ${rejectedFileNames.join(', ')}`, { style: { color: '#fff' }, duration: 5000, @@ -388,7 +453,7 @@ export function WorkspaceCanvasDropzone({ children }: WorkspaceCanvasDropzonePro Create Card

- Drop PDF or image files here to create cards + Drop PDF, image, or audio files here to create cards

diff --git a/src/components/workspace-canvas/WorkspaceCard.tsx b/src/components/workspace-canvas/WorkspaceCard.tsx index 410ec344..0aa595c1 100644 --- a/src/components/workspace-canvas/WorkspaceCard.tsx +++ b/src/components/workspace-canvas/WorkspaceCard.tsx @@ -15,6 +15,7 @@ import { plainTextToBlocks, type Block } from "@/components/editor/BlockNoteEdit import { serializeBlockNote } from "@/lib/utils/serialize-blocknote"; import { BlockNotePreview } from "@/components/editor/BlockNotePreview"; import { DeepResearchCardContent } from "./DeepResearchCardContent"; +import { AudioCardContent } from "./AudioCardContent"; import LazyAppPdfViewer from "@/components/pdf/LazyAppPdfViewer"; import { LightweightPdfPreview } from "@/components/pdf/LightweightPdfPreview"; @@ -983,6 +984,13 @@ function WorkspaceCard({ )} + {/* Audio Content - render audio player and transcript */} + {!isOpenInPanel && item.type === 'audio' && shouldShowPreview && ( +
+ +
+ )} + {/* Deep Research Note - render streaming UI when research is in progress */} {item.type === 'note' && (() => { const noteData = item.data as NoteData; @@ -1137,6 +1145,11 @@ export const WorkspaceCardMemoized = memo(WorkspaceCard, (prevProps, nextProps) const nextData = nextProps.item.data; if (JSON.stringify(prevData) !== JSON.stringify(nextData)) return false; } + if (prevProps.item.type === 'audio' && nextProps.item.type === 'audio') { + const prevData = prevProps.item.data; + const nextData = nextProps.item.data; + if (JSON.stringify(prevData) !== JSON.stringify(nextData)) return false; + } // Compare layout (use lg breakpoint for comparison) const prevLayout = getLayoutForBreakpoint(prevProps.item, 'lg'); diff --git a/src/components/workspace-canvas/WorkspaceContent.tsx b/src/components/workspace-canvas/WorkspaceContent.tsx index 5c4a182f..7322f082 100644 --- a/src/components/workspace-canvas/WorkspaceContent.tsx +++ b/src/components/workspace-canvas/WorkspaceContent.tsx @@ -1,5 +1,5 @@ import ShikiHighlighter from "react-shiki/web"; -import { useMemo, useCallback, useRef, useState } from "react"; +import { useMemo, useCallback, useRef, useState, useEffect } from "react"; import { Plus, Copy, Check, Download, Upload } from "lucide-react"; import { EmptyState } from "@/components/empty-state"; import type { AgentState, Item, CardType } from "@/lib/workspace-state/types"; @@ -157,6 +157,39 @@ export default function WorkspaceContent({ onOpenFolder?.(folderId); }, [setActiveFolderId, onOpenFolder]); + // Listen for audio processing completion events + useEffect(() => { + const handleAudioComplete = (e: Event) => { + const { itemId, summary, transcript, segments, error } = (e as CustomEvent).detail; + if (!itemId) return; + + if (error) { + updateItem(itemId, { + data: { + ...viewState.items.find((i) => i.id === itemId)?.data, + processingStatus: "failed", + error, + } as any, + }); + } else { + updateItem(itemId, { + data: { + ...viewState.items.find((i) => i.id === itemId)?.data, + summary, + transcript, + segments, + processingStatus: "complete", + } as any, + }); + } + }; + + window.addEventListener("audio-processing-complete", handleAudioComplete); + return () => { + window.removeEventListener("audio-processing-complete", handleAudioComplete); + }; + }, [updateItem, viewState.items]); + // OPTIMIZED: Wrap callbacks to ensure stable references const handleUpdateItem = useCallback((itemId: string, updates: Partial) => { updateItem(itemId, updates); diff --git a/src/components/workspace-canvas/WorkspaceHeader.tsx b/src/components/workspace-canvas/WorkspaceHeader.tsx index 7dd0ffbe..982e6b87 100644 --- a/src/components/workspace-canvas/WorkspaceHeader.tsx +++ b/src/components/workspace-canvas/WorkspaceHeader.tsx @@ -5,7 +5,7 @@ import { useState, useRef, useEffect, useCallback } from "react"; import Image from "next/image"; import Link from "next/link"; import { usePathname } from "next/navigation"; -import { Search, X, ChevronRight, ChevronDown, FolderOpen, Plus, Upload, FileText, Folder as FolderIcon, Settings, Share2, Play, MoreHorizontal, Globe, Brain, Maximize, File, Newspaper, ImageIcon } from "lucide-react"; +import { Search, X, ChevronRight, ChevronDown, FolderOpen, Plus, Upload, FileText, Folder as FolderIcon, Settings, Share2, Play, MoreHorizontal, Globe, Brain, Maximize, File, Newspaper, ImageIcon, Mic } from "lucide-react"; import { LuBook, LuPanelLeftOpen } from "react-icons/lu"; import { PiCardsThreeBold } from "react-icons/pi"; import { cn } from "@/lib/utils"; @@ -52,6 +52,8 @@ import { CreateWebsiteDialog } from "@/components/modals/CreateWebsiteDialog"; import { useQueryClient } from "@tanstack/react-query"; import { CollaboratorAvatars } from "@/components/workspace/CollaboratorAvatars"; import { UploadDialog } from "@/components/modals/UploadDialog"; +import { AudioRecorderDialog } from "@/components/modals/AudioRecorderDialog"; +import { useAudioRecordingStore } from "@/lib/stores/audio-recording-store"; import { getBestFrameForRatio } from "@/lib/workspace-state/aspect-ratios"; interface WorkspaceHeaderProps { titleInputRef: React.RefObject; @@ -145,6 +147,9 @@ export default function WorkspaceHeader({ const [showYouTubeDialog, setShowYouTubeDialog] = useState(false); const [showWebsiteDialog, setShowWebsiteDialog] = useState(false); const [showUploadDialog, setShowUploadDialog] = useState(false); + const showAudioDialog = useAudioRecordingStore((s) => s.isDialogOpen); + const openAudioDialog = useAudioRecordingStore((s) => s.openDialog); + const closeAudioDialog = useAudioRecordingStore((s) => s.closeDialog); const renameInputRef = useRef(null); const searchInputRef = useRef(null); const pathname = usePathname(); @@ -324,6 +329,85 @@ export default function WorkspaceHeader({ setIsNewMenuOpen(false); }, [addItem]); + const handleAudioReady = useCallback(async (file: File) => { + if (!addItem) return; + + // Import uploadFileDirect dynamically to avoid top-level client import issues + const { uploadFileDirect } = await import("@/lib/uploads/client-upload"); + + const loadingToastId = toast.loading("Uploading audio..."); + + try { + // Upload the audio file to storage + const { url: fileUrl } = await uploadFileDirect(file); + + // Create the audio card immediately (shows "processing" state) + const itemId = addItem("audio", file.name.replace(/\.[^/.]+$/, ""), { + fileUrl, + filename: file.name, + fileSize: file.size, + mimeType: file.type || "audio/webm", + processingStatus: "processing", + } as any); + + if (onItemCreated && itemId) { + onItemCreated([itemId]); + } + + toast.dismiss(loadingToastId); + toast.success("Audio uploaded — analyzing with Gemini..."); + + // Kick off Gemini processing in the background + fetch("/api/audio/process", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + fileUrl, + filename: file.name, + mimeType: file.type || "audio/webm", + }), + }) + .then((res) => res.json()) + .then((result) => { + if (result.success) { + // Dispatch a custom event to update the audio card data + window.dispatchEvent( + new CustomEvent("audio-processing-complete", { + detail: { + itemId, + summary: result.summary, + transcript: result.transcript, + segments: result.segments, + }, + }) + ); + } else { + window.dispatchEvent( + new CustomEvent("audio-processing-complete", { + detail: { + itemId, + error: result.error || "Processing failed", + }, + }) + ); + } + }) + .catch((err) => { + window.dispatchEvent( + new CustomEvent("audio-processing-complete", { + detail: { + itemId, + error: err.message || "Processing failed", + }, + }) + ); + }); + } catch (error: any) { + toast.dismiss(loadingToastId); + toast.error(error.message || "Failed to upload audio"); + } + }, [addItem, onItemCreated]); + const handleImageCreate = useCallback(async (url: string, name: string) => { if (!addItem) return; @@ -946,6 +1030,16 @@ export default function WorkspaceHeader({ + { + openAudioDialog(); + setIsNewMenuOpen(false); + }} + className="flex items-center gap-2 cursor-pointer" + > + + Audio + { setShowYouTubeDialog(true); @@ -1063,6 +1157,12 @@ export default function WorkspaceHeader({ onPDFUpload={onPDFUpload} /> )} + {/* Audio Recorder Dialog */} + { if (open) openAudioDialog(); else closeAudioDialog(); }} + onAudioReady={handleAudioReady} + /> ); } diff --git a/src/components/workspace-canvas/WorkspaceSection.tsx b/src/components/workspace-canvas/WorkspaceSection.tsx index 728e6bf4..f6d261c6 100644 --- a/src/components/workspace-canvas/WorkspaceSection.tsx +++ b/src/components/workspace-canvas/WorkspaceSection.tsx @@ -48,6 +48,7 @@ import type { WorkspaceWithState } from "@/lib/workspace-state/types"; import { useAui } from "@assistant-ui/react"; import { focusComposerInput } from "@/lib/utils/composer-utils"; import { UploadDialog } from "@/components/modals/UploadDialog"; +import { AudioRecordingIndicator } from "./AudioRecordingIndicator"; import { getBestFrameForRatio } from "@/lib/workspace-state/aspect-ratios"; import { useReactiveNavigation } from "@/hooks/ui/use-reactive-navigation"; @@ -709,6 +710,9 @@ export function WorkspaceSection({ onPDFUpload={handlePDFUpload} /> )} + + {/* Floating recording indicator (visible when dialog is closed but recording is active) */} + ); } diff --git a/src/hooks/audio/use-audio-recorder.ts b/src/hooks/audio/use-audio-recorder.ts new file mode 100644 index 00000000..be2256d1 --- /dev/null +++ b/src/hooks/audio/use-audio-recorder.ts @@ -0,0 +1,171 @@ +"use client"; + +import { useState, useRef, useCallback, useEffect } from "react"; + +export interface UseAudioRecorderReturn { + isRecording: boolean; + isPaused: boolean; + duration: number; // seconds + audioBlob: Blob | null; + startRecording: () => Promise; + stopRecording: () => void; + pauseRecording: () => void; + resumeRecording: () => void; + resetRecording: () => void; + error: string | null; +} + +export function useAudioRecorder(): UseAudioRecorderReturn { + const [isRecording, setIsRecording] = useState(false); + const [isPaused, setIsPaused] = useState(false); + const [duration, setDuration] = useState(0); + const [audioBlob, setAudioBlob] = useState(null); + const [error, setError] = useState(null); + + const mediaRecorderRef = useRef(null); + const chunksRef = useRef([]); + const timerRef = useRef | null>(null); + const streamRef = useRef(null); + + const clearTimer = useCallback(() => { + if (timerRef.current) { + clearInterval(timerRef.current); + timerRef.current = null; + } + }, []); + + const startTimer = useCallback(() => { + clearTimer(); + timerRef.current = setInterval(() => { + setDuration((prev) => prev + 1); + }, 1000); + }, [clearTimer]); + + const getSupportedMimeType = (): string => { + const types = [ + "audio/webm;codecs=opus", + "audio/webm", + "audio/mp4", + "audio/ogg;codecs=opus", + "audio/ogg", + ]; + for (const type of types) { + if (MediaRecorder.isTypeSupported(type)) { + return type; + } + } + return "audio/webm"; // Fallback + }; + + const startRecording = useCallback(async () => { + try { + setError(null); + setAudioBlob(null); + setDuration(0); + chunksRef.current = []; + + const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); + streamRef.current = stream; + + const mimeType = getSupportedMimeType(); + const recorder = new MediaRecorder(stream, { mimeType }); + mediaRecorderRef.current = recorder; + + recorder.ondataavailable = (e) => { + if (e.data.size > 0) { + chunksRef.current.push(e.data); + } + }; + + recorder.onstop = () => { + const blob = new Blob(chunksRef.current, { type: mimeType }); + setAudioBlob(blob); + clearTimer(); + + // Stop all tracks + if (streamRef.current) { + streamRef.current.getTracks().forEach((track) => track.stop()); + streamRef.current = null; + } + }; + + recorder.onerror = () => { + setError("Recording failed. Please try again."); + setIsRecording(false); + clearTimer(); + }; + + recorder.start(1000); // Collect data every second + setIsRecording(true); + setIsPaused(false); + startTimer(); + } catch (err: any) { + if (err.name === "NotAllowedError") { + setError("Microphone access denied. Please allow microphone access."); + } else if (err.name === "NotFoundError") { + setError("No microphone found. Please connect a microphone."); + } else { + setError(err.message || "Failed to start recording."); + } + } + }, [startTimer, clearTimer]); + + const stopRecording = useCallback(() => { + if (mediaRecorderRef.current && mediaRecorderRef.current.state !== "inactive") { + mediaRecorderRef.current.stop(); + setIsRecording(false); + setIsPaused(false); + clearTimer(); + } + }, [clearTimer]); + + const pauseRecording = useCallback(() => { + if (mediaRecorderRef.current && mediaRecorderRef.current.state === "recording") { + mediaRecorderRef.current.pause(); + setIsPaused(true); + clearTimer(); + } + }, [clearTimer]); + + const resumeRecording = useCallback(() => { + if (mediaRecorderRef.current && mediaRecorderRef.current.state === "paused") { + mediaRecorderRef.current.resume(); + setIsPaused(false); + startTimer(); + } + }, [startTimer]); + + const resetRecording = useCallback(() => { + stopRecording(); + setAudioBlob(null); + setDuration(0); + setError(null); + chunksRef.current = []; + }, [stopRecording]); + + // Cleanup on unmount + useEffect(() => { + return () => { + clearTimer(); + if (mediaRecorderRef.current && mediaRecorderRef.current.state !== "inactive") { + mediaRecorderRef.current.stop(); + } + if (streamRef.current) { + streamRef.current.getTracks().forEach((track) => track.stop()); + } + }; + }, [clearTimer]); + + return { + isRecording, + isPaused, + duration, + audioBlob, + startRecording, + stopRecording, + pauseRecording, + resumeRecording, + resetRecording, + error, + }; +} diff --git a/src/hooks/workspace/use-workspace-operations.ts b/src/hooks/workspace/use-workspace-operations.ts index eb314dfc..319ef5e4 100644 --- a/src/hooks/workspace/use-workspace-operations.ts +++ b/src/hooks/workspace/use-workspace-operations.ts @@ -93,7 +93,7 @@ export function useWorkspaceOperations( // Validate type is a valid CardType logger.debug("🔧 [CREATE-ITEM] Received type:", type, "typeof:", typeof type, "value:", JSON.stringify(type)); - const validTypes: CardType[] = ["note", "pdf", "flashcard", "folder", "youtube", "image"]; + const validTypes: CardType[] = ["note", "pdf", "flashcard", "folder", "youtube", "image", "audio"]; const validType = validTypes.includes(type) ? type : "note"; if (validType !== type) { @@ -160,7 +160,7 @@ export function useWorkspaceOperations( // Create all items const createdItems: Item[] = items.map(({ type, name, initialData, initialLayout }) => { // Validate type is a valid CardType - const validTypes: CardType[] = ["note", "pdf", "flashcard", "folder", "youtube", "image"]; + const validTypes: CardType[] = ["note", "pdf", "flashcard", "folder", "youtube", "image", "audio"]; const validType = validTypes.includes(type) ? type : "note"; if (validType !== type) { diff --git a/src/lib/ai/workers/workspace-worker.ts b/src/lib/ai/workers/workspace-worker.ts index afa8f947..82d85cca 100644 --- a/src/lib/ai/workers/workspace-worker.ts +++ b/src/lib/ai/workers/workspace-worker.ts @@ -74,7 +74,7 @@ export async function workspaceWorker( content?: string; // For notes itemId?: string; - itemType?: "note" | "flashcard" | "quiz" | "youtube" | "image"; // Defaults to "note" if undefined + itemType?: "note" | "flashcard" | "quiz" | "youtube" | "image" | "audio"; // Defaults to "note" if undefined pdfTextContent?: string; // For caching extracted PDF text content flashcardData?: { cards?: { front: string; back: string }[]; // For creating flashcards @@ -90,6 +90,13 @@ export async function workspaceWorker( altText?: string; caption?: string; }; + audioData?: { + fileUrl: string; + filename: string; + fileSize?: number; + mimeType?: string; + duration?: number; + }; // Optional: deep research metadata to attach to a note deepResearchData?: { prompt: string; @@ -222,6 +229,19 @@ export async function workspaceWorker( altText: params.imageData.altText, caption: params.imageData.caption }; + } else if (itemType === "audio") { + // Audio type + if (!params.audioData || !params.audioData.fileUrl) { + throw new Error("Audio data required for audio card creation"); + } + itemData = { + fileUrl: params.audioData.fileUrl, + filename: params.audioData.filename || "Recording", + fileSize: params.audioData.fileSize, + mimeType: params.audioData.mimeType, + duration: params.audioData.duration, + processingStatus: "processing", + }; } else if (itemType === "quiz") { // Quiz type if (!params.quizData) { @@ -270,7 +290,7 @@ export async function workspaceWorker( const item: Item = { id: itemId, type: itemType, - name: params.title || (itemType === "youtube" ? "YouTube Video" : itemType === "image" ? "Image" : itemType === "quiz" ? "New Quiz" : itemType === "flashcard" ? "New Flashcard Deck" : "New Note"), + name: params.title || (itemType === "youtube" ? "YouTube Video" : itemType === "image" ? "Image" : itemType === "quiz" ? "New Quiz" : itemType === "flashcard" ? "New Flashcard Deck" : itemType === "audio" ? "Audio Recording" : "New Note"), subtitle: "", data: itemData, color: getRandomCardColor(), diff --git a/src/lib/attachments/supabase-attachment-adapter.ts b/src/lib/attachments/supabase-attachment-adapter.ts index 86194dd4..fbe3367a 100644 --- a/src/lib/attachments/supabase-attachment-adapter.ts +++ b/src/lib/attachments/supabase-attachment-adapter.ts @@ -18,7 +18,7 @@ const MAX_FILE_SIZE_BYTES = 50 * 1024 * 1024; // 50MB to match server limit */ export class SupabaseAttachmentAdapter implements AttachmentAdapter { accept = - "image/*,video/*,.pdf,.doc,.docx,.ppt,.pptx,.xls,.xlsx,.txt,.md,.csv,.json"; + "image/*,video/*,audio/*,.pdf,.doc,.docx,.ppt,.pptx,.xls,.xlsx,.txt,.md,.csv,.json,.mp3,.wav,.ogg,.aac,.flac,.aiff,.webm,.m4a"; // Map of attachment ID → upload promise (started eagerly in add()) private pendingUploads = new Map>(); diff --git a/src/lib/stores/audio-recording-store.ts b/src/lib/stores/audio-recording-store.ts new file mode 100644 index 00000000..35fd996b --- /dev/null +++ b/src/lib/stores/audio-recording-store.ts @@ -0,0 +1,187 @@ +import { create } from "zustand"; + +export interface AudioRecordingState { + // Recording state + isRecording: boolean; + isPaused: boolean; + duration: number; // seconds + audioBlob: Blob | null; + error: string | null; + + // Internals (not reactive, but stored for access) + _mediaRecorder: MediaRecorder | null; + _stream: MediaStream | null; + _chunks: Blob[]; + _timerId: ReturnType | null; + + // Dialog visibility (recording continues regardless) + isDialogOpen: boolean; + + // Actions + startRecording: () => Promise; + stopRecording: () => void; + pauseRecording: () => void; + resumeRecording: () => void; + resetRecording: () => void; + openDialog: () => void; + closeDialog: () => void; + setDuration: (d: number) => void; +} + +function getSupportedMimeType(): string { + const types = [ + "audio/webm;codecs=opus", + "audio/webm", + "audio/mp4", + "audio/ogg;codecs=opus", + "audio/ogg", + ]; + for (const type of types) { + if (typeof MediaRecorder !== "undefined" && MediaRecorder.isTypeSupported(type)) { + return type; + } + } + return "audio/webm"; +} + +export const useAudioRecordingStore = create((set, get) => ({ + isRecording: false, + isPaused: false, + duration: 0, + audioBlob: null, + error: null, + _mediaRecorder: null, + _stream: null, + _chunks: [], + _timerId: null, + isDialogOpen: false, + + openDialog: () => set({ isDialogOpen: true }), + closeDialog: () => set({ isDialogOpen: false }), + + setDuration: (d) => set({ duration: d }), + + startRecording: async () => { + const state = get(); + // Clean up any previous recording + if (state._timerId) clearInterval(state._timerId); + if (state._mediaRecorder && state._mediaRecorder.state !== "inactive") { + state._mediaRecorder.stop(); + } + if (state._stream) { + state._stream.getTracks().forEach((t) => t.stop()); + } + + try { + const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); + const mimeType = getSupportedMimeType(); + const recorder = new MediaRecorder(stream, { mimeType }); + const chunks: Blob[] = []; + + recorder.ondataavailable = (e) => { + if (e.data.size > 0) chunks.push(e.data); + }; + + recorder.onstop = () => { + const blob = new Blob(chunks, { type: mimeType }); + const s = get(); + if (s._timerId) clearInterval(s._timerId); + if (s._stream) { + s._stream.getTracks().forEach((t) => t.stop()); + } + set({ + audioBlob: blob, + isRecording: false, + isPaused: false, + _mediaRecorder: null, + _stream: null, + _chunks: [], + _timerId: null, + // Auto-open dialog when recording stops so user can review + isDialogOpen: true, + }); + }; + + recorder.onerror = () => { + set({ error: "Recording failed. Please try again.", isRecording: false }); + const s = get(); + if (s._timerId) clearInterval(s._timerId); + }; + + recorder.start(1000); + + const timerId = setInterval(() => { + set((s) => ({ duration: s.duration + 1 })); + }, 1000); + + set({ + isRecording: true, + isPaused: false, + duration: 0, + audioBlob: null, + error: null, + _mediaRecorder: recorder, + _stream: stream, + _chunks: chunks, + _timerId: timerId, + }); + } catch (err: any) { + if (err.name === "NotAllowedError") { + set({ error: "Microphone access denied. Please allow microphone access." }); + } else if (err.name === "NotFoundError") { + set({ error: "No microphone found. Please connect a microphone." }); + } else { + set({ error: err.message || "Failed to start recording." }); + } + } + }, + + stopRecording: () => { + const state = get(); + if (state._mediaRecorder && state._mediaRecorder.state !== "inactive") { + state._mediaRecorder.stop(); // triggers onstop which updates state + } + }, + + pauseRecording: () => { + const state = get(); + if (state._mediaRecorder && state._mediaRecorder.state === "recording") { + state._mediaRecorder.pause(); + if (state._timerId) clearInterval(state._timerId); + set({ isPaused: true, _timerId: null }); + } + }, + + resumeRecording: () => { + const state = get(); + if (state._mediaRecorder && state._mediaRecorder.state === "paused") { + state._mediaRecorder.resume(); + const timerId = setInterval(() => { + set((s) => ({ duration: s.duration + 1 })); + }, 1000); + set({ isPaused: false, _timerId: timerId }); + } + }, + + resetRecording: () => { + const state = get(); + if (state._timerId) clearInterval(state._timerId); + if (state._mediaRecorder && state._mediaRecorder.state !== "inactive") { + state._mediaRecorder.stop(); + } + if (state._stream) { + state._stream.getTracks().forEach((t) => t.stop()); + } + set({ + isRecording: false, + isPaused: false, + duration: 0, + audioBlob: null, + error: null, + _mediaRecorder: null, + _stream: null, + _chunks: [], + _timerId: null, + }); + }, +})); diff --git a/src/lib/workspace-state/grid-layout-helpers.ts b/src/lib/workspace-state/grid-layout-helpers.ts index 8b5077a9..91b4905f 100644 --- a/src/lib/workspace-state/grid-layout-helpers.ts +++ b/src/lib/workspace-state/grid-layout-helpers.ts @@ -12,6 +12,7 @@ export const DEFAULT_CARD_DIMENSIONS: Record youtube: { w: 4, h: 10 }, quiz: { w: 1, h: 4 }, image: { w: 4, h: 10 }, + audio: { w: 2, h: 8 }, }; /** diff --git a/src/lib/workspace-state/item-helpers.ts b/src/lib/workspace-state/item-helpers.ts index 41983ad6..389ab77d 100644 --- a/src/lib/workspace-state/item-helpers.ts +++ b/src/lib/workspace-state/item-helpers.ts @@ -1,4 +1,4 @@ -import type { CardType, ItemData, NoteData, PdfData, FlashcardData, FolderData, YouTubeData, ImageData } from "./types"; +import type { CardType, ItemData, NoteData, PdfData, FlashcardData, FolderData, YouTubeData, ImageData, AudioData } from "./types"; /** * Generate a unique item ID @@ -32,6 +32,8 @@ export function defaultDataFor(type: CardType): ItemData { return { url: "" } as YouTubeData; case "image": return { url: "" } as ImageData; + case "audio": + return { fileUrl: "", filename: "", processingStatus: "uploading" } as AudioData; default: return { field1: "" } as NoteData; } diff --git a/src/lib/workspace-state/types.ts b/src/lib/workspace-state/types.ts index d67f6667..9b7857c0 100644 --- a/src/lib/workspace-state/types.ts +++ b/src/lib/workspace-state/types.ts @@ -1,6 +1,6 @@ import type { CardColor } from './colors'; -export type CardType = "note" | "pdf" | "flashcard" | "folder" | "youtube" | "quiz" | "image"; +export type CardType = "note" | "pdf" | "flashcard" | "folder" | "youtube" | "quiz" | "image" | "audio"; /** * Source attribution for notes created from web search or deep research @@ -100,7 +100,31 @@ export interface QuizData { session?: QuizSessionData; // Session state for resuming } -export type ItemData = NoteData | PdfData | FlashcardData | FolderData | YouTubeData | QuizData | ImageData; +// Audio Types +export interface AudioSegment { + speaker: string; + timestamp: string; + content: string; + language?: string; + language_code?: string; + translation?: string; + emotion?: "happy" | "sad" | "angry" | "neutral"; +} + +export interface AudioData { + fileUrl: string; // Supabase/local storage URL + filename: string; // Original filename + fileSize?: number; // File size in bytes + duration?: number; // Duration in seconds + mimeType?: string; // MIME type of the audio file + summary?: string; // Gemini-generated summary + transcript?: string; // Full plain-text transcript + segments?: AudioSegment[];// Timestamped speaker segments + processingStatus: "uploading" | "processing" | "complete" | "failed"; + error?: string; // Error message if processing failed +} + +export type ItemData = NoteData | PdfData | FlashcardData | FolderData | YouTubeData | QuizData | ImageData | AudioData; // ===================================================== // FOLDER TYPES (DEPRECATED) From 1a67cea263023644fec639cef7ce83558d3976a6 Mon Sep 17 00:00:00 2001 From: Urjit Chakraborty <135136842+urjitc@users.noreply.github.com> Date: Tue, 10 Feb 2026 00:49:42 -0500 Subject: [PATCH 2/9] feat: v1 audio compelte --- src/components/chat/MentionMenu.tsx | 3 + src/components/modals/AudioRecorderDialog.tsx | 64 +- .../workspace-canvas/AudioCardContent.tsx | 560 +++++++++++++++--- .../AudioRecordingIndicator.tsx | 11 +- .../workspace-canvas/AudioWaveform.tsx | 136 +++++ .../workspace-canvas/WorkspaceCard.tsx | 2 +- .../workspace-canvas/WorkspaceHeader.tsx | 87 +-- src/components/workspace/SidebarCardList.tsx | 4 +- src/lib/stores/audio-recording-store.ts | 24 + src/lib/utils/format-workspace-context.ts | 38 +- .../workspace-state/grid-layout-helpers.ts | 2 +- 11 files changed, 780 insertions(+), 151 deletions(-) create mode 100644 src/components/workspace-canvas/AudioWaveform.tsx diff --git a/src/components/chat/MentionMenu.tsx b/src/components/chat/MentionMenu.tsx index 66eaec6d..0826fe37 100644 --- a/src/components/chat/MentionMenu.tsx +++ b/src/components/chat/MentionMenu.tsx @@ -12,6 +12,7 @@ import { LayoutGrid, Brain, ImageIcon, + Mic, } from "lucide-react"; import { PiCardsThreeBold } from "react-icons/pi"; import { Popover, PopoverContent, PopoverAnchor } from "@/components/ui/popover"; @@ -39,6 +40,8 @@ function getCardTypeIcon(type: CardType) { return ; case "image": return ; + case "audio": + return ; default: return ; } diff --git a/src/components/modals/AudioRecorderDialog.tsx b/src/components/modals/AudioRecorderDialog.tsx index 8cd58109..352f4bf4 100644 --- a/src/components/modals/AudioRecorderDialog.tsx +++ b/src/components/modals/AudioRecorderDialog.tsx @@ -13,6 +13,7 @@ import { import { Button } from "@/components/ui/button"; import { useAudioRecordingStore } from "@/lib/stores/audio-recording-store"; import { cn } from "@/lib/utils"; +import { AudioWaveform } from "@/components/workspace-canvas/AudioWaveform"; const ACCEPTED_AUDIO_TYPES = [ "audio/mp3", @@ -127,31 +128,25 @@ export function AudioRecorderDialog({ {isRecording ? "Recording in progress. You can close this dialog — recording continues in the background." - : "Record audio or upload an existing audio file."} + : "Process audio like meetings and lectures"}
{/* Recording Visualizer */} -
- {isRecording && !isPaused && ( -
- )} - - {isRecording ? ( -
+ {isRecording ? ( +
+ +
- ) : audioBlob ? ( -
- - - {formatDuration(duration)} - -
- ) : ( - - )} -
+
+ ) : ( +
+ {audioBlob ? ( +
+ + + {formatDuration(duration)} + +
+ ) : ( + + )} +
+ )} {/* Controls */}
diff --git a/src/components/workspace-canvas/AudioCardContent.tsx b/src/components/workspace-canvas/AudioCardContent.tsx index 3ad659f4..60c6a810 100644 --- a/src/components/workspace-canvas/AudioCardContent.tsx +++ b/src/components/workspace-canvas/AudioCardContent.tsx @@ -1,34 +1,81 @@ "use client"; -import { useState } from "react"; -import { Mic, ChevronDown, ChevronUp, AlertCircle, Loader2, Clock, User, Globe } from "lucide-react"; +import { useState, useRef, useEffect, useCallback } from "react"; +import { + Mic, + ChevronDown, + ChevronUp, + AlertCircle, + Loader2, + Play, + Pause, + Volume2, + VolumeX, + FileText, + User, + Globe, +} from "lucide-react"; import type { Item, AudioData, AudioSegment } from "@/lib/workspace-state/types"; import { cn } from "@/lib/utils"; +// ─── Helpers ──────────────────────────────────────────────────────────────── + +/** Parse "MM:SS" or "H:MM:SS" timestamp to seconds */ +function parseTimestamp(ts: string): number { + const parts = ts.split(":").map(Number); + if (parts.length === 3) return parts[0] * 3600 + parts[1] * 60 + parts[2]; + if (parts.length === 2) return parts[0] * 60 + parts[1]; + return 0; +} + +/** Format seconds to MM:SS */ +function fmtTime(s: number): string { + if (!isFinite(s) || s < 0) return "0:00"; + const mins = Math.floor(s / 60); + const secs = Math.floor(s % 60); + return `${mins}:${secs.toString().padStart(2, "0")}`; +} + +const SPEAKER_COLORS = [ + "bg-blue-500/15 text-blue-700 dark:text-blue-400", + "bg-purple-500/15 text-purple-700 dark:text-purple-400", + "bg-emerald-500/15 text-emerald-700 dark:text-emerald-400", + "bg-amber-500/15 text-amber-700 dark:text-amber-400", + "bg-rose-500/15 text-rose-700 dark:text-rose-400", + "bg-cyan-500/15 text-cyan-700 dark:text-cyan-400", +]; + +const EMOTION_ICONS: Record = { + happy: "😊", + sad: "😢", + angry: "😠", +}; + +// ─── Component ────────────────────────────────────────────────────────────── + interface AudioCardContentProps { item: Item; isCompact?: boolean; + isScrollLocked?: boolean; } -export function AudioCardContent({ item, isCompact = false }: AudioCardContentProps) { +export function AudioCardContent({ item, isCompact = false, isScrollLocked = false }: AudioCardContentProps) { const audioData = item.data as AudioData; - const [showTranscript, setShowTranscript] = useState(false); - const [showSegments, setShowSegments] = useState(false); - // Uploading state + // ── Loading / Error states ────────────────────────────────────────────── + if (audioData.processingStatus === "uploading") { return ( -
+

Uploading audio...

); } - // Processing state if (audioData.processingStatus === "processing") { return ( -
+
@@ -43,10 +90,9 @@ export function AudioCardContent({ item, isCompact = false }: AudioCardContentPr ); } - // Failed state if (audioData.processingStatus === "failed") { return ( -
+

Processing failed

@@ -58,106 +104,452 @@ export function AudioCardContent({ item, isCompact = false }: AudioCardContentPr ); } - // Complete state + // ── Complete state ────────────────────────────────────────────────────── + + return ; +} + +// ─── Complete state (with player) ─────────────────────────────────────────── + +function AudioCardComplete({ + audioData, + isCompact, + isScrollLocked, +}: { + audioData: AudioData; + isCompact: boolean; + isScrollLocked?: boolean; +}) { + const audioRef = useRef(null); + const progressRef = useRef(null); + + const [isPlaying, setIsPlaying] = useState(false); + const [currentTime, setCurrentTime] = useState(0); + const [duration, setDuration] = useState(audioData.duration ?? 0); + const [isMuted, setIsMuted] = useState(false); + const [playbackRate, setPlaybackRate] = useState(1); + const [showTranscript, setShowTranscript] = useState(false); + const [activeSegmentIdx, setActiveSegmentIdx] = useState(-1); + const [showSummary, setShowSummary] = useState(true); + const [showTimeline, setShowTimeline] = useState(true); + + // Build speaker → color map + const speakerColorMap = useRef(new Map()); + if (audioData.segments) { + let colorIdx = 0; + for (const seg of audioData.segments) { + if (!speakerColorMap.current.has(seg.speaker)) { + speakerColorMap.current.set( + seg.speaker, + SPEAKER_COLORS[colorIdx % SPEAKER_COLORS.length] + ); + colorIdx++; + } + } + } + + // ── Audio event handlers (only for non-compact view) ──────────────────────── + + useEffect(() => { + if (isCompact) return; // Skip audio setup in compact view + + const audio = audioRef.current; + if (!audio) return; + + const onTimeUpdate = () => setCurrentTime(audio.currentTime); + const onLoadedMetadata = () => { + if (audio.duration && isFinite(audio.duration)) { + setDuration(audio.duration); + } + }; + const onPlay = () => setIsPlaying(true); + const onPause = () => setIsPlaying(false); + const onEnded = () => { + setIsPlaying(false); + setCurrentTime(0); + }; + + audio.addEventListener("timeupdate", onTimeUpdate); + audio.addEventListener("loadedmetadata", onLoadedMetadata); + audio.addEventListener("play", onPlay); + audio.addEventListener("pause", onPause); + audio.addEventListener("ended", onEnded); + + return () => { + audio.removeEventListener("timeupdate", onTimeUpdate); + audio.removeEventListener("loadedmetadata", onLoadedMetadata); + audio.removeEventListener("play", onPlay); + audio.removeEventListener("pause", onPause); + audio.removeEventListener("ended", onEnded); + // Pause audio when component unmounts (e.g. closing modal mid-play) + audio.pause(); + }; + }, [isCompact]); + + // Update audio playback rate when it changes + useEffect(() => { + if (audioRef.current && !isCompact) { + audioRef.current.playbackRate = playbackRate; + } + }, [playbackRate, isCompact]); + + // Track active segment based on current time (only for non-compact view) + useEffect(() => { + if (isCompact || !audioData.segments || audioData.segments.length === 0) return; + let active = -1; + for (let i = 0; i < audioData.segments.length; i++) { + const segTime = parseTimestamp(audioData.segments[i].timestamp); + if (currentTime >= segTime) active = i; + else break; + } + setActiveSegmentIdx(active); + }, [currentTime, audioData.segments, isCompact]); + + const togglePlay = useCallback(() => { + if (isCompact) return; // Disable in compact view + + const audio = audioRef.current; + if (!audio) return; + if (audio.paused) { + audio.play().catch(() => { + // play() can reject if interrupted by another call or browser policy + }); + } else { + audio.pause(); + } + // isPlaying state is driven by 'play'/'pause' events — no manual set here + }, [isCompact]); + + // Spacebar to toggle play/pause (only for non-compact/modal view) + useEffect(() => { + if (isCompact) return; + + const onKeyDown = (e: KeyboardEvent) => { + if (e.code !== "Space") return; + // Don't hijack spacebar when user is typing in an input/textarea/button + const tag = (e.target as HTMLElement)?.tagName; + if (tag === "INPUT" || tag === "TEXTAREA" || tag === "SELECT") return; + e.preventDefault(); + togglePlay(); + }; + + window.addEventListener("keydown", onKeyDown); + return () => window.removeEventListener("keydown", onKeyDown); + }, [isCompact, togglePlay]); + + const seekTo = useCallback((seconds: number) => { + if (isCompact) return; // Disable in compact view + + const audio = audioRef.current; + if (!audio) return; + audio.currentTime = seconds; + setCurrentTime(seconds); + if (audio.paused) { + audio.play().catch(() => { + // play() can reject if interrupted by another call or browser policy + }); + } + // isPlaying state is driven by 'play'/'pause' events — no manual set here + }, [isCompact]); + + const handleProgressClick = useCallback( + (e: React.MouseEvent) => { + if (isCompact) return; // Disable in compact view + + const bar = progressRef.current; + if (!bar || !duration) return; + const rect = bar.getBoundingClientRect(); + const pct = Math.max(0, Math.min(1, (e.clientX - rect.left) / rect.width)); + seekTo(pct * duration); + }, + [duration, seekTo, isCompact] + ); + + const progress = duration > 0 ? (currentTime / duration) * 100 : 0; + return ( -
- {/* Summary */} - {audioData.summary && ( -
-

- Summary -

-

- {audioData.summary} -

+
+ {/* Hidden audio element (only for non-compact view) */} + {!isCompact &&
); } -function SegmentRow({ segment }: { segment: AudioSegment }) { +// ─── Segment Row ──────────────────────────────────────────────────────────── + +function SegmentRow({ + segment, + isActive, + speakerColor, + onSeek, + isCompact, +}: { + segment: AudioSegment; + isActive: boolean; + speakerColor: string; + onSeek?: () => void; + isCompact: boolean; +}) { return ( -
-
- - +
+ ); } diff --git a/src/components/workspace-canvas/AudioRecordingIndicator.tsx b/src/components/workspace-canvas/AudioRecordingIndicator.tsx index 6b646667..977c33c7 100644 --- a/src/components/workspace-canvas/AudioRecordingIndicator.tsx +++ b/src/components/workspace-canvas/AudioRecordingIndicator.tsx @@ -3,6 +3,7 @@ import { Mic, Square, Pause, Play } from "lucide-react"; import { useAudioRecordingStore } from "@/lib/stores/audio-recording-store"; import { cn } from "@/lib/utils"; +import { AudioWaveform } from "@/components/workspace-canvas/AudioWaveform"; function formatDuration(seconds: number): string { const mins = Math.floor(seconds / 60); @@ -29,7 +30,7 @@ export function AudioRecordingIndicator() { if (!isRecording || isDialogOpen) return null; return ( -
+
+ {/* Mini Waveform */} + + {/* Timer */} {formatDuration(duration)} diff --git a/src/components/workspace-canvas/AudioWaveform.tsx b/src/components/workspace-canvas/AudioWaveform.tsx new file mode 100644 index 00000000..f38f0d0a --- /dev/null +++ b/src/components/workspace-canvas/AudioWaveform.tsx @@ -0,0 +1,136 @@ +"use client"; + +import { useRef, useEffect } from "react"; +import { useAudioRecordingStore } from "@/lib/stores/audio-recording-store"; +import { cn } from "@/lib/utils"; + +interface AudioWaveformProps { + /** Width of the canvas */ + width?: number; + /** Height of the canvas */ + height?: number; + /** Number of bars to render */ + barCount?: number; + /** Color of the bars (CSS color string) */ + barColor?: string; + /** Additional className for the canvas */ + className?: string; +} + +export function AudioWaveform({ + width = 200, + height = 60, + barCount = 32, + barColor = "currentColor", + className, +}: AudioWaveformProps) { + const canvasRef = useRef(null); + const animFrameRef = useRef(0); + const smoothedRef = useRef(null); + const analyser = useAudioRecordingStore((s) => s._analyser); + const isRecording = useAudioRecordingStore((s) => s.isRecording); + const isPaused = useAudioRecordingStore((s) => s.isPaused); + + useEffect(() => { + const canvas = canvasRef.current; + if (!canvas || !analyser || !isRecording) return; + + const ctx = canvas.getContext("2d"); + if (!ctx) return; + + const bufferLength = analyser.frequencyBinCount; + const dataArray = new Uint8Array(bufferLength); + + // Initialise smoothed values + if (!smoothedRef.current || smoothedRef.current.length !== barCount) { + smoothedRef.current = new Float32Array(barCount); + } + const smoothed = smoothedRef.current; + const smoothing = 0.15; // lower = smoother / slower + const decay = 0.92; // how quickly bars fall back down + + const draw = () => { + animFrameRef.current = requestAnimationFrame(draw); + + if (isPaused) { + // When paused, draw flat lines + ctx.clearRect(0, 0, canvas.width, canvas.height); + const barWidth = canvas.width / barCount; + const gap = 2; + ctx.fillStyle = barColor; + for (let i = 0; i < barCount; i++) { + const x = i * barWidth; + const barH = 2; + const y = (canvas.height - barH) / 2; + ctx.fillRect(x + gap / 2, y, barWidth - gap, barH); + } + return; + } + + analyser.getByteTimeDomainData(dataArray); + + ctx.clearRect(0, 0, canvas.width, canvas.height); + + const barWidth = canvas.width / barCount; + const gap = 2; + const samplesPerBar = Math.floor(bufferLength / barCount); + + ctx.fillStyle = barColor; + + for (let i = 0; i < barCount; i++) { + // Average the amplitude of a chunk of samples for this bar + let sum = 0; + const start = i * samplesPerBar; + for (let j = start; j < start + samplesPerBar; j++) { + const v = (dataArray[j] - 128) / 128; // normalize to -1..1 + sum += Math.abs(v); + } + const raw = sum / samplesPerBar; + + // Smooth: rise fast, fall slow + if (raw > smoothed[i]) { + smoothed[i] += (raw - smoothed[i]) * smoothing * 3; // rise faster + } else { + smoothed[i] *= decay; // gentle decay + } + + const barH = Math.min(canvas.height, Math.max(2, smoothed[i] * canvas.height * 5)); + const x = i * barWidth; + const y = (canvas.height - barH) / 2; + + // Rounded bars + const radius = Math.min((barWidth - gap) / 2, barH / 2, 3); + ctx.beginPath(); + ctx.roundRect(x + gap / 2, y, barWidth - gap, barH, radius); + ctx.fill(); + } + }; + + draw(); + + return () => { + cancelAnimationFrame(animFrameRef.current); + }; + }, [analyser, isRecording, isPaused, barCount, barColor]); + + // Clear canvas when not recording + useEffect(() => { + if (!isRecording) { + const canvas = canvasRef.current; + if (canvas) { + const ctx = canvas.getContext("2d"); + if (ctx) ctx.clearRect(0, 0, canvas.width, canvas.height); + } + } + }, [isRecording]); + + return ( + + ); +} diff --git a/src/components/workspace-canvas/WorkspaceCard.tsx b/src/components/workspace-canvas/WorkspaceCard.tsx index 0aa595c1..43285f14 100644 --- a/src/components/workspace-canvas/WorkspaceCard.tsx +++ b/src/components/workspace-canvas/WorkspaceCard.tsx @@ -987,7 +987,7 @@ function WorkspaceCard({ {/* Audio Content - render audio player and transcript */} {!isOpenInPanel && item.type === 'audio' && shouldShowPreview && (
- +
)} diff --git a/src/components/workspace-canvas/WorkspaceHeader.tsx b/src/components/workspace-canvas/WorkspaceHeader.tsx index 982e6b87..b5d7d9cb 100644 --- a/src/components/workspace-canvas/WorkspaceHeader.tsx +++ b/src/components/workspace-canvas/WorkspaceHeader.tsx @@ -342,7 +342,20 @@ export default function WorkspaceHeader({ const { url: fileUrl } = await uploadFileDirect(file); // Create the audio card immediately (shows "processing" state) - const itemId = addItem("audio", file.name.replace(/\.[^/.]+$/, ""), { + const now = new Date(); + const dateStr = now.toLocaleDateString('en-US', { + month: 'short', + day: 'numeric', + year: now.getFullYear() !== new Date().getFullYear() ? 'numeric' : undefined + }); + const timeStr = now.toLocaleTimeString('en-US', { + hour: 'numeric', + minute: '2-digit', + hour12: true + }); + const title = `${dateStr} ${timeStr} Recording`; + + const itemId = addItem("audio", title, { fileUrl, filename: file.name, fileSize: file.size, @@ -753,6 +766,7 @@ export default function WorkspaceHeader({ {activeItems[0].type === 'youtube' && } {activeItems[0].type === 'quiz' && } {activeItems[0].type === 'image' && } + {activeItems[0].type === 'audio' && } {activeItems[0].type === 'folder' && } @@ -949,7 +963,7 @@ export default function WorkspaceHeader({ {!isCompactMode && New} - + { if (addItem) { @@ -985,10 +999,46 @@ export default function WorkspaceHeader({ setShowUploadDialog(true); setIsNewMenuOpen(false); }} - className="flex items-center gap-2 cursor-pointer" + className="flex items-center gap-2 cursor-pointer p-2" > - Upload (PDF, Image) +
+ Upload + PDF/Image +
+ + + { + openAudioDialog(); + setIsNewMenuOpen(false); + }} + className="flex items-center gap-2 cursor-pointer p-2" + > + +
+ Audio + Lecture/Meeting +
+ + { + setShowYouTubeDialog(true); + }} + className="flex items-center gap-2 cursor-pointer" + > + + YouTube + + { + setShowWebsiteDialog(true); + setIsNewMenuOpen(false); + }} + className="flex items-center gap-2 cursor-pointer" + > + + Website @@ -1030,35 +1080,6 @@ export default function WorkspaceHeader({ - { - openAudioDialog(); - setIsNewMenuOpen(false); - }} - className="flex items-center gap-2 cursor-pointer" - > - - Audio - - { - setShowYouTubeDialog(true); - }} - className="flex items-center gap-2 cursor-pointer" - > - - YouTube - - { - setShowWebsiteDialog(true); - setIsNewMenuOpen(false); - }} - className="flex items-center gap-2 cursor-pointer" - > - - Website - {/* { toast.success("Deep Research action selected"); diff --git a/src/components/workspace/SidebarCardList.tsx b/src/components/workspace/SidebarCardList.tsx index ec6a5b7d..0de0bf54 100644 --- a/src/components/workspace/SidebarCardList.tsx +++ b/src/components/workspace/SidebarCardList.tsx @@ -1,7 +1,7 @@ "use client"; import { memo, useMemo, useCallback, useState, useEffect, useRef } from "react"; -import { ChevronRight, FileText, File, FolderOpen, Folder as FolderIcon, MoreVertical, Trash2, Pencil, FolderInput, Play, Brain, ImageIcon } from "lucide-react"; +import { ChevronRight, FileText, File, FolderOpen, Folder as FolderIcon, MoreVertical, Trash2, Pencil, FolderInput, Play, Brain, ImageIcon, Mic } from "lucide-react"; import { PiCardsThreeBold } from "react-icons/pi"; import { SidebarMenu, @@ -61,6 +61,8 @@ function getCardTypeIcon(type: CardType) { return ; case "image": return ; + case "audio": + return ; default: return ; } diff --git a/src/lib/stores/audio-recording-store.ts b/src/lib/stores/audio-recording-store.ts index 35fd996b..b45765ce 100644 --- a/src/lib/stores/audio-recording-store.ts +++ b/src/lib/stores/audio-recording-store.ts @@ -13,6 +13,8 @@ export interface AudioRecordingState { _stream: MediaStream | null; _chunks: Blob[]; _timerId: ReturnType | null; + _audioContext: AudioContext | null; + _analyser: AnalyserNode | null; // Dialog visibility (recording continues regardless) isDialogOpen: boolean; @@ -54,6 +56,8 @@ export const useAudioRecordingStore = create((set, get) => _stream: null, _chunks: [], _timerId: null, + _audioContext: null, + _analyser: null, isDialogOpen: false, openDialog: () => set({ isDialogOpen: true }), @@ -76,6 +80,13 @@ export const useAudioRecordingStore = create((set, get) => const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); const mimeType = getSupportedMimeType(); const recorder = new MediaRecorder(stream, { mimeType }); + + // Set up Web Audio API analyser for waveform visualisation + const audioContext = new AudioContext(); + const source = audioContext.createMediaStreamSource(stream); + const analyser = audioContext.createAnalyser(); + analyser.fftSize = 256; + source.connect(analyser); const chunks: Blob[] = []; recorder.ondataavailable = (e) => { @@ -89,6 +100,10 @@ export const useAudioRecordingStore = create((set, get) => if (s._stream) { s._stream.getTracks().forEach((t) => t.stop()); } + const currentState = get(); + if (currentState._audioContext) { + currentState._audioContext.close().catch(() => {}); + } set({ audioBlob: blob, isRecording: false, @@ -97,6 +112,8 @@ export const useAudioRecordingStore = create((set, get) => _stream: null, _chunks: [], _timerId: null, + _audioContext: null, + _analyser: null, // Auto-open dialog when recording stops so user can review isDialogOpen: true, }); @@ -124,6 +141,8 @@ export const useAudioRecordingStore = create((set, get) => _stream: stream, _chunks: chunks, _timerId: timerId, + _audioContext: audioContext, + _analyser: analyser, }); } catch (err: any) { if (err.name === "NotAllowedError") { @@ -172,6 +191,9 @@ export const useAudioRecordingStore = create((set, get) => if (state._stream) { state._stream.getTracks().forEach((t) => t.stop()); } + if (state._audioContext) { + state._audioContext.close().catch(() => {}); + } set({ isRecording: false, isPaused: false, @@ -182,6 +204,8 @@ export const useAudioRecordingStore = create((set, get) => _stream: null, _chunks: [], _timerId: null, + _audioContext: null, + _analyser: null, }); }, })); diff --git a/src/lib/utils/format-workspace-context.ts b/src/lib/utils/format-workspace-context.ts index f462d825..344356aa 100644 --- a/src/lib/utils/format-workspace-context.ts +++ b/src/lib/utils/format-workspace-context.ts @@ -1,4 +1,4 @@ -import type { AgentState, Item, NoteData, PdfData, FlashcardData, FlashcardItem, YouTubeData, QuizData, QuizQuestion, ImageData } from "@/lib/workspace-state/types"; +import type { AgentState, Item, NoteData, PdfData, FlashcardData, FlashcardItem, YouTubeData, QuizData, QuizQuestion, ImageData, AudioData } from "@/lib/workspace-state/types"; import { serializeBlockNote } from "./serialize-blocknote"; import { type Block } from "@/components/editor/BlockNoteEditor"; @@ -376,6 +376,9 @@ function formatSelectedCardFull(item: Item, index: number): string { case "image": lines.push(...formatImageDetailsFull(item.data as ImageData)); break; + case "audio": + lines.push(...formatAudioDetailsFull(item.data as AudioData)); + break; } lines.push(``); @@ -657,3 +660,36 @@ function formatQuizDetailsFull(data: QuizData): string[] { return lines; } + +/** + * Formats audio details — timeline segments only + */ +function formatAudioDetailsFull(data: AudioData): string[] { + const lines: string[] = []; + + if (data.filename) { + lines.push(` - Filename: ${data.filename}`); + } + + if (data.duration) { + const mins = Math.floor(data.duration / 60); + const secs = Math.floor(data.duration % 60); + lines.push(` - Duration: ${mins}:${secs.toString().padStart(2, "0")}`); + } + + if (data.segments && data.segments.length > 0) { + lines.push(` - Timeline (${data.segments.length} segments):`); + for (const seg of data.segments) { + const lang = seg.language && seg.language !== "English" ? ` [${seg.language}]` : ""; + const emotion = seg.emotion && seg.emotion !== "neutral" ? ` (${seg.emotion})` : ""; + lines.push(` [${seg.timestamp}] ${seg.speaker}${lang}${emotion}: ${seg.content}`); + if (seg.translation) { + lines.push(` Translation: ${seg.translation}`); + } + } + } else { + lines.push(` - (No timeline segments available)`); + } + + return lines; +} diff --git a/src/lib/workspace-state/grid-layout-helpers.ts b/src/lib/workspace-state/grid-layout-helpers.ts index 91b4905f..044328f3 100644 --- a/src/lib/workspace-state/grid-layout-helpers.ts +++ b/src/lib/workspace-state/grid-layout-helpers.ts @@ -12,7 +12,7 @@ export const DEFAULT_CARD_DIMENSIONS: Record youtube: { w: 4, h: 10 }, quiz: { w: 1, h: 4 }, image: { w: 4, h: 10 }, - audio: { w: 2, h: 8 }, + audio: { w: 2, h: 10 }, }; /** From a5ff9a69d8008a1e8b9fbf5d66430d86ca8bd624 Mon Sep 17 00:00:00 2001 From: Urjit Chakraborty <135136842+urjitc@users.noreply.github.com> Date: Tue, 10 Feb 2026 01:28:35 -0500 Subject: [PATCH 3/9] feat: prevent multiple recording sessions --- src/components/workspace-canvas/WorkspaceHeader.tsx | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/components/workspace-canvas/WorkspaceHeader.tsx b/src/components/workspace-canvas/WorkspaceHeader.tsx index b5d7d9cb..69c6c7d0 100644 --- a/src/components/workspace-canvas/WorkspaceHeader.tsx +++ b/src/components/workspace-canvas/WorkspaceHeader.tsx @@ -1010,6 +1010,11 @@ export default function WorkspaceHeader({ { + const isRecording = useAudioRecordingStore.getState().isRecording; + if (isRecording) { + toast.error("Recording already in progress"); + return; + } openAudioDialog(); setIsNewMenuOpen(false); }} From e7b9772d5f0e3d2dd9aca8ed1b1bf15bbd437fa0 Mon Sep 17 00:00:00 2001 From: Urjit Chakraborty <135136842+urjitc@users.noreply.github.com> Date: Tue, 10 Feb 2026 01:39:51 -0500 Subject: [PATCH 4/9] fix: centralize new workspace item menus --- .../workspace-canvas/WorkspaceHeader.tsx | 167 +++--------- .../workspace-canvas/WorkspaceSection.tsx | 253 +++++++++++------- .../workspace-canvas/workspace-menu-items.tsx | 146 ++++++++++ 3 files changed, 336 insertions(+), 230 deletions(-) create mode 100644 src/components/workspace-canvas/workspace-menu-items.tsx diff --git a/src/components/workspace-canvas/WorkspaceHeader.tsx b/src/components/workspace-canvas/WorkspaceHeader.tsx index 69c6c7d0..a12329b7 100644 --- a/src/components/workspace-canvas/WorkspaceHeader.tsx +++ b/src/components/workspace-canvas/WorkspaceHeader.tsx @@ -55,6 +55,7 @@ import { UploadDialog } from "@/components/modals/UploadDialog"; import { AudioRecorderDialog } from "@/components/modals/AudioRecorderDialog"; import { useAudioRecordingStore } from "@/lib/stores/audio-recording-store"; import { getBestFrameForRatio } from "@/lib/workspace-state/aspect-ratios"; +import { renderWorkspaceMenuItems } from "./workspace-menu-items"; interface WorkspaceHeaderProps { titleInputRef: React.RefObject; searchQuery: string; @@ -964,141 +965,43 @@ export default function WorkspaceHeader({ - { - if (addItem) { - const itemId = addItem("note"); - // Auto-navigate to the newly created note instead of opening modal - if (onItemCreated && itemId) { - onItemCreated([itemId]); - } - } - }} - className="flex items-center gap-2 cursor-pointer" - > - - Note - - - {addItem && ( - { + {renderWorkspaceMenuItems({ + callbacks: { + onCreateNote: () => { if (addItem) { - addItem("folder"); - } - }} - className="flex items-center gap-2 cursor-pointer" - > - - Folder - - )} - - { - setShowUploadDialog(true); - setIsNewMenuOpen(false); - }} - className="flex items-center gap-2 cursor-pointer p-2" - > - -
- Upload - PDF/Image -
- - - { - const isRecording = useAudioRecordingStore.getState().isRecording; - if (isRecording) { - toast.error("Recording already in progress"); - return; - } - openAudioDialog(); - setIsNewMenuOpen(false); - }} - className="flex items-center gap-2 cursor-pointer p-2" - > - -
- Audio - Lecture/Meeting -
- - { - setShowYouTubeDialog(true); - }} - className="flex items-center gap-2 cursor-pointer" - > - - YouTube - - { - setShowWebsiteDialog(true); - setIsNewMenuOpen(false); - }} - className="flex items-center gap-2 cursor-pointer" - > - - Website - - - - - - Learn - - - { - if (addItem) { - const itemId = addItem("flashcard"); - if (onItemCreated && itemId) { - onItemCreated([itemId]); - } + const itemId = addItem("note"); + if (onItemCreated && itemId) { + onItemCreated([itemId]); } - }} - className="flex items-center gap-2 cursor-pointer" - > - - Flashcards - - { - // Open chat if closed - if (setIsChatExpanded && !isChatExpanded) { - setIsChatExpanded(true); + } + }, + onCreateFolder: () => { if (addItem) addItem("folder"); }, + onUpload: () => { setShowUploadDialog(true); setIsNewMenuOpen(false); }, + onAudio: () => { openAudioDialog(); setIsNewMenuOpen(false); }, + onYouTube: () => { setShowYouTubeDialog(true); setIsNewMenuOpen(false); }, + onWebsite: () => { setShowWebsiteDialog(true); setIsNewMenuOpen(false); }, + onFlashcards: () => { + if (addItem) { + const itemId = addItem("flashcard"); + if (onItemCreated && itemId) { + onItemCreated([itemId]); } - // Fill composer with quiz creation prompt - aui?.composer().setText("Create a quiz about "); - // Focus the composer input - focusComposerInput(); - toast.success("Quiz creation started"); - }} - className="flex items-center gap-2 cursor-pointer" - > - - Quiz - - - - {/* { - toast.success("Deep Research action selected"); - setSelectedActions(["deep-research"]); - aui?.composer().setText("I want to do research on "); - if (setIsChatExpanded && !isChatExpanded) { - setIsChatExpanded(true); - } - }} - className="flex items-center gap-2 cursor-pointer" - > - - Deep Research - */} + } + }, + onQuiz: () => { + if (setIsChatExpanded && !isChatExpanded) { + setIsChatExpanded(true); + } + aui?.composer().setText("Create a quiz about "); + focusComposerInput(); + toast.success("Quiz creation started"); + }, + }, + MenuItem: DropdownMenuItem, + MenuSub: DropdownMenuSub, + MenuSubTrigger: DropdownMenuSubTrigger, + MenuSubContent: DropdownMenuSubContent, + })} )} diff --git a/src/components/workspace-canvas/WorkspaceSection.tsx b/src/components/workspace-canvas/WorkspaceSection.tsx index f6d261c6..7ac5829e 100644 --- a/src/components/workspace-canvas/WorkspaceSection.tsx +++ b/src/components/workspace-canvas/WorkspaceSection.tsx @@ -51,6 +51,11 @@ import { UploadDialog } from "@/components/modals/UploadDialog"; import { AudioRecordingIndicator } from "./AudioRecordingIndicator"; import { getBestFrameForRatio } from "@/lib/workspace-state/aspect-ratios"; import { useReactiveNavigation } from "@/hooks/ui/use-reactive-navigation"; +import { renderWorkspaceMenuItems } from "./workspace-menu-items"; +import { useAudioRecordingStore } from "@/lib/stores/audio-recording-store"; +import { AudioRecorderDialog } from "@/components/modals/AudioRecorderDialog"; +import { CreateWebsiteDialog } from "@/components/modals/CreateWebsiteDialog"; +import { useQueryClient } from "@tanstack/react-query"; interface WorkspaceSectionProps { // Loading states @@ -195,6 +200,13 @@ export function WorkspaceSection({ // Workspace settings and share modal state const [showYouTubeDialog, setShowYouTubeDialog] = useState(false); const [showUploadDialog, setShowUploadDialog] = useState(false); + const [showWebsiteDialog, setShowWebsiteDialog] = useState(false); + const showAudioDialog = useAudioRecordingStore((s) => s.isDialogOpen); + const openAudioDialog = useAudioRecordingStore((s) => s.openDialog); + const closeAudioDialog = useAudioRecordingStore((s) => s.closeDialog); + + // React Query client for cache invalidation + const queryClient = useQueryClient(); // Get workspace data from context const { workspaces } = useWorkspaceContext(); @@ -467,6 +479,91 @@ export function WorkspaceSection({ // Use reactive navigation hook for auto-scroll/selection const { handleCreatedItems } = useReactiveNavigation(state); + const handleAudioReady = useCallback(async (file: File) => { + if (!addItem) return; + + const loadingToastId = toast.loading("Uploading audio..."); + + try { + const { url: fileUrl } = await uploadFileDirect(file); + + const now = new Date(); + const dateStr = now.toLocaleDateString('en-US', { + month: 'short', + day: 'numeric', + year: now.getFullYear() !== new Date().getFullYear() ? 'numeric' : undefined + }); + const timeStr = now.toLocaleTimeString('en-US', { + hour: 'numeric', + minute: '2-digit', + hour12: true + }); + const title = `${dateStr} ${timeStr} Recording`; + + const itemId = addItem("audio", title, { + fileUrl, + filename: file.name, + fileSize: file.size, + mimeType: file.type || "audio/webm", + processingStatus: "processing", + } as any); + + if (handleCreatedItems && itemId) { + handleCreatedItems([itemId]); + } + + toast.dismiss(loadingToastId); + toast.success("Audio uploaded \u2014 analyzing with Gemini..."); + + fetch("/api/audio/process", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + fileUrl, + filename: file.name, + mimeType: file.type || "audio/webm", + }), + }) + .then((res) => res.json()) + .then((result) => { + if (result.success) { + window.dispatchEvent( + new CustomEvent("audio-processing-complete", { + detail: { + itemId, + summary: result.summary, + transcript: result.transcript, + segments: result.segments, + }, + }) + ); + } else { + window.dispatchEvent( + new CustomEvent("audio-processing-complete", { + detail: { + itemId, + error: result.error || "Processing failed", + }, + }) + ); + } + }) + .catch((err) => { + window.dispatchEvent( + new CustomEvent("audio-processing-complete", { + detail: { + itemId, + error: err.message || "Processing failed", + }, + }) + ); + }); + } catch (error: any) { + toast.dismiss(loadingToastId); + toast.error(error.message || "Failed to upload audio"); + } + }, [addItem, handleCreatedItems]); + // Get search params for invite check const searchParams = useSearchParams(); const hasInviteParam = searchParams.get('invite'); @@ -543,106 +640,46 @@ export function WorkspaceSection({ {/* Right-Click Context Menu */} {addItem && ( - - Create - { - if (addItem) { - const itemId = addItem("note"); - // Auto-navigate to the newly created note instead of opening modal - if (handleCreatedItems && itemId) { - handleCreatedItems([itemId]); - } - } - }} - className="flex items-center gap-2 cursor-pointer" - > - - Note - - - { - if (addItem) { - addItem("folder"); - } - }} - className="flex items-center gap-2 cursor-pointer" - > - - Folder - - - {operations && currentWorkspaceId && ( - - - Upload (PDF, Image) - - )} - - - - - Learn - - - { - if (addItem) { - const itemId = addItem("flashcard"); - if (handleCreatedItems && itemId) { - handleCreatedItems([itemId]); - } + + {renderWorkspaceMenuItems({ + callbacks: { + onCreateNote: () => { + if (addItem) { + const itemId = addItem("note"); + if (handleCreatedItems && itemId) { + handleCreatedItems([itemId]); } - }} - className="flex items-center gap-2 cursor-pointer" - > - - Flashcards - - { - // Open chat if closed - if (setIsChatExpanded && !isChatExpanded && isDesktop) { - setIsChatExpanded(true); + } + }, + onCreateFolder: () => { if (addItem) addItem("folder"); }, + onUpload: () => handleUploadMenuItemClick(), + onAudio: () => openAudioDialog(), + onYouTube: () => setShowYouTubeDialog(true), + onWebsite: () => setShowWebsiteDialog(true), + onFlashcards: () => { + if (addItem) { + const itemId = addItem("flashcard"); + if (handleCreatedItems && itemId) { + handleCreatedItems([itemId]); } - // Fill composer with quiz creation prompt - aui.composer().setText("Create a quiz about "); - // Focus the composer input - focusComposerInput(); - toast.success("Quiz creation started"); - }} - className="flex items-center gap-2 cursor-pointer" - > - - Quiz - - - - setShowYouTubeDialog(true)} - className="flex items-center gap-2 cursor-pointer" - > - - YouTube - - {/* { - toast.success("Deep Research action selected"); - setSelectedActions(["deep-research"]); - aui.composer().setText("I want to do research on "); - if (setIsChatExpanded && !isChatExpanded && isDesktop) { - setIsChatExpanded(true); - } - }} - className="flex items-center gap-2 cursor-pointer" - > - - Deep Research - */} + } + }, + onQuiz: () => { + if (setIsChatExpanded && !isChatExpanded && isDesktop) { + setIsChatExpanded(true); + } + aui.composer().setText("Create a quiz about "); + focusComposerInput(); + toast.success("Quiz creation started"); + }, + }, + MenuItem: ContextMenuItem, + MenuSub: ContextMenuSub, + MenuSubTrigger: ContextMenuSubTrigger, + MenuSubContent: ContextMenuSubContent, + MenuLabel: ContextMenuLabel, + showUpload: !!(operations && currentWorkspaceId), + })} )} @@ -711,6 +748,26 @@ export function WorkspaceSection({ /> )} + {/* Website Dialog */} + {currentWorkspaceId && ( + { + void queryClient.invalidateQueries({ + queryKey: ["workspace", currentWorkspaceId, "events"], + }); + }} + /> + )} + {/* Audio Recorder Dialog */} + { if (open) openAudioDialog(); else closeAudioDialog(); }} + onAudioReady={handleAudioReady} + /> {/* Floating recording indicator (visible when dialog is closed but recording is active) */}
diff --git a/src/components/workspace-canvas/workspace-menu-items.tsx b/src/components/workspace-canvas/workspace-menu-items.tsx new file mode 100644 index 00000000..078e42a6 --- /dev/null +++ b/src/components/workspace-canvas/workspace-menu-items.tsx @@ -0,0 +1,146 @@ +"use client"; + +import type React from "react"; +import { FileText, Folder, Upload, Play, Brain, Mic, Newspaper } from "lucide-react"; +import { LuBook } from "react-icons/lu"; +import { PiCardsThreeBold } from "react-icons/pi"; +import { useAudioRecordingStore } from "@/lib/stores/audio-recording-store"; +import { toast } from "sonner"; + +export interface WorkspaceMenuCallbacks { + onCreateNote: () => void; + onCreateFolder: () => void; + onUpload: () => void; + onAudio: () => void; + onYouTube: () => void; + onWebsite: () => void; + onFlashcards: () => void; + onQuiz: () => void; +} + +/** + * Shared "Create" menu items for both the header dropdown and the right-click + * context menu. The caller passes render helpers so the same logical items + * can be rendered as either DropdownMenuItems or ContextMenuItems. + */ +export function renderWorkspaceMenuItems({ + callbacks, + MenuItem, + MenuSub, + MenuSubTrigger, + MenuSubContent, + MenuLabel, + showUpload = true, +}: { + callbacks: WorkspaceMenuCallbacks; + MenuItem: React.ComponentType<{ onSelect?: () => void; onClick?: () => void; className?: string; children: React.ReactNode }>; + MenuSub: React.ComponentType<{ children: React.ReactNode }>; + MenuSubTrigger: React.ComponentType<{ className?: string; children: React.ReactNode }>; + MenuSubContent: React.ComponentType<{ children: React.ReactNode }>; + MenuLabel?: React.ComponentType<{ className?: string; children: React.ReactNode }>; + showUpload?: boolean; +}) { + const handleAudioClick = () => { + const isRecording = useAudioRecordingStore.getState().isRecording; + if (isRecording) { + toast.error("Recording already in progress"); + return; + } + callbacks.onAudio(); + }; + + return ( + <> + {MenuLabel && ( + Create + )} + + + + Note + + + + + Folder + + + {showUpload && ( + + +
+ Upload + PDF/Image +
+ + )} + + + +
+ Audio + Lecture/Meeting +
+ + + + + YouTube + + + + + Website + + + + + + Learn + + + + + Flashcards + + + + Quiz + + + + + ); +} From 6604aba91ac8ce7130839f995f4bc19dd332d522 Mon Sep 17 00:00:00 2001 From: Urjit Chakraborty <135136842+urjitc@users.noreply.github.com> Date: Tue, 10 Feb 2026 01:46:45 -0500 Subject: [PATCH 5/9] fixes --- src/app/api/audio/process/route.ts | 18 +- src/components/modals/AudioRecorderDialog.tsx | 384 ++++++++++-------- 2 files changed, 222 insertions(+), 180 deletions(-) diff --git a/src/app/api/audio/process/route.ts b/src/app/api/audio/process/route.ts index 30805fc6..4eff6d06 100644 --- a/src/app/api/audio/process/route.ts +++ b/src/app/api/audio/process/route.ts @@ -3,7 +3,6 @@ import { NextRequest, NextResponse } from "next/server"; import { auth } from "@/lib/auth"; import { headers } from "next/headers"; -export const maxDuration = 60; export const dynamic = "force-dynamic"; const apiKey = process.env.GOOGLE_GENERATIVE_AI_API_KEY; @@ -60,13 +59,10 @@ export async function POST(req: NextRequest) { const prompt = `Process this audio file and generate a detailed transcription and summary. Requirements: -1. Provide a brief summary of the entire audio content. +1. Provide a comprehensive summary of the entire audio content. 2. Identify distinct speakers (e.g., Speaker 1, Speaker 2, or names if context allows). 3. Provide accurate timestamps for each segment (Format: MM:SS). -4. Detect the primary language of each segment. -5. If the segment is in a language different than English, also provide the English translation. -6. Identify the primary emotion of the speaker in this segment. You MUST choose exactly one of the following: happy, sad, angry, neutral. -7. Provide the full plain-text transcript combining all segments.`; +4. Provide the full plain-text transcript combining all segments.`; const response = await client.models.generateContent({ model: "gemini-2.5-flash", @@ -108,21 +104,11 @@ Requirements: speaker: { type: Type.STRING }, timestamp: { type: Type.STRING }, content: { type: Type.STRING }, - language: { type: Type.STRING }, - language_code: { type: Type.STRING }, - translation: { type: Type.STRING }, - emotion: { - type: Type.STRING, - enum: ["happy", "sad", "angry", "neutral"], - }, }, required: [ "speaker", "timestamp", "content", - "language", - "language_code", - "emotion", ], }, }, diff --git a/src/components/modals/AudioRecorderDialog.tsx b/src/components/modals/AudioRecorderDialog.tsx index 352f4bf4..97964861 100644 --- a/src/components/modals/AudioRecorderDialog.tsx +++ b/src/components/modals/AudioRecorderDialog.tsx @@ -10,6 +10,16 @@ import { DialogHeader, DialogTitle, } from "@/components/ui/dialog"; +import { + AlertDialog, + AlertDialogAction, + AlertDialogCancel, + AlertDialogContent, + AlertDialogDescription, + AlertDialogFooter, + AlertDialogHeader, + AlertDialogTitle, +} from "@/components/ui/alert-dialog"; import { Button } from "@/components/ui/button"; import { useAudioRecordingStore } from "@/lib/stores/audio-recording-store"; import { cn } from "@/lib/utils"; @@ -60,6 +70,7 @@ export function AudioRecorderDialog({ const fileInputRef = useRef(null); const [isSubmitting, setIsSubmitting] = useState(false); + const [showConfirmClose, setShowConfirmClose] = useState(false); // Stable blob URL for audio preview (avoids creating new URLs on every render) const audioBlobUrl = useMemo(() => { @@ -67,18 +78,23 @@ export function AudioRecorderDialog({ return null; }, [audioBlob]); - // Close dialog — if recording, just hide; if idle/done, fully close + // Close dialog — if recording, just hide; if ready, confirm; if idle, fully close const handleClose = useCallback(() => { if (isRecording) { // Just hide the dialog — recording continues in the background onOpenChange(false); return; } - // Not recording — fully reset and close + if (audioBlob) { + // Show confirmation dialog - recording will be lost + setShowConfirmClose(true); + return; + } + // Not recording and no blob — fully reset and close resetRecording(); setIsSubmitting(false); onOpenChange(false); - }, [isRecording, resetRecording, onOpenChange]); + }, [isRecording, audioBlob, resetRecording, onOpenChange]); const handleFileSelect = useCallback( (e: React.ChangeEvent) => { @@ -120,196 +136,236 @@ export function AudioRecorderDialog({ onOpenChange(false); }, [audioBlob, onAudioReady, resetRecording, onOpenChange]); + const handleConfirmClose = useCallback(() => { + // User confirmed - discard recording and close + resetRecording(); + setIsSubmitting(false); + setShowConfirmClose(false); + onOpenChange(false); + }, [resetRecording, onOpenChange]); + + const handleCancelClose = useCallback(() => { + // User cancelled - keep dialog open + setShowConfirmClose(false); + }, []); + return ( - - e.stopPropagation()}> - - Audio - - {isRecording - ? "Recording in progress. You can close this dialog — recording continues in the background." - : "Process audio like meetings and lectures"} - - + <> + + e.stopPropagation()}> + + + {isRecording + ? "Recording in progress..." + : audioBlob + ? "Recording ready" + : "Record audio" + } + + + {isRecording + ? "You can close this dialog, recording continues in the background." + : audioBlob + ? "" + : "Generate AI-powered summaries, transcripts, and timestamps from your audio"} + + -
- {/* Recording Visualizer */} - {isRecording ? ( -
- -
- + {/* Recording Visualizer */} + {isRecording ? ( +
+ - - {formatDuration(duration)} - -
-
- ) : ( -
- {audioBlob ? ( -
- - +
+ + {formatDuration(duration)}
- ) : ( - - )} -
- )} - - {/* Controls */} -
- {!isRecording && !audioBlob && ( - + {audioBlob ? ( +
+ + + {formatDuration(duration)} + +
+ ) : ( + + )} +
)} - {isRecording && ( - <> - + {/* Controls */} +
+ {!isRecording && !audioBlob && ( - + )} + + {isRecording && ( + <> + + + + )} + + {audioBlob && !isRecording && ( + <> + + + + )} +
+ + {/* Preview audio playback */} + {audioBlobUrl && !isRecording && ( +
- - {isRecording ? ( - - ) : ( - - )} - - - + + + Discard Recording + + + + + ); } From 609ba368f229edad4cad5fc1d1b04e1d2ba77213 Mon Sep 17 00:00:00 2001 From: Urjit Chakraborty <135136842+urjitc@users.noreply.github.com> Date: Tue, 10 Feb 2026 01:55:15 -0500 Subject: [PATCH 6/9] fixes --- src/app/api/audio/process/route.ts | 48 +++++++++++++++++-- src/components/modals/AudioRecorderDialog.tsx | 20 +++++--- .../workspace-canvas/AudioCardContent.tsx | 19 ++++++-- .../workspace-canvas/AudioWaveform.tsx | 2 +- .../WorkspaceCanvasDropzone.tsx | 21 ++++---- .../workspace-canvas/WorkspaceContent.tsx | 6 ++- .../workspace-canvas/WorkspaceHeader.tsx | 2 +- src/hooks/audio/use-audio-recorder.ts | 9 +++- src/lib/stores/audio-recording-store.ts | 39 +++++++++++---- 9 files changed, 127 insertions(+), 39 deletions(-) diff --git a/src/app/api/audio/process/route.ts b/src/app/api/audio/process/route.ts index 4eff6d06..7cac2cec 100644 --- a/src/app/api/audio/process/route.ts +++ b/src/app/api/audio/process/route.ts @@ -39,6 +39,32 @@ export async function POST(req: NextRequest) { ); } + // Validate URL origin to prevent SSRF + const allowedHosts = [ + process.env.NEXT_PUBLIC_SUPABASE_URL + ? new URL(process.env.NEXT_PUBLIC_SUPABASE_URL).hostname + : null, + ].filter(Boolean) as string[]; + + let parsedUrl: URL; + try { + parsedUrl = new URL(fileUrl); + } catch { + return NextResponse.json( + { error: "Invalid fileUrl" }, + { status: 400 } + ); + } + + if ( + !allowedHosts.some((host) => parsedUrl.hostname === host || parsedUrl.hostname.endsWith(`.${host}`)) + ) { + return NextResponse.json( + { error: "fileUrl origin is not allowed" }, + { status: 400 } + ); + } + // Determine MIME type const audioMimeType = mimeType || guessMimeType(filename || fileUrl); @@ -46,12 +72,28 @@ export async function POST(req: NextRequest) { const audioResponse = await fetch(fileUrl); if (!audioResponse.ok) { return NextResponse.json( - { error: `Failed to download audio: ${audioResponse.statusText}` }, + { error: "Failed to download audio" }, { status: 500 } ); } + // Enforce a 50 MB size limit before buffering into memory + const MAX_AUDIO_SIZE = 50 * 1024 * 1024; + const contentLength = Number(audioResponse.headers.get("content-length") || "0"); + if (contentLength > MAX_AUDIO_SIZE) { + return NextResponse.json( + { error: "Audio file exceeds the 50 MB size limit" }, + { status: 400 } + ); + } + const audioBuffer = await audioResponse.arrayBuffer(); + if (audioBuffer.byteLength > MAX_AUDIO_SIZE) { + return NextResponse.json( + { error: "Audio file exceeds the 50 MB size limit" }, + { status: 400 } + ); + } const base64Audio = Buffer.from(audioBuffer).toString("base64"); const client = new GoogleGenAI({ apiKey }); @@ -134,10 +176,10 @@ Requirements: transcript: result.transcript, segments: result.segments, }); - } catch (error: any) { + } catch (error: unknown) { console.error("[AUDIO_PROCESS] Error:", error); return NextResponse.json( - { error: error?.message || "Failed to process audio" }, + { error: "Failed to process audio" }, { status: 500 } ); } diff --git a/src/components/modals/AudioRecorderDialog.tsx b/src/components/modals/AudioRecorderDialog.tsx index 97964861..513d6ca1 100644 --- a/src/components/modals/AudioRecorderDialog.tsx +++ b/src/components/modals/AudioRecorderDialog.tsx @@ -1,6 +1,6 @@ "use client"; -import { useState, useRef, useCallback, useMemo } from "react"; +import { useState, useRef, useCallback, useEffect } from "react"; import { Mic, Square, Upload, Loader2, Pause, Play } from "lucide-react"; import { Dialog, @@ -72,10 +72,18 @@ export function AudioRecorderDialog({ const [isSubmitting, setIsSubmitting] = useState(false); const [showConfirmClose, setShowConfirmClose] = useState(false); - // Stable blob URL for audio preview (avoids creating new URLs on every render) - const audioBlobUrl = useMemo(() => { - if (audioBlob) return URL.createObjectURL(audioBlob); - return null; + // Stable blob URL for audio preview — revoked on cleanup to avoid memory leaks + const [audioBlobUrl, setAudioBlobUrl] = useState(null); + useEffect(() => { + if (!audioBlob) { + setAudioBlobUrl(null); + return; + } + const url = URL.createObjectURL(audioBlob); + setAudioBlobUrl(url); + return () => { + URL.revokeObjectURL(url); + }; }, [audioBlob]); // Close dialog — if recording, just hide; if ready, confirm; if idle, fully close @@ -151,7 +159,7 @@ export function AudioRecorderDialog({ return ( <> - + { if (!isOpen) handleClose(); }}> e.stopPropagation()}> diff --git a/src/components/workspace-canvas/AudioCardContent.tsx b/src/components/workspace-canvas/AudioCardContent.tsx index 60c6a810..930c77fc 100644 --- a/src/components/workspace-canvas/AudioCardContent.tsx +++ b/src/components/workspace-canvas/AudioCardContent.tsx @@ -9,11 +9,9 @@ import { Loader2, Play, Pause, - Volume2, - VolumeX, FileText, - User, Globe, + RotateCcw, } from "lucide-react"; import type { Item, AudioData, AudioSegment } from "@/lib/workspace-state/types"; import { cn } from "@/lib/utils"; @@ -57,9 +55,10 @@ interface AudioCardContentProps { item: Item; isCompact?: boolean; isScrollLocked?: boolean; + onRetryProcessing?: (itemId: string) => void; } -export function AudioCardContent({ item, isCompact = false, isScrollLocked = false }: AudioCardContentProps) { +export function AudioCardContent({ item, isCompact = false, isScrollLocked = false, onRetryProcessing }: AudioCardContentProps) { const audioData = item.data as AudioData; // ── Loading / Error states ────────────────────────────────────────────── @@ -100,6 +99,16 @@ export function AudioCardContent({ item, isCompact = false, isScrollLocked = fal {audioData.error || "An error occurred while processing the audio."}

+ {onRetryProcessing && audioData.fileUrl && ( + + )}
); } @@ -126,7 +135,7 @@ function AudioCardComplete({ const [isPlaying, setIsPlaying] = useState(false); const [currentTime, setCurrentTime] = useState(0); const [duration, setDuration] = useState(audioData.duration ?? 0); - const [isMuted, setIsMuted] = useState(false); + const [isMuted] = useState(false); const [playbackRate, setPlaybackRate] = useState(1); const [showTranscript, setShowTranscript] = useState(false); const [activeSegmentIdx, setActiveSegmentIdx] = useState(-1); diff --git a/src/components/workspace-canvas/AudioWaveform.tsx b/src/components/workspace-canvas/AudioWaveform.tsx index f38f0d0a..55a32f55 100644 --- a/src/components/workspace-canvas/AudioWaveform.tsx +++ b/src/components/workspace-canvas/AudioWaveform.tsx @@ -73,7 +73,7 @@ export function AudioWaveform({ const barWidth = canvas.width / barCount; const gap = 2; - const samplesPerBar = Math.floor(bufferLength / barCount); + const samplesPerBar = Math.max(1, Math.floor(bufferLength / barCount)); ctx.fillStyle = barColor; diff --git a/src/components/workspace-canvas/WorkspaceCanvasDropzone.tsx b/src/components/workspace-canvas/WorkspaceCanvasDropzone.tsx index 723820bb..a60e7545 100644 --- a/src/components/workspace-canvas/WorkspaceCanvasDropzone.tsx +++ b/src/components/workspace-canvas/WorkspaceCanvasDropzone.tsx @@ -139,7 +139,7 @@ export function WorkspaceCanvasDropzone({ children }: WorkspaceCanvasDropzonePro ); try { - // Upload all files in parallel + // Upload all files in parallel, keeping the original File reference const uploadPromises = filteredFiles.map(async (file) => { try { const { url, filename } = await uploadFileToStorage(file); @@ -148,6 +148,7 @@ export function WorkspaceCanvasDropzone({ children }: WorkspaceCanvasDropzonePro filename: file.name, fileSize: file.size, name: file.name.replace(/\.pdf$/i, ''), // Remove .pdf extension for card name + originalFile: file, }; } catch (error) { console.error("Failed to upload file:", error); @@ -167,16 +168,16 @@ export function WorkspaceCanvasDropzone({ children }: WorkspaceCanvasDropzonePro toast.dismiss(loadingToastId); if (validResults.length > 0) { - // Separate files by type + // Separate files by type using the original file reference (avoids index misalignment) const pdfResults: typeof validResults = []; const imageResults: typeof validResults = []; const audioResults: typeof validResults = []; - validResults.forEach((result, index) => { - const file = filteredFiles[index]; - if (file.type === 'application/pdf') { + validResults.forEach((result) => { + const fileType = result.originalFile.type; + if (fileType === 'application/pdf') { pdfResults.push(result); - } else if (file.type.startsWith('audio/')) { + } else if (fileType.startsWith('audio/')) { audioResults.push(result); } else { imageResults.push(result); @@ -299,13 +300,12 @@ export function WorkspaceCanvasDropzone({ children }: WorkspaceCanvasDropzonePro // Create audio cards and trigger Gemini processing if (audioResults.length > 0) { - const audioCardDefinitions = audioResults.map((result, index) => { - const file = filteredFiles.find(f => f.name === result.filename) || filteredFiles[validResults.indexOf(result)]; + const audioCardDefinitions = audioResults.map((result) => { const audioData: Partial = { fileUrl: result.fileUrl, filename: result.filename, fileSize: result.fileSize, - mimeType: file?.type || 'audio/mpeg', + mimeType: result.originalFile.type || 'audio/mpeg', processingStatus: 'processing', }; return { @@ -321,14 +321,13 @@ export function WorkspaceCanvasDropzone({ children }: WorkspaceCanvasDropzonePro // Trigger Gemini processing for each audio file audioResults.forEach((result, index) => { const itemId = audioCreatedIds[index]; - const file = filteredFiles.find(f => f.name === result.filename) || filteredFiles[validResults.indexOf(result)]; fetch('/api/audio/process', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ fileUrl: result.fileUrl, filename: result.filename, - mimeType: file?.type || 'audio/mpeg', + mimeType: result.originalFile.type || 'audio/mpeg', }), }) .then(res => res.json()) diff --git a/src/components/workspace-canvas/WorkspaceContent.tsx b/src/components/workspace-canvas/WorkspaceContent.tsx index 7322f082..5b43ddcc 100644 --- a/src/components/workspace-canvas/WorkspaceContent.tsx +++ b/src/components/workspace-canvas/WorkspaceContent.tsx @@ -163,10 +163,12 @@ export default function WorkspaceContent({ const { itemId, summary, transcript, segments, error } = (e as CustomEvent).detail; if (!itemId) return; + const existingData = viewState.items.find((i) => i.id === itemId)?.data ?? {}; + if (error) { updateItem(itemId, { data: { - ...viewState.items.find((i) => i.id === itemId)?.data, + ...existingData, processingStatus: "failed", error, } as any, @@ -174,7 +176,7 @@ export default function WorkspaceContent({ } else { updateItem(itemId, { data: { - ...viewState.items.find((i) => i.id === itemId)?.data, + ...existingData, summary, transcript, segments, diff --git a/src/components/workspace-canvas/WorkspaceHeader.tsx b/src/components/workspace-canvas/WorkspaceHeader.tsx index a12329b7..ba0c555d 100644 --- a/src/components/workspace-canvas/WorkspaceHeader.tsx +++ b/src/components/workspace-canvas/WorkspaceHeader.tsx @@ -5,7 +5,7 @@ import { useState, useRef, useEffect, useCallback } from "react"; import Image from "next/image"; import Link from "next/link"; import { usePathname } from "next/navigation"; -import { Search, X, ChevronRight, ChevronDown, FolderOpen, Plus, Upload, FileText, Folder as FolderIcon, Settings, Share2, Play, MoreHorizontal, Globe, Brain, Maximize, File, Newspaper, ImageIcon, Mic } from "lucide-react"; +import { Search, X, ChevronDown, FolderOpen, Plus, Upload, FileText, Folder as FolderIcon, Settings, Share2, Play, Brain, File, Newspaper, ImageIcon, Mic } from "lucide-react"; import { LuBook, LuPanelLeftOpen } from "react-icons/lu"; import { PiCardsThreeBold } from "react-icons/pi"; import { cn } from "@/lib/utils"; diff --git a/src/hooks/audio/use-audio-recorder.ts b/src/hooks/audio/use-audio-recorder.ts index be2256d1..2f069f5e 100644 --- a/src/hooks/audio/use-audio-recorder.ts +++ b/src/hooks/audio/use-audio-recorder.ts @@ -84,7 +84,7 @@ export function useAudioRecorder(): UseAudioRecorderReturn { // Stop all tracks if (streamRef.current) { - streamRef.current.getTracks().forEach((track) => track.stop()); + streamRef.current.getTracks().forEach((track) => { track.stop(); }); streamRef.current = null; } }; @@ -100,6 +100,11 @@ export function useAudioRecorder(): UseAudioRecorderReturn { setIsPaused(false); startTimer(); } catch (err: any) { + // Clean up any acquired stream on failure + if (streamRef.current) { + streamRef.current.getTracks().forEach((track) => { track.stop(); }); + streamRef.current = null; + } if (err.name === "NotAllowedError") { setError("Microphone access denied. Please allow microphone access."); } else if (err.name === "NotFoundError") { @@ -151,7 +156,7 @@ export function useAudioRecorder(): UseAudioRecorderReturn { mediaRecorderRef.current.stop(); } if (streamRef.current) { - streamRef.current.getTracks().forEach((track) => track.stop()); + streamRef.current.getTracks().forEach((track) => { track.stop(); }); } }; }, [clearTimer]); diff --git a/src/lib/stores/audio-recording-store.ts b/src/lib/stores/audio-recording-store.ts index b45765ce..45e51fdc 100644 --- a/src/lib/stores/audio-recording-store.ts +++ b/src/lib/stores/audio-recording-store.ts @@ -15,6 +15,7 @@ export interface AudioRecordingState { _timerId: ReturnType | null; _audioContext: AudioContext | null; _analyser: AnalyserNode | null; + _resetting: boolean; // Dialog visibility (recording continues regardless) isDialogOpen: boolean; @@ -58,6 +59,7 @@ export const useAudioRecordingStore = create((set, get) => _timerId: null, _audioContext: null, _analyser: null, + _resetting: false, isDialogOpen: false, openDialog: () => set({ isDialogOpen: true }), @@ -73,7 +75,7 @@ export const useAudioRecordingStore = create((set, get) => state._mediaRecorder.stop(); } if (state._stream) { - state._stream.getTracks().forEach((t) => t.stop()); + state._stream.getTracks().forEach((t) => { t.stop(); }); } try { @@ -94,15 +96,17 @@ export const useAudioRecordingStore = create((set, get) => }; recorder.onstop = () => { - const blob = new Blob(chunks, { type: mimeType }); const s = get(); + // If resetRecording triggered this stop, skip — reset already cleaned up + if (s._resetting) return; + + const blob = new Blob(chunks, { type: mimeType }); if (s._timerId) clearInterval(s._timerId); if (s._stream) { - s._stream.getTracks().forEach((t) => t.stop()); + s._stream.getTracks().forEach((t) => { t.stop(); }); } - const currentState = get(); - if (currentState._audioContext) { - currentState._audioContext.close().catch(() => {}); + if (s._audioContext) { + s._audioContext.close().catch(() => {}); } set({ audioBlob: blob, @@ -120,9 +124,25 @@ export const useAudioRecordingStore = create((set, get) => }; recorder.onerror = () => { - set({ error: "Recording failed. Please try again.", isRecording: false }); const s = get(); if (s._timerId) clearInterval(s._timerId); + if (s._stream) { + s._stream.getTracks().forEach((t) => { t.stop(); }); + } + if (s._audioContext) { + s._audioContext.close().catch(() => {}); + } + set({ + error: "Recording failed. Please try again.", + isRecording: false, + isPaused: false, + _mediaRecorder: null, + _stream: null, + _chunks: [], + _timerId: null, + _audioContext: null, + _analyser: null, + }); }; recorder.start(1000); @@ -185,11 +205,13 @@ export const useAudioRecordingStore = create((set, get) => resetRecording: () => { const state = get(); if (state._timerId) clearInterval(state._timerId); + // Set _resetting flag so onstop handler knows to skip its state update + set({ _resetting: true }); if (state._mediaRecorder && state._mediaRecorder.state !== "inactive") { state._mediaRecorder.stop(); } if (state._stream) { - state._stream.getTracks().forEach((t) => t.stop()); + state._stream.getTracks().forEach((t) => { t.stop(); }); } if (state._audioContext) { state._audioContext.close().catch(() => {}); @@ -206,6 +228,7 @@ export const useAudioRecordingStore = create((set, get) => _timerId: null, _audioContext: null, _analyser: null, + _resetting: false, }); }, })); From c4853849eb5640c8b66ae8eadb4f627546f7a176 Mon Sep 17 00:00:00 2001 From: Urjit Chakraborty <135136842+urjitc@users.noreply.github.com> Date: Tue, 10 Feb 2026 01:56:20 -0500 Subject: [PATCH 7/9] feat: add retry logic --- .../workspace-canvas/AudioCardContent.tsx | 64 +++++++++++++++++-- .../workspace-canvas/WorkspaceContent.tsx | 14 +++- 2 files changed, 70 insertions(+), 8 deletions(-) diff --git a/src/components/workspace-canvas/AudioCardContent.tsx b/src/components/workspace-canvas/AudioCardContent.tsx index 930c77fc..3b6d5e81 100644 --- a/src/components/workspace-canvas/AudioCardContent.tsx +++ b/src/components/workspace-canvas/AudioCardContent.tsx @@ -55,11 +55,56 @@ interface AudioCardContentProps { item: Item; isCompact?: boolean; isScrollLocked?: boolean; - onRetryProcessing?: (itemId: string) => void; } -export function AudioCardContent({ item, isCompact = false, isScrollLocked = false, onRetryProcessing }: AudioCardContentProps) { +export function AudioCardContent({ item, isCompact = false, isScrollLocked = false }: AudioCardContentProps) { const audioData = item.data as AudioData; + const [isRetrying, setIsRetrying] = useState(false); + + const handleRetry = useCallback(() => { + if (!audioData.fileUrl || isRetrying) return; + setIsRetrying(true); + + // Immediately transition card to "processing" via the same event system + window.dispatchEvent( + new CustomEvent("audio-processing-complete", { + detail: { itemId: item.id, retrying: true }, + }) + ); + + fetch("/api/audio/process", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + fileUrl: audioData.fileUrl, + filename: audioData.filename, + mimeType: audioData.mimeType || "audio/webm", + }), + }) + .then((res) => res.json()) + .then((result) => { + window.dispatchEvent( + new CustomEvent("audio-processing-complete", { + detail: result.success + ? { + itemId: item.id, + summary: result.summary, + transcript: result.transcript, + segments: result.segments, + } + : { itemId: item.id, error: result.error || "Processing failed" }, + }) + ); + }) + .catch((err) => { + window.dispatchEvent( + new CustomEvent("audio-processing-complete", { + detail: { itemId: item.id, error: err.message || "Processing failed" }, + }) + ); + }) + .finally(() => setIsRetrying(false)); + }, [audioData.fileUrl, audioData.filename, audioData.mimeType, item.id, isRetrying]); // ── Loading / Error states ────────────────────────────────────────────── @@ -99,14 +144,19 @@ export function AudioCardContent({ item, isCompact = false, isScrollLocked = fal {audioData.error || "An error occurred while processing the audio."}

- {onRetryProcessing && audioData.fileUrl && ( + {audioData.fileUrl && ( )}
diff --git a/src/components/workspace-canvas/WorkspaceContent.tsx b/src/components/workspace-canvas/WorkspaceContent.tsx index 5b43ddcc..8139d4e6 100644 --- a/src/components/workspace-canvas/WorkspaceContent.tsx +++ b/src/components/workspace-canvas/WorkspaceContent.tsx @@ -160,11 +160,23 @@ export default function WorkspaceContent({ // Listen for audio processing completion events useEffect(() => { const handleAudioComplete = (e: Event) => { - const { itemId, summary, transcript, segments, error } = (e as CustomEvent).detail; + const { itemId, summary, transcript, segments, error, retrying } = (e as CustomEvent).detail; if (!itemId) return; const existingData = viewState.items.find((i) => i.id === itemId)?.data ?? {}; + // Retry: transition back to "processing" state + if (retrying) { + updateItem(itemId, { + data: { + ...existingData, + processingStatus: "processing", + error: undefined, + } as any, + }); + return; + } + if (error) { updateItem(itemId, { data: { From c04ba045d8251a05aff1fe9abf57ab1c9da19c64 Mon Sep 17 00:00:00 2001 From: Urjit Chakraborty <135136842+urjitc@users.noreply.github.com> Date: Tue, 10 Feb 2026 01:58:57 -0500 Subject: [PATCH 8/9] fix: remove new tags --- src/components/assistant-ui/thread.tsx | 3 --- src/components/workspace-canvas/WorkspaceHeader.tsx | 3 --- 2 files changed, 6 deletions(-) diff --git a/src/components/assistant-ui/thread.tsx b/src/components/assistant-ui/thread.tsx index f5bcda85..1f44a1e7 100644 --- a/src/components/assistant-ui/thread.tsx +++ b/src/components/assistant-ui/thread.tsx @@ -893,9 +893,6 @@ const ComposerAction: FC = ({ items }) => { > {getModelIcon(selectedModel.id)} {getModelDisplayName(selectedModel.id)} - - NEW - e.preventDefault()}> diff --git a/src/components/workspace-canvas/WorkspaceHeader.tsx b/src/components/workspace-canvas/WorkspaceHeader.tsx index ba0c555d..d8cba625 100644 --- a/src/components/workspace-canvas/WorkspaceHeader.tsx +++ b/src/components/workspace-canvas/WorkspaceHeader.tsx @@ -873,9 +873,6 @@ export default function WorkspaceHeader({ className="h-8 px-2 text-muted-foreground hover:text-foreground font-normal relative" > Share - - NEW - )} From 11b293d2376173823bafa064607fb6c527222070 Mon Sep 17 00:00:00 2001 From: Urjit Chakraborty <135136842+urjitc@users.noreply.github.com> Date: Tue, 10 Feb 2026 02:11:09 -0500 Subject: [PATCH 9/9] feat: reusable new feature badge --- src/components/ui/new-feature-badge.tsx | 77 +++++++++++++ .../workspace-canvas/WorkspaceHeader.tsx | 7 +- .../workspace-canvas/workspace-menu-items.tsx | 5 +- src/lib/utils/new-feature.ts | 109 ++++++++++++++++++ 4 files changed, 196 insertions(+), 2 deletions(-) create mode 100644 src/components/ui/new-feature-badge.tsx create mode 100644 src/lib/utils/new-feature.ts diff --git a/src/components/ui/new-feature-badge.tsx b/src/components/ui/new-feature-badge.tsx new file mode 100644 index 00000000..dcc3d68a --- /dev/null +++ b/src/components/ui/new-feature-badge.tsx @@ -0,0 +1,77 @@ +"use client"; + +import React from "react"; +import { Gift } from "lucide-react"; +import { useNewFeature } from "@/lib/utils/new-feature"; +import { cn } from "@/lib/utils"; + +interface NewFeatureBadgeProps { + /** Unique key identifying this feature */ + featureKey: string; + /** How long (in ms) the badge stays visible. Default: 14 days */ + ttl?: number; + /** If provided, the badge won't show before this date */ + startDate?: Date; + /** If provided, the badge won't show after this date */ + endDate?: Date; + /** Dismiss when the wrapped element is clicked. Default: true */ + dismissOnClick?: boolean; + /** The content to wrap */ + children: React.ReactNode; + /** Additional className for the wrapper */ + className?: string; + /** Style variant for the badge. Default: "dot" */ + variant?: "dot" | "badge" | "icon"; + /** Custom label text for the "badge" variant. Default: "New" */ + label?: string; +} + +export function NewFeatureBadge({ + featureKey, + ttl, + startDate, + endDate, + dismissOnClick = true, + children, + className, + variant = "dot", + label = "New", +}: NewFeatureBadgeProps) { + const { isNew, dismiss } = useNewFeature({ featureKey, ttl, startDate, endDate }); + + const handleClick = () => { + if (dismissOnClick && isNew) { + dismiss(); + } + }; + + if (!isNew) { + return <>{children}; + } + + return ( + + {children} + + {variant === "dot" && ( + + + + + )} + + {variant === "badge" && ( + + {label} + + )} + + {variant === "icon" && ( + + )} + + ); +} diff --git a/src/components/workspace-canvas/WorkspaceHeader.tsx b/src/components/workspace-canvas/WorkspaceHeader.tsx index d8cba625..70c8cb0f 100644 --- a/src/components/workspace-canvas/WorkspaceHeader.tsx +++ b/src/components/workspace-canvas/WorkspaceHeader.tsx @@ -11,6 +11,7 @@ import { PiCardsThreeBold } from "react-icons/pi"; import { cn } from "@/lib/utils"; import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; import { SidebarTrigger } from "@/components/ui/sidebar"; +import { NewFeatureBadge } from "@/components/ui/new-feature-badge"; import { formatKeyboardShortcut } from "@/lib/utils/keyboard-shortcut"; import WorkspaceSaveIndicator from "@/components/workspace/WorkspaceSaveIndicator"; import ChatFloatingButton from "@/components/chat/ChatFloatingButton"; @@ -958,7 +959,11 @@ export default function WorkspaceHeader({ data-tour="add-card-button" > - {!isCompactMode && New} + {!isCompactMode && ( + + New + + )} diff --git a/src/components/workspace-canvas/workspace-menu-items.tsx b/src/components/workspace-canvas/workspace-menu-items.tsx index 078e42a6..1b6cb56d 100644 --- a/src/components/workspace-canvas/workspace-menu-items.tsx +++ b/src/components/workspace-canvas/workspace-menu-items.tsx @@ -6,6 +6,7 @@ import { LuBook } from "react-icons/lu"; import { PiCardsThreeBold } from "react-icons/pi"; import { useAudioRecordingStore } from "@/lib/stores/audio-recording-store"; import { toast } from "sonner"; +import { NewFeatureBadge } from "@/components/ui/new-feature-badge"; export interface WorkspaceMenuCallbacks { onCreateNote: () => void; @@ -94,7 +95,9 @@ export function renderWorkspaceMenuItems({ >
- Audio + + Audio + Lecture/Meeting
diff --git a/src/lib/utils/new-feature.ts b/src/lib/utils/new-feature.ts new file mode 100644 index 00000000..55954373 --- /dev/null +++ b/src/lib/utils/new-feature.ts @@ -0,0 +1,109 @@ +"use client"; + +import { useState, useEffect, useCallback } from "react"; + +const STORAGE_PREFIX = "thinkex_feature_seen_"; + +interface NewFeatureOptions { + /** Unique key identifying this feature */ + featureKey: string; + /** How long (in ms) the badge stays visible after first seen. Default: 14 days */ + ttl?: number; + /** If provided, the badge won't show before this date */ + startDate?: Date; + /** If provided, the badge won't show after this date (regardless of TTL) */ + endDate?: Date; +} + +interface NewFeatureState { + /** Whether the "new" badge should be visible */ + isNew: boolean; + /** Call this to permanently dismiss the badge for this user */ + dismiss: () => void; +} + +const DEFAULT_TTL = 14 * 24 * 60 * 60 * 1000; // 14 days + +function getStorageKey(featureKey: string) { + return `${STORAGE_PREFIX}${featureKey}`; +} + +function isFeatureDismissed(featureKey: string): boolean { + if (typeof window === "undefined") return true; + try { + const raw = localStorage.getItem(getStorageKey(featureKey)); + if (!raw) return false; + const data = JSON.parse(raw) as { dismissedAt: number }; + return !!data.dismissedAt; + } catch { + return false; + } +} + +function dismissFeature(featureKey: string) { + if (typeof window === "undefined") return; + try { + localStorage.setItem( + getStorageKey(featureKey), + JSON.stringify({ dismissedAt: Date.now() }), + ); + } catch { + // localStorage full or unavailable — silently ignore + } +} + +function isWithinWindow( + ttl: number, + startDate?: Date, + endDate?: Date, +): boolean { + const now = Date.now(); + if (startDate && now < startDate.getTime()) return false; + if (endDate && now > endDate.getTime()) return false; + + // If there's a startDate, check TTL from startDate; otherwise always in window + if (startDate) { + return now - startDate.getTime() < ttl; + } + return true; +} + +/** + * Hook that tracks whether a feature should show a "new" badge. + * + * Uses localStorage so the badge is per-device. Once dismissed (either + * manually or after the TTL / endDate expires), it won't reappear. + * + * @example + * const { isNew, dismiss } = useNewFeature({ featureKey: "audio-upload", ttl: 14 * 24 * 60 * 60 * 1000 }); + */ +export function useNewFeature({ + featureKey, + ttl = DEFAULT_TTL, + startDate, + endDate, +}: NewFeatureOptions): NewFeatureState { + const [isNew, setIsNew] = useState(false); + + useEffect(() => { + if (isFeatureDismissed(featureKey)) { + setIsNew(false); + return; + } + const inWindow = isWithinWindow(ttl, startDate, endDate); + setIsNew(inWindow); + }, [featureKey, ttl, startDate, endDate]); + + const dismiss = useCallback(() => { + dismissFeature(featureKey); + setIsNew(false); + }, [featureKey]); + + return { isNew, dismiss }; +} + +/** Imperative helper — useful outside of React components */ +export const newFeatureUtils = { + isDismissed: isFeatureDismissed, + dismiss: dismissFeature, +};