diff --git a/src/app/api/audio/process/route.ts b/src/app/api/audio/process/route.ts new file mode 100644 index 00000000..7cac2cec --- /dev/null +++ b/src/app/api/audio/process/route.ts @@ -0,0 +1,199 @@ +import { GoogleGenAI, Type } from "@google/genai"; +import { NextRequest, NextResponse } from "next/server"; +import { auth } from "@/lib/auth"; +import { headers } from "next/headers"; + +export const dynamic = "force-dynamic"; + +const apiKey = process.env.GOOGLE_GENERATIVE_AI_API_KEY; + +/** + * POST /api/audio/process + * Receives an audio file URL, downloads it, uploads to Gemini Files API, + * and returns a structured transcript + summary. + */ +export async function POST(req: NextRequest) { + try { + // Auth check + const session = await auth.api.getSession({ + headers: await headers(), + }); + if (!session) { + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + } + + if (!apiKey) { + return NextResponse.json( + { error: "GOOGLE_GENERATIVE_AI_API_KEY is not set" }, + { status: 500 } + ); + } + + const body = await req.json(); + const { fileUrl, filename, mimeType } = body; + + if (!fileUrl) { + return NextResponse.json( + { error: "fileUrl is required" }, + { status: 400 } + ); + } + + // Validate URL origin to prevent SSRF + const allowedHosts = [ + process.env.NEXT_PUBLIC_SUPABASE_URL + ? new URL(process.env.NEXT_PUBLIC_SUPABASE_URL).hostname + : null, + ].filter(Boolean) as string[]; + + let parsedUrl: URL; + try { + parsedUrl = new URL(fileUrl); + } catch { + return NextResponse.json( + { error: "Invalid fileUrl" }, + { status: 400 } + ); + } + + if ( + !allowedHosts.some((host) => parsedUrl.hostname === host || parsedUrl.hostname.endsWith(`.${host}`)) + ) { + return NextResponse.json( + { error: "fileUrl origin is not allowed" }, + { status: 400 } + ); + } + + // Determine MIME type + const audioMimeType = mimeType || guessMimeType(filename || fileUrl); + + // Download the audio file from storage + const audioResponse = await fetch(fileUrl); + if (!audioResponse.ok) { + return NextResponse.json( + { error: "Failed to download audio" }, + { status: 500 } + ); + } + + // Enforce a 50 MB size limit before buffering into memory + const MAX_AUDIO_SIZE = 50 * 1024 * 1024; + const contentLength = Number(audioResponse.headers.get("content-length") || "0"); + if (contentLength > MAX_AUDIO_SIZE) { + return NextResponse.json( + { error: "Audio file exceeds the 50 MB size limit" }, + { status: 400 } + ); + } + + const audioBuffer = await audioResponse.arrayBuffer(); + if (audioBuffer.byteLength > MAX_AUDIO_SIZE) { + return NextResponse.json( + { error: "Audio file exceeds the 50 MB size limit" }, + { status: 400 } + ); + } + const base64Audio = Buffer.from(audioBuffer).toString("base64"); + + const client = new GoogleGenAI({ apiKey }); + + const prompt = `Process this audio file and generate a detailed transcription and summary. + +Requirements: +1. Provide a comprehensive summary of the entire audio content. +2. Identify distinct speakers (e.g., Speaker 1, Speaker 2, or names if context allows). +3. Provide accurate timestamps for each segment (Format: MM:SS). +4. Provide the full plain-text transcript combining all segments.`; + + const response = await client.models.generateContent({ + model: "gemini-2.5-flash", + contents: [ + { + role: "user", + parts: [ + { + inlineData: { + mimeType: audioMimeType, + data: base64Audio, + }, + }, + { text: prompt }, + ], + }, + ], + config: { + responseMimeType: "application/json", + responseSchema: { + type: Type.OBJECT, + properties: { + summary: { + type: Type.STRING, + description: "A concise summary of the audio content.", + }, + transcript: { + type: Type.STRING, + description: + "Full plain-text transcript of the audio, combining all segments.", + }, + segments: { + type: Type.ARRAY, + description: + "List of transcribed segments with speaker and timestamp.", + items: { + type: Type.OBJECT, + properties: { + speaker: { type: Type.STRING }, + timestamp: { type: Type.STRING }, + content: { type: Type.STRING }, + }, + required: [ + "speaker", + "timestamp", + "content", + ], + }, + }, + }, + required: ["summary", "transcript", "segments"], + }, + }, + }); + + const resultText = response.text; + if (!resultText) { + return NextResponse.json( + { error: "No response from Gemini" }, + { status: 500 } + ); + } + + const result = JSON.parse(resultText); + + return NextResponse.json({ + success: true, + summary: result.summary, + transcript: result.transcript, + segments: result.segments, + }); + } catch (error: unknown) { + console.error("[AUDIO_PROCESS] Error:", error); + return NextResponse.json( + { error: "Failed to process audio" }, + { status: 500 } + ); + } +} + +function guessMimeType(filenameOrUrl: string): string { + const lower = filenameOrUrl.toLowerCase(); + if (lower.endsWith(".mp3")) return "audio/mp3"; + if (lower.endsWith(".wav")) return "audio/wav"; + if (lower.endsWith(".ogg")) return "audio/ogg"; + if (lower.endsWith(".aac")) return "audio/aac"; + if (lower.endsWith(".flac")) return "audio/flac"; + if (lower.endsWith(".aiff")) return "audio/aiff"; + if (lower.endsWith(".webm")) return "audio/webm"; + if (lower.endsWith(".m4a")) return "audio/mp4"; + return "audio/mp3"; // Default fallback +} diff --git a/src/components/assistant-ui/thread.tsx b/src/components/assistant-ui/thread.tsx index f5bcda85..1f44a1e7 100644 --- a/src/components/assistant-ui/thread.tsx +++ b/src/components/assistant-ui/thread.tsx @@ -893,9 +893,6 @@ const ComposerAction: FC = ({ items }) => { > {getModelIcon(selectedModel.id)} {getModelDisplayName(selectedModel.id)} - - NEW - e.preventDefault()}> diff --git a/src/components/chat/MentionMenu.tsx b/src/components/chat/MentionMenu.tsx index 66eaec6d..0826fe37 100644 --- a/src/components/chat/MentionMenu.tsx +++ b/src/components/chat/MentionMenu.tsx @@ -12,6 +12,7 @@ import { LayoutGrid, Brain, ImageIcon, + Mic, } from "lucide-react"; import { PiCardsThreeBold } from "react-icons/pi"; import { Popover, PopoverContent, PopoverAnchor } from "@/components/ui/popover"; @@ -39,6 +40,8 @@ function getCardTypeIcon(type: CardType) { return ; case "image": return ; + case "audio": + return ; default: return ; } diff --git a/src/components/modals/AudioRecorderDialog.tsx b/src/components/modals/AudioRecorderDialog.tsx new file mode 100644 index 00000000..513d6ca1 --- /dev/null +++ b/src/components/modals/AudioRecorderDialog.tsx @@ -0,0 +1,380 @@ +"use client"; + +import { useState, useRef, useCallback, useEffect } from "react"; +import { Mic, Square, Upload, Loader2, Pause, Play } from "lucide-react"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog"; +import { + AlertDialog, + AlertDialogAction, + AlertDialogCancel, + AlertDialogContent, + AlertDialogDescription, + AlertDialogFooter, + AlertDialogHeader, + AlertDialogTitle, +} from "@/components/ui/alert-dialog"; +import { Button } from "@/components/ui/button"; +import { useAudioRecordingStore } from "@/lib/stores/audio-recording-store"; +import { cn } from "@/lib/utils"; +import { AudioWaveform } from "@/components/workspace-canvas/AudioWaveform"; + +const ACCEPTED_AUDIO_TYPES = [ + "audio/mp3", + "audio/mpeg", + "audio/wav", + "audio/ogg", + "audio/aac", + "audio/flac", + "audio/aiff", + "audio/webm", + "audio/mp4", + "audio/x-m4a", +]; + +const ACCEPTED_EXTENSIONS = ".mp3,.wav,.ogg,.aac,.flac,.aiff,.webm,.m4a"; + +interface AudioRecorderDialogProps { + open: boolean; + onOpenChange: (open: boolean) => void; + onAudioReady: (file: File) => void; +} + +function formatDuration(seconds: number): string { + const mins = Math.floor(seconds / 60); + const secs = seconds % 60; + return `${mins.toString().padStart(2, "0")}:${secs.toString().padStart(2, "0")}`; +} + +export function AudioRecorderDialog({ + open, + onOpenChange, + onAudioReady, +}: AudioRecorderDialogProps) { + const isRecording = useAudioRecordingStore((s) => s.isRecording); + const isPaused = useAudioRecordingStore((s) => s.isPaused); + const duration = useAudioRecordingStore((s) => s.duration); + const audioBlob = useAudioRecordingStore((s) => s.audioBlob); + const error = useAudioRecordingStore((s) => s.error); + const startRecording = useAudioRecordingStore((s) => s.startRecording); + const stopRecording = useAudioRecordingStore((s) => s.stopRecording); + const pauseRecording = useAudioRecordingStore((s) => s.pauseRecording); + const resumeRecording = useAudioRecordingStore((s) => s.resumeRecording); + const resetRecording = useAudioRecordingStore((s) => s.resetRecording); + + const fileInputRef = useRef(null); + const [isSubmitting, setIsSubmitting] = useState(false); + const [showConfirmClose, setShowConfirmClose] = useState(false); + + // Stable blob URL for audio preview — revoked on cleanup to avoid memory leaks + const [audioBlobUrl, setAudioBlobUrl] = useState(null); + useEffect(() => { + if (!audioBlob) { + setAudioBlobUrl(null); + return; + } + const url = URL.createObjectURL(audioBlob); + setAudioBlobUrl(url); + return () => { + URL.revokeObjectURL(url); + }; + }, [audioBlob]); + + // Close dialog — if recording, just hide; if ready, confirm; if idle, fully close + const handleClose = useCallback(() => { + if (isRecording) { + // Just hide the dialog — recording continues in the background + onOpenChange(false); + return; + } + if (audioBlob) { + // Show confirmation dialog - recording will be lost + setShowConfirmClose(true); + return; + } + // Not recording and no blob — fully reset and close + resetRecording(); + setIsSubmitting(false); + onOpenChange(false); + }, [isRecording, audioBlob, resetRecording, onOpenChange]); + + const handleFileSelect = useCallback( + (e: React.ChangeEvent) => { + const file = e.target.files?.[0]; + if (!file) return; + + if ( + !ACCEPTED_AUDIO_TYPES.includes(file.type) && + !file.name.match(/\.(mp3|wav|ogg|aac|flac|aiff|webm|m4a)$/i) + ) { + return; + } + + setIsSubmitting(true); + onAudioReady(file); + resetRecording(); + setIsSubmitting(false); + onOpenChange(false); + }, + [onAudioReady, resetRecording, onOpenChange] + ); + + const handleSubmitRecording = useCallback(() => { + if (!audioBlob) return; + + const mimeType = audioBlob.type; + let ext = "webm"; + if (mimeType.includes("mp4")) ext = "m4a"; + else if (mimeType.includes("ogg")) ext = "ogg"; + else if (mimeType.includes("wav")) ext = "wav"; + + const filename = `recording-${Date.now()}.${ext}`; + const file = new File([audioBlob], filename, { type: mimeType }); + + setIsSubmitting(true); + onAudioReady(file); + resetRecording(); + setIsSubmitting(false); + onOpenChange(false); + }, [audioBlob, onAudioReady, resetRecording, onOpenChange]); + + const handleConfirmClose = useCallback(() => { + // User confirmed - discard recording and close + resetRecording(); + setIsSubmitting(false); + setShowConfirmClose(false); + onOpenChange(false); + }, [resetRecording, onOpenChange]); + + const handleCancelClose = useCallback(() => { + // User cancelled - keep dialog open + setShowConfirmClose(false); + }, []); + + return ( + <> + { if (!isOpen) handleClose(); }}> + e.stopPropagation()}> + + + {isRecording + ? "Recording in progress..." + : audioBlob + ? "Recording ready" + : "Record audio" + } + + + {isRecording + ? "You can close this dialog, recording continues in the background." + : audioBlob + ? "" + : "Generate AI-powered summaries, transcripts, and timestamps from your audio"} + + + +
+ {/* Recording Visualizer */} + {isRecording ? ( +
+ +
+ + + {formatDuration(duration)} + +
+
+ ) : ( +
+ {audioBlob ? ( +
+ + + {formatDuration(duration)} + +
+ ) : ( + + )} +
+ )} + + {/* Controls */} +
+ {!isRecording && !audioBlob && ( + + )} + + {isRecording && ( + <> + + + + )} + + {audioBlob && !isRecording && ( + <> + + + + )} +
+ + {/* Preview audio playback */} + {audioBlobUrl && !isRecording && ( +
+ + {/* Confirmation Dialog */} + + + + Discard recording? + + Are you sure you want to close? Your recording will be lost if you don't save it. + + + + + Cancel + + + Discard Recording + + + + + + ); +} + +export default AudioRecorderDialog; diff --git a/src/components/modals/MoveToDialog.tsx b/src/components/modals/MoveToDialog.tsx index 67b9ca0f..b8259a1c 100644 --- a/src/components/modals/MoveToDialog.tsx +++ b/src/components/modals/MoveToDialog.tsx @@ -1,7 +1,7 @@ "use client"; import { useState, useMemo, useCallback, useEffect } from "react"; -import { ChevronRight, Folder as FolderIcon, FolderOpen, Check, FileText, File, Play, Brain, ImageIcon } from "lucide-react"; +import { ChevronRight, Folder as FolderIcon, FolderOpen, Check, FileText, File, Play, Brain, ImageIcon, Mic } from "lucide-react"; import { PiCardsThreeBold } from "react-icons/pi"; import { Dialog, @@ -36,6 +36,8 @@ function getCardTypeIcon(type: CardType) { return ; case "image": return ; + case "audio": + return ; default: return ; } diff --git a/src/components/ui/new-feature-badge.tsx b/src/components/ui/new-feature-badge.tsx new file mode 100644 index 00000000..dcc3d68a --- /dev/null +++ b/src/components/ui/new-feature-badge.tsx @@ -0,0 +1,77 @@ +"use client"; + +import React from "react"; +import { Gift } from "lucide-react"; +import { useNewFeature } from "@/lib/utils/new-feature"; +import { cn } from "@/lib/utils"; + +interface NewFeatureBadgeProps { + /** Unique key identifying this feature */ + featureKey: string; + /** How long (in ms) the badge stays visible. Default: 14 days */ + ttl?: number; + /** If provided, the badge won't show before this date */ + startDate?: Date; + /** If provided, the badge won't show after this date */ + endDate?: Date; + /** Dismiss when the wrapped element is clicked. Default: true */ + dismissOnClick?: boolean; + /** The content to wrap */ + children: React.ReactNode; + /** Additional className for the wrapper */ + className?: string; + /** Style variant for the badge. Default: "dot" */ + variant?: "dot" | "badge" | "icon"; + /** Custom label text for the "badge" variant. Default: "New" */ + label?: string; +} + +export function NewFeatureBadge({ + featureKey, + ttl, + startDate, + endDate, + dismissOnClick = true, + children, + className, + variant = "dot", + label = "New", +}: NewFeatureBadgeProps) { + const { isNew, dismiss } = useNewFeature({ featureKey, ttl, startDate, endDate }); + + const handleClick = () => { + if (dismissOnClick && isNew) { + dismiss(); + } + }; + + if (!isNew) { + return <>{children}; + } + + return ( + + {children} + + {variant === "dot" && ( + + + + + )} + + {variant === "badge" && ( + + {label} + + )} + + {variant === "icon" && ( + + )} + + ); +} diff --git a/src/components/workspace-canvas/AudioCardContent.tsx b/src/components/workspace-canvas/AudioCardContent.tsx new file mode 100644 index 00000000..3b6d5e81 --- /dev/null +++ b/src/components/workspace-canvas/AudioCardContent.tsx @@ -0,0 +1,615 @@ +"use client"; + +import { useState, useRef, useEffect, useCallback } from "react"; +import { + Mic, + ChevronDown, + ChevronUp, + AlertCircle, + Loader2, + Play, + Pause, + FileText, + Globe, + RotateCcw, +} from "lucide-react"; +import type { Item, AudioData, AudioSegment } from "@/lib/workspace-state/types"; +import { cn } from "@/lib/utils"; + +// ─── Helpers ──────────────────────────────────────────────────────────────── + +/** Parse "MM:SS" or "H:MM:SS" timestamp to seconds */ +function parseTimestamp(ts: string): number { + const parts = ts.split(":").map(Number); + if (parts.length === 3) return parts[0] * 3600 + parts[1] * 60 + parts[2]; + if (parts.length === 2) return parts[0] * 60 + parts[1]; + return 0; +} + +/** Format seconds to MM:SS */ +function fmtTime(s: number): string { + if (!isFinite(s) || s < 0) return "0:00"; + const mins = Math.floor(s / 60); + const secs = Math.floor(s % 60); + return `${mins}:${secs.toString().padStart(2, "0")}`; +} + +const SPEAKER_COLORS = [ + "bg-blue-500/15 text-blue-700 dark:text-blue-400", + "bg-purple-500/15 text-purple-700 dark:text-purple-400", + "bg-emerald-500/15 text-emerald-700 dark:text-emerald-400", + "bg-amber-500/15 text-amber-700 dark:text-amber-400", + "bg-rose-500/15 text-rose-700 dark:text-rose-400", + "bg-cyan-500/15 text-cyan-700 dark:text-cyan-400", +]; + +const EMOTION_ICONS: Record = { + happy: "😊", + sad: "😢", + angry: "😠", +}; + +// ─── Component ────────────────────────────────────────────────────────────── + +interface AudioCardContentProps { + item: Item; + isCompact?: boolean; + isScrollLocked?: boolean; +} + +export function AudioCardContent({ item, isCompact = false, isScrollLocked = false }: AudioCardContentProps) { + const audioData = item.data as AudioData; + const [isRetrying, setIsRetrying] = useState(false); + + const handleRetry = useCallback(() => { + if (!audioData.fileUrl || isRetrying) return; + setIsRetrying(true); + + // Immediately transition card to "processing" via the same event system + window.dispatchEvent( + new CustomEvent("audio-processing-complete", { + detail: { itemId: item.id, retrying: true }, + }) + ); + + fetch("/api/audio/process", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + fileUrl: audioData.fileUrl, + filename: audioData.filename, + mimeType: audioData.mimeType || "audio/webm", + }), + }) + .then((res) => res.json()) + .then((result) => { + window.dispatchEvent( + new CustomEvent("audio-processing-complete", { + detail: result.success + ? { + itemId: item.id, + summary: result.summary, + transcript: result.transcript, + segments: result.segments, + } + : { itemId: item.id, error: result.error || "Processing failed" }, + }) + ); + }) + .catch((err) => { + window.dispatchEvent( + new CustomEvent("audio-processing-complete", { + detail: { itemId: item.id, error: err.message || "Processing failed" }, + }) + ); + }) + .finally(() => setIsRetrying(false)); + }, [audioData.fileUrl, audioData.filename, audioData.mimeType, item.id, isRetrying]); + + // ── Loading / Error states ────────────────────────────────────────────── + + if (audioData.processingStatus === "uploading") { + return ( +
+ +

Uploading audio...

+
+ ); + } + + if (audioData.processingStatus === "processing") { + return ( +
+
+ + +
+
+

Analyzing audio...

+

+ Generating transcript and summary with Gemini +

+
+
+ ); + } + + if (audioData.processingStatus === "failed") { + return ( +
+ +
+

Processing failed

+

+ {audioData.error || "An error occurred while processing the audio."} +

+
+ {audioData.fileUrl && ( + + )} +
+ ); + } + + // ── Complete state ────────────────────────────────────────────────────── + + return ; +} + +// ─── Complete state (with player) ─────────────────────────────────────────── + +function AudioCardComplete({ + audioData, + isCompact, + isScrollLocked, +}: { + audioData: AudioData; + isCompact: boolean; + isScrollLocked?: boolean; +}) { + const audioRef = useRef(null); + const progressRef = useRef(null); + + const [isPlaying, setIsPlaying] = useState(false); + const [currentTime, setCurrentTime] = useState(0); + const [duration, setDuration] = useState(audioData.duration ?? 0); + const [isMuted] = useState(false); + const [playbackRate, setPlaybackRate] = useState(1); + const [showTranscript, setShowTranscript] = useState(false); + const [activeSegmentIdx, setActiveSegmentIdx] = useState(-1); + const [showSummary, setShowSummary] = useState(true); + const [showTimeline, setShowTimeline] = useState(true); + + // Build speaker → color map + const speakerColorMap = useRef(new Map()); + if (audioData.segments) { + let colorIdx = 0; + for (const seg of audioData.segments) { + if (!speakerColorMap.current.has(seg.speaker)) { + speakerColorMap.current.set( + seg.speaker, + SPEAKER_COLORS[colorIdx % SPEAKER_COLORS.length] + ); + colorIdx++; + } + } + } + + // ── Audio event handlers (only for non-compact view) ──────────────────────── + + useEffect(() => { + if (isCompact) return; // Skip audio setup in compact view + + const audio = audioRef.current; + if (!audio) return; + + const onTimeUpdate = () => setCurrentTime(audio.currentTime); + const onLoadedMetadata = () => { + if (audio.duration && isFinite(audio.duration)) { + setDuration(audio.duration); + } + }; + const onPlay = () => setIsPlaying(true); + const onPause = () => setIsPlaying(false); + const onEnded = () => { + setIsPlaying(false); + setCurrentTime(0); + }; + + audio.addEventListener("timeupdate", onTimeUpdate); + audio.addEventListener("loadedmetadata", onLoadedMetadata); + audio.addEventListener("play", onPlay); + audio.addEventListener("pause", onPause); + audio.addEventListener("ended", onEnded); + + return () => { + audio.removeEventListener("timeupdate", onTimeUpdate); + audio.removeEventListener("loadedmetadata", onLoadedMetadata); + audio.removeEventListener("play", onPlay); + audio.removeEventListener("pause", onPause); + audio.removeEventListener("ended", onEnded); + // Pause audio when component unmounts (e.g. closing modal mid-play) + audio.pause(); + }; + }, [isCompact]); + + // Update audio playback rate when it changes + useEffect(() => { + if (audioRef.current && !isCompact) { + audioRef.current.playbackRate = playbackRate; + } + }, [playbackRate, isCompact]); + + // Track active segment based on current time (only for non-compact view) + useEffect(() => { + if (isCompact || !audioData.segments || audioData.segments.length === 0) return; + let active = -1; + for (let i = 0; i < audioData.segments.length; i++) { + const segTime = parseTimestamp(audioData.segments[i].timestamp); + if (currentTime >= segTime) active = i; + else break; + } + setActiveSegmentIdx(active); + }, [currentTime, audioData.segments, isCompact]); + + const togglePlay = useCallback(() => { + if (isCompact) return; // Disable in compact view + + const audio = audioRef.current; + if (!audio) return; + if (audio.paused) { + audio.play().catch(() => { + // play() can reject if interrupted by another call or browser policy + }); + } else { + audio.pause(); + } + // isPlaying state is driven by 'play'/'pause' events — no manual set here + }, [isCompact]); + + // Spacebar to toggle play/pause (only for non-compact/modal view) + useEffect(() => { + if (isCompact) return; + + const onKeyDown = (e: KeyboardEvent) => { + if (e.code !== "Space") return; + // Don't hijack spacebar when user is typing in an input/textarea/button + const tag = (e.target as HTMLElement)?.tagName; + if (tag === "INPUT" || tag === "TEXTAREA" || tag === "SELECT") return; + e.preventDefault(); + togglePlay(); + }; + + window.addEventListener("keydown", onKeyDown); + return () => window.removeEventListener("keydown", onKeyDown); + }, [isCompact, togglePlay]); + + const seekTo = useCallback((seconds: number) => { + if (isCompact) return; // Disable in compact view + + const audio = audioRef.current; + if (!audio) return; + audio.currentTime = seconds; + setCurrentTime(seconds); + if (audio.paused) { + audio.play().catch(() => { + // play() can reject if interrupted by another call or browser policy + }); + } + // isPlaying state is driven by 'play'/'pause' events — no manual set here + }, [isCompact]); + + const handleProgressClick = useCallback( + (e: React.MouseEvent) => { + if (isCompact) return; // Disable in compact view + + const bar = progressRef.current; + if (!bar || !duration) return; + const rect = bar.getBoundingClientRect(); + const pct = Math.max(0, Math.min(1, (e.clientX - rect.left) / rect.width)); + seekTo(pct * duration); + }, + [duration, seekTo, isCompact] + ); + + const progress = duration > 0 ? (currentTime / duration) * 100 : 0; + + return ( +
+ {/* Hidden audio element (only for non-compact view) */} + {!isCompact &&
); } diff --git a/src/components/workspace-canvas/WorkspaceSection.tsx b/src/components/workspace-canvas/WorkspaceSection.tsx index 728e6bf4..7ac5829e 100644 --- a/src/components/workspace-canvas/WorkspaceSection.tsx +++ b/src/components/workspace-canvas/WorkspaceSection.tsx @@ -48,8 +48,14 @@ import type { WorkspaceWithState } from "@/lib/workspace-state/types"; import { useAui } from "@assistant-ui/react"; import { focusComposerInput } from "@/lib/utils/composer-utils"; import { UploadDialog } from "@/components/modals/UploadDialog"; +import { AudioRecordingIndicator } from "./AudioRecordingIndicator"; import { getBestFrameForRatio } from "@/lib/workspace-state/aspect-ratios"; import { useReactiveNavigation } from "@/hooks/ui/use-reactive-navigation"; +import { renderWorkspaceMenuItems } from "./workspace-menu-items"; +import { useAudioRecordingStore } from "@/lib/stores/audio-recording-store"; +import { AudioRecorderDialog } from "@/components/modals/AudioRecorderDialog"; +import { CreateWebsiteDialog } from "@/components/modals/CreateWebsiteDialog"; +import { useQueryClient } from "@tanstack/react-query"; interface WorkspaceSectionProps { // Loading states @@ -194,6 +200,13 @@ export function WorkspaceSection({ // Workspace settings and share modal state const [showYouTubeDialog, setShowYouTubeDialog] = useState(false); const [showUploadDialog, setShowUploadDialog] = useState(false); + const [showWebsiteDialog, setShowWebsiteDialog] = useState(false); + const showAudioDialog = useAudioRecordingStore((s) => s.isDialogOpen); + const openAudioDialog = useAudioRecordingStore((s) => s.openDialog); + const closeAudioDialog = useAudioRecordingStore((s) => s.closeDialog); + + // React Query client for cache invalidation + const queryClient = useQueryClient(); // Get workspace data from context const { workspaces } = useWorkspaceContext(); @@ -466,6 +479,91 @@ export function WorkspaceSection({ // Use reactive navigation hook for auto-scroll/selection const { handleCreatedItems } = useReactiveNavigation(state); + const handleAudioReady = useCallback(async (file: File) => { + if (!addItem) return; + + const loadingToastId = toast.loading("Uploading audio..."); + + try { + const { url: fileUrl } = await uploadFileDirect(file); + + const now = new Date(); + const dateStr = now.toLocaleDateString('en-US', { + month: 'short', + day: 'numeric', + year: now.getFullYear() !== new Date().getFullYear() ? 'numeric' : undefined + }); + const timeStr = now.toLocaleTimeString('en-US', { + hour: 'numeric', + minute: '2-digit', + hour12: true + }); + const title = `${dateStr} ${timeStr} Recording`; + + const itemId = addItem("audio", title, { + fileUrl, + filename: file.name, + fileSize: file.size, + mimeType: file.type || "audio/webm", + processingStatus: "processing", + } as any); + + if (handleCreatedItems && itemId) { + handleCreatedItems([itemId]); + } + + toast.dismiss(loadingToastId); + toast.success("Audio uploaded \u2014 analyzing with Gemini..."); + + fetch("/api/audio/process", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + fileUrl, + filename: file.name, + mimeType: file.type || "audio/webm", + }), + }) + .then((res) => res.json()) + .then((result) => { + if (result.success) { + window.dispatchEvent( + new CustomEvent("audio-processing-complete", { + detail: { + itemId, + summary: result.summary, + transcript: result.transcript, + segments: result.segments, + }, + }) + ); + } else { + window.dispatchEvent( + new CustomEvent("audio-processing-complete", { + detail: { + itemId, + error: result.error || "Processing failed", + }, + }) + ); + } + }) + .catch((err) => { + window.dispatchEvent( + new CustomEvent("audio-processing-complete", { + detail: { + itemId, + error: err.message || "Processing failed", + }, + }) + ); + }); + } catch (error: any) { + toast.dismiss(loadingToastId); + toast.error(error.message || "Failed to upload audio"); + } + }, [addItem, handleCreatedItems]); + // Get search params for invite check const searchParams = useSearchParams(); const hasInviteParam = searchParams.get('invite'); @@ -542,106 +640,46 @@ export function WorkspaceSection({ {/* Right-Click Context Menu */} {addItem && ( - - Create - { - if (addItem) { - const itemId = addItem("note"); - // Auto-navigate to the newly created note instead of opening modal - if (handleCreatedItems && itemId) { - handleCreatedItems([itemId]); - } - } - }} - className="flex items-center gap-2 cursor-pointer" - > - - Note - - - { - if (addItem) { - addItem("folder"); - } - }} - className="flex items-center gap-2 cursor-pointer" - > - - Folder - - - {operations && currentWorkspaceId && ( - - - Upload (PDF, Image) - - )} - - - - - Learn - - - { - if (addItem) { - const itemId = addItem("flashcard"); - if (handleCreatedItems && itemId) { - handleCreatedItems([itemId]); - } + + {renderWorkspaceMenuItems({ + callbacks: { + onCreateNote: () => { + if (addItem) { + const itemId = addItem("note"); + if (handleCreatedItems && itemId) { + handleCreatedItems([itemId]); } - }} - className="flex items-center gap-2 cursor-pointer" - > - - Flashcards - - { - // Open chat if closed - if (setIsChatExpanded && !isChatExpanded && isDesktop) { - setIsChatExpanded(true); + } + }, + onCreateFolder: () => { if (addItem) addItem("folder"); }, + onUpload: () => handleUploadMenuItemClick(), + onAudio: () => openAudioDialog(), + onYouTube: () => setShowYouTubeDialog(true), + onWebsite: () => setShowWebsiteDialog(true), + onFlashcards: () => { + if (addItem) { + const itemId = addItem("flashcard"); + if (handleCreatedItems && itemId) { + handleCreatedItems([itemId]); } - // Fill composer with quiz creation prompt - aui.composer().setText("Create a quiz about "); - // Focus the composer input - focusComposerInput(); - toast.success("Quiz creation started"); - }} - className="flex items-center gap-2 cursor-pointer" - > - - Quiz - - - - setShowYouTubeDialog(true)} - className="flex items-center gap-2 cursor-pointer" - > - - YouTube - - {/* { - toast.success("Deep Research action selected"); - setSelectedActions(["deep-research"]); - aui.composer().setText("I want to do research on "); - if (setIsChatExpanded && !isChatExpanded && isDesktop) { - setIsChatExpanded(true); - } - }} - className="flex items-center gap-2 cursor-pointer" - > - - Deep Research - */} + } + }, + onQuiz: () => { + if (setIsChatExpanded && !isChatExpanded && isDesktop) { + setIsChatExpanded(true); + } + aui.composer().setText("Create a quiz about "); + focusComposerInput(); + toast.success("Quiz creation started"); + }, + }, + MenuItem: ContextMenuItem, + MenuSub: ContextMenuSub, + MenuSubTrigger: ContextMenuSubTrigger, + MenuSubContent: ContextMenuSubContent, + MenuLabel: ContextMenuLabel, + showUpload: !!(operations && currentWorkspaceId), + })} )} @@ -709,6 +747,29 @@ export function WorkspaceSection({ onPDFUpload={handlePDFUpload} /> )} + + {/* Website Dialog */} + {currentWorkspaceId && ( + { + void queryClient.invalidateQueries({ + queryKey: ["workspace", currentWorkspaceId, "events"], + }); + }} + /> + )} + {/* Audio Recorder Dialog */} + { if (open) openAudioDialog(); else closeAudioDialog(); }} + onAudioReady={handleAudioReady} + /> + {/* Floating recording indicator (visible when dialog is closed but recording is active) */} + ); } diff --git a/src/components/workspace-canvas/workspace-menu-items.tsx b/src/components/workspace-canvas/workspace-menu-items.tsx new file mode 100644 index 00000000..1b6cb56d --- /dev/null +++ b/src/components/workspace-canvas/workspace-menu-items.tsx @@ -0,0 +1,149 @@ +"use client"; + +import type React from "react"; +import { FileText, Folder, Upload, Play, Brain, Mic, Newspaper } from "lucide-react"; +import { LuBook } from "react-icons/lu"; +import { PiCardsThreeBold } from "react-icons/pi"; +import { useAudioRecordingStore } from "@/lib/stores/audio-recording-store"; +import { toast } from "sonner"; +import { NewFeatureBadge } from "@/components/ui/new-feature-badge"; + +export interface WorkspaceMenuCallbacks { + onCreateNote: () => void; + onCreateFolder: () => void; + onUpload: () => void; + onAudio: () => void; + onYouTube: () => void; + onWebsite: () => void; + onFlashcards: () => void; + onQuiz: () => void; +} + +/** + * Shared "Create" menu items for both the header dropdown and the right-click + * context menu. The caller passes render helpers so the same logical items + * can be rendered as either DropdownMenuItems or ContextMenuItems. + */ +export function renderWorkspaceMenuItems({ + callbacks, + MenuItem, + MenuSub, + MenuSubTrigger, + MenuSubContent, + MenuLabel, + showUpload = true, +}: { + callbacks: WorkspaceMenuCallbacks; + MenuItem: React.ComponentType<{ onSelect?: () => void; onClick?: () => void; className?: string; children: React.ReactNode }>; + MenuSub: React.ComponentType<{ children: React.ReactNode }>; + MenuSubTrigger: React.ComponentType<{ className?: string; children: React.ReactNode }>; + MenuSubContent: React.ComponentType<{ children: React.ReactNode }>; + MenuLabel?: React.ComponentType<{ className?: string; children: React.ReactNode }>; + showUpload?: boolean; +}) { + const handleAudioClick = () => { + const isRecording = useAudioRecordingStore.getState().isRecording; + if (isRecording) { + toast.error("Recording already in progress"); + return; + } + callbacks.onAudio(); + }; + + return ( + <> + {MenuLabel && ( + Create + )} + + + + Note + + + + + Folder + + + {showUpload && ( + + +
+ Upload + PDF/Image +
+ + )} + + + +
+ + Audio + + Lecture/Meeting +
+ + + + + YouTube + + + + + Website + + + + + + Learn + + + + + Flashcards + + + + Quiz + + + + + ); +} diff --git a/src/components/workspace/SidebarCardList.tsx b/src/components/workspace/SidebarCardList.tsx index ec6a5b7d..0de0bf54 100644 --- a/src/components/workspace/SidebarCardList.tsx +++ b/src/components/workspace/SidebarCardList.tsx @@ -1,7 +1,7 @@ "use client"; import { memo, useMemo, useCallback, useState, useEffect, useRef } from "react"; -import { ChevronRight, FileText, File, FolderOpen, Folder as FolderIcon, MoreVertical, Trash2, Pencil, FolderInput, Play, Brain, ImageIcon } from "lucide-react"; +import { ChevronRight, FileText, File, FolderOpen, Folder as FolderIcon, MoreVertical, Trash2, Pencil, FolderInput, Play, Brain, ImageIcon, Mic } from "lucide-react"; import { PiCardsThreeBold } from "react-icons/pi"; import { SidebarMenu, @@ -61,6 +61,8 @@ function getCardTypeIcon(type: CardType) { return ; case "image": return ; + case "audio": + return ; default: return ; } diff --git a/src/hooks/audio/use-audio-recorder.ts b/src/hooks/audio/use-audio-recorder.ts new file mode 100644 index 00000000..2f069f5e --- /dev/null +++ b/src/hooks/audio/use-audio-recorder.ts @@ -0,0 +1,176 @@ +"use client"; + +import { useState, useRef, useCallback, useEffect } from "react"; + +export interface UseAudioRecorderReturn { + isRecording: boolean; + isPaused: boolean; + duration: number; // seconds + audioBlob: Blob | null; + startRecording: () => Promise; + stopRecording: () => void; + pauseRecording: () => void; + resumeRecording: () => void; + resetRecording: () => void; + error: string | null; +} + +export function useAudioRecorder(): UseAudioRecorderReturn { + const [isRecording, setIsRecording] = useState(false); + const [isPaused, setIsPaused] = useState(false); + const [duration, setDuration] = useState(0); + const [audioBlob, setAudioBlob] = useState(null); + const [error, setError] = useState(null); + + const mediaRecorderRef = useRef(null); + const chunksRef = useRef([]); + const timerRef = useRef | null>(null); + const streamRef = useRef(null); + + const clearTimer = useCallback(() => { + if (timerRef.current) { + clearInterval(timerRef.current); + timerRef.current = null; + } + }, []); + + const startTimer = useCallback(() => { + clearTimer(); + timerRef.current = setInterval(() => { + setDuration((prev) => prev + 1); + }, 1000); + }, [clearTimer]); + + const getSupportedMimeType = (): string => { + const types = [ + "audio/webm;codecs=opus", + "audio/webm", + "audio/mp4", + "audio/ogg;codecs=opus", + "audio/ogg", + ]; + for (const type of types) { + if (MediaRecorder.isTypeSupported(type)) { + return type; + } + } + return "audio/webm"; // Fallback + }; + + const startRecording = useCallback(async () => { + try { + setError(null); + setAudioBlob(null); + setDuration(0); + chunksRef.current = []; + + const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); + streamRef.current = stream; + + const mimeType = getSupportedMimeType(); + const recorder = new MediaRecorder(stream, { mimeType }); + mediaRecorderRef.current = recorder; + + recorder.ondataavailable = (e) => { + if (e.data.size > 0) { + chunksRef.current.push(e.data); + } + }; + + recorder.onstop = () => { + const blob = new Blob(chunksRef.current, { type: mimeType }); + setAudioBlob(blob); + clearTimer(); + + // Stop all tracks + if (streamRef.current) { + streamRef.current.getTracks().forEach((track) => { track.stop(); }); + streamRef.current = null; + } + }; + + recorder.onerror = () => { + setError("Recording failed. Please try again."); + setIsRecording(false); + clearTimer(); + }; + + recorder.start(1000); // Collect data every second + setIsRecording(true); + setIsPaused(false); + startTimer(); + } catch (err: any) { + // Clean up any acquired stream on failure + if (streamRef.current) { + streamRef.current.getTracks().forEach((track) => { track.stop(); }); + streamRef.current = null; + } + if (err.name === "NotAllowedError") { + setError("Microphone access denied. Please allow microphone access."); + } else if (err.name === "NotFoundError") { + setError("No microphone found. Please connect a microphone."); + } else { + setError(err.message || "Failed to start recording."); + } + } + }, [startTimer, clearTimer]); + + const stopRecording = useCallback(() => { + if (mediaRecorderRef.current && mediaRecorderRef.current.state !== "inactive") { + mediaRecorderRef.current.stop(); + setIsRecording(false); + setIsPaused(false); + clearTimer(); + } + }, [clearTimer]); + + const pauseRecording = useCallback(() => { + if (mediaRecorderRef.current && mediaRecorderRef.current.state === "recording") { + mediaRecorderRef.current.pause(); + setIsPaused(true); + clearTimer(); + } + }, [clearTimer]); + + const resumeRecording = useCallback(() => { + if (mediaRecorderRef.current && mediaRecorderRef.current.state === "paused") { + mediaRecorderRef.current.resume(); + setIsPaused(false); + startTimer(); + } + }, [startTimer]); + + const resetRecording = useCallback(() => { + stopRecording(); + setAudioBlob(null); + setDuration(0); + setError(null); + chunksRef.current = []; + }, [stopRecording]); + + // Cleanup on unmount + useEffect(() => { + return () => { + clearTimer(); + if (mediaRecorderRef.current && mediaRecorderRef.current.state !== "inactive") { + mediaRecorderRef.current.stop(); + } + if (streamRef.current) { + streamRef.current.getTracks().forEach((track) => { track.stop(); }); + } + }; + }, [clearTimer]); + + return { + isRecording, + isPaused, + duration, + audioBlob, + startRecording, + stopRecording, + pauseRecording, + resumeRecording, + resetRecording, + error, + }; +} diff --git a/src/hooks/workspace/use-workspace-operations.ts b/src/hooks/workspace/use-workspace-operations.ts index eb314dfc..319ef5e4 100644 --- a/src/hooks/workspace/use-workspace-operations.ts +++ b/src/hooks/workspace/use-workspace-operations.ts @@ -93,7 +93,7 @@ export function useWorkspaceOperations( // Validate type is a valid CardType logger.debug("🔧 [CREATE-ITEM] Received type:", type, "typeof:", typeof type, "value:", JSON.stringify(type)); - const validTypes: CardType[] = ["note", "pdf", "flashcard", "folder", "youtube", "image"]; + const validTypes: CardType[] = ["note", "pdf", "flashcard", "folder", "youtube", "image", "audio"]; const validType = validTypes.includes(type) ? type : "note"; if (validType !== type) { @@ -160,7 +160,7 @@ export function useWorkspaceOperations( // Create all items const createdItems: Item[] = items.map(({ type, name, initialData, initialLayout }) => { // Validate type is a valid CardType - const validTypes: CardType[] = ["note", "pdf", "flashcard", "folder", "youtube", "image"]; + const validTypes: CardType[] = ["note", "pdf", "flashcard", "folder", "youtube", "image", "audio"]; const validType = validTypes.includes(type) ? type : "note"; if (validType !== type) { diff --git a/src/lib/ai/workers/workspace-worker.ts b/src/lib/ai/workers/workspace-worker.ts index afa8f947..82d85cca 100644 --- a/src/lib/ai/workers/workspace-worker.ts +++ b/src/lib/ai/workers/workspace-worker.ts @@ -74,7 +74,7 @@ export async function workspaceWorker( content?: string; // For notes itemId?: string; - itemType?: "note" | "flashcard" | "quiz" | "youtube" | "image"; // Defaults to "note" if undefined + itemType?: "note" | "flashcard" | "quiz" | "youtube" | "image" | "audio"; // Defaults to "note" if undefined pdfTextContent?: string; // For caching extracted PDF text content flashcardData?: { cards?: { front: string; back: string }[]; // For creating flashcards @@ -90,6 +90,13 @@ export async function workspaceWorker( altText?: string; caption?: string; }; + audioData?: { + fileUrl: string; + filename: string; + fileSize?: number; + mimeType?: string; + duration?: number; + }; // Optional: deep research metadata to attach to a note deepResearchData?: { prompt: string; @@ -222,6 +229,19 @@ export async function workspaceWorker( altText: params.imageData.altText, caption: params.imageData.caption }; + } else if (itemType === "audio") { + // Audio type + if (!params.audioData || !params.audioData.fileUrl) { + throw new Error("Audio data required for audio card creation"); + } + itemData = { + fileUrl: params.audioData.fileUrl, + filename: params.audioData.filename || "Recording", + fileSize: params.audioData.fileSize, + mimeType: params.audioData.mimeType, + duration: params.audioData.duration, + processingStatus: "processing", + }; } else if (itemType === "quiz") { // Quiz type if (!params.quizData) { @@ -270,7 +290,7 @@ export async function workspaceWorker( const item: Item = { id: itemId, type: itemType, - name: params.title || (itemType === "youtube" ? "YouTube Video" : itemType === "image" ? "Image" : itemType === "quiz" ? "New Quiz" : itemType === "flashcard" ? "New Flashcard Deck" : "New Note"), + name: params.title || (itemType === "youtube" ? "YouTube Video" : itemType === "image" ? "Image" : itemType === "quiz" ? "New Quiz" : itemType === "flashcard" ? "New Flashcard Deck" : itemType === "audio" ? "Audio Recording" : "New Note"), subtitle: "", data: itemData, color: getRandomCardColor(), diff --git a/src/lib/attachments/supabase-attachment-adapter.ts b/src/lib/attachments/supabase-attachment-adapter.ts index 86194dd4..fbe3367a 100644 --- a/src/lib/attachments/supabase-attachment-adapter.ts +++ b/src/lib/attachments/supabase-attachment-adapter.ts @@ -18,7 +18,7 @@ const MAX_FILE_SIZE_BYTES = 50 * 1024 * 1024; // 50MB to match server limit */ export class SupabaseAttachmentAdapter implements AttachmentAdapter { accept = - "image/*,video/*,.pdf,.doc,.docx,.ppt,.pptx,.xls,.xlsx,.txt,.md,.csv,.json"; + "image/*,video/*,audio/*,.pdf,.doc,.docx,.ppt,.pptx,.xls,.xlsx,.txt,.md,.csv,.json,.mp3,.wav,.ogg,.aac,.flac,.aiff,.webm,.m4a"; // Map of attachment ID → upload promise (started eagerly in add()) private pendingUploads = new Map>(); diff --git a/src/lib/stores/audio-recording-store.ts b/src/lib/stores/audio-recording-store.ts new file mode 100644 index 00000000..45e51fdc --- /dev/null +++ b/src/lib/stores/audio-recording-store.ts @@ -0,0 +1,234 @@ +import { create } from "zustand"; + +export interface AudioRecordingState { + // Recording state + isRecording: boolean; + isPaused: boolean; + duration: number; // seconds + audioBlob: Blob | null; + error: string | null; + + // Internals (not reactive, but stored for access) + _mediaRecorder: MediaRecorder | null; + _stream: MediaStream | null; + _chunks: Blob[]; + _timerId: ReturnType | null; + _audioContext: AudioContext | null; + _analyser: AnalyserNode | null; + _resetting: boolean; + + // Dialog visibility (recording continues regardless) + isDialogOpen: boolean; + + // Actions + startRecording: () => Promise; + stopRecording: () => void; + pauseRecording: () => void; + resumeRecording: () => void; + resetRecording: () => void; + openDialog: () => void; + closeDialog: () => void; + setDuration: (d: number) => void; +} + +function getSupportedMimeType(): string { + const types = [ + "audio/webm;codecs=opus", + "audio/webm", + "audio/mp4", + "audio/ogg;codecs=opus", + "audio/ogg", + ]; + for (const type of types) { + if (typeof MediaRecorder !== "undefined" && MediaRecorder.isTypeSupported(type)) { + return type; + } + } + return "audio/webm"; +} + +export const useAudioRecordingStore = create((set, get) => ({ + isRecording: false, + isPaused: false, + duration: 0, + audioBlob: null, + error: null, + _mediaRecorder: null, + _stream: null, + _chunks: [], + _timerId: null, + _audioContext: null, + _analyser: null, + _resetting: false, + isDialogOpen: false, + + openDialog: () => set({ isDialogOpen: true }), + closeDialog: () => set({ isDialogOpen: false }), + + setDuration: (d) => set({ duration: d }), + + startRecording: async () => { + const state = get(); + // Clean up any previous recording + if (state._timerId) clearInterval(state._timerId); + if (state._mediaRecorder && state._mediaRecorder.state !== "inactive") { + state._mediaRecorder.stop(); + } + if (state._stream) { + state._stream.getTracks().forEach((t) => { t.stop(); }); + } + + try { + const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); + const mimeType = getSupportedMimeType(); + const recorder = new MediaRecorder(stream, { mimeType }); + + // Set up Web Audio API analyser for waveform visualisation + const audioContext = new AudioContext(); + const source = audioContext.createMediaStreamSource(stream); + const analyser = audioContext.createAnalyser(); + analyser.fftSize = 256; + source.connect(analyser); + const chunks: Blob[] = []; + + recorder.ondataavailable = (e) => { + if (e.data.size > 0) chunks.push(e.data); + }; + + recorder.onstop = () => { + const s = get(); + // If resetRecording triggered this stop, skip — reset already cleaned up + if (s._resetting) return; + + const blob = new Blob(chunks, { type: mimeType }); + if (s._timerId) clearInterval(s._timerId); + if (s._stream) { + s._stream.getTracks().forEach((t) => { t.stop(); }); + } + if (s._audioContext) { + s._audioContext.close().catch(() => {}); + } + set({ + audioBlob: blob, + isRecording: false, + isPaused: false, + _mediaRecorder: null, + _stream: null, + _chunks: [], + _timerId: null, + _audioContext: null, + _analyser: null, + // Auto-open dialog when recording stops so user can review + isDialogOpen: true, + }); + }; + + recorder.onerror = () => { + const s = get(); + if (s._timerId) clearInterval(s._timerId); + if (s._stream) { + s._stream.getTracks().forEach((t) => { t.stop(); }); + } + if (s._audioContext) { + s._audioContext.close().catch(() => {}); + } + set({ + error: "Recording failed. Please try again.", + isRecording: false, + isPaused: false, + _mediaRecorder: null, + _stream: null, + _chunks: [], + _timerId: null, + _audioContext: null, + _analyser: null, + }); + }; + + recorder.start(1000); + + const timerId = setInterval(() => { + set((s) => ({ duration: s.duration + 1 })); + }, 1000); + + set({ + isRecording: true, + isPaused: false, + duration: 0, + audioBlob: null, + error: null, + _mediaRecorder: recorder, + _stream: stream, + _chunks: chunks, + _timerId: timerId, + _audioContext: audioContext, + _analyser: analyser, + }); + } catch (err: any) { + if (err.name === "NotAllowedError") { + set({ error: "Microphone access denied. Please allow microphone access." }); + } else if (err.name === "NotFoundError") { + set({ error: "No microphone found. Please connect a microphone." }); + } else { + set({ error: err.message || "Failed to start recording." }); + } + } + }, + + stopRecording: () => { + const state = get(); + if (state._mediaRecorder && state._mediaRecorder.state !== "inactive") { + state._mediaRecorder.stop(); // triggers onstop which updates state + } + }, + + pauseRecording: () => { + const state = get(); + if (state._mediaRecorder && state._mediaRecorder.state === "recording") { + state._mediaRecorder.pause(); + if (state._timerId) clearInterval(state._timerId); + set({ isPaused: true, _timerId: null }); + } + }, + + resumeRecording: () => { + const state = get(); + if (state._mediaRecorder && state._mediaRecorder.state === "paused") { + state._mediaRecorder.resume(); + const timerId = setInterval(() => { + set((s) => ({ duration: s.duration + 1 })); + }, 1000); + set({ isPaused: false, _timerId: timerId }); + } + }, + + resetRecording: () => { + const state = get(); + if (state._timerId) clearInterval(state._timerId); + // Set _resetting flag so onstop handler knows to skip its state update + set({ _resetting: true }); + if (state._mediaRecorder && state._mediaRecorder.state !== "inactive") { + state._mediaRecorder.stop(); + } + if (state._stream) { + state._stream.getTracks().forEach((t) => { t.stop(); }); + } + if (state._audioContext) { + state._audioContext.close().catch(() => {}); + } + set({ + isRecording: false, + isPaused: false, + duration: 0, + audioBlob: null, + error: null, + _mediaRecorder: null, + _stream: null, + _chunks: [], + _timerId: null, + _audioContext: null, + _analyser: null, + _resetting: false, + }); + }, +})); diff --git a/src/lib/utils/format-workspace-context.ts b/src/lib/utils/format-workspace-context.ts index b25969f3..a7eb5f36 100644 --- a/src/lib/utils/format-workspace-context.ts +++ b/src/lib/utils/format-workspace-context.ts @@ -1,4 +1,4 @@ -import type { AgentState, Item, NoteData, PdfData, FlashcardData, FlashcardItem, YouTubeData, QuizData, QuizQuestion, ImageData } from "@/lib/workspace-state/types"; +import type { AgentState, Item, NoteData, PdfData, FlashcardData, FlashcardItem, YouTubeData, QuizData, QuizQuestion, ImageData, AudioData } from "@/lib/workspace-state/types"; import { serializeBlockNote } from "./serialize-blocknote"; import { type Block } from "@/components/editor/BlockNoteEditor"; @@ -389,6 +389,9 @@ function formatSelectedCardFull(item: Item, index: number): string { case "image": lines.push(...formatImageDetailsFull(item.data as ImageData)); break; + case "audio": + lines.push(...formatAudioDetailsFull(item.data as AudioData)); + break; } lines.push(``); @@ -670,3 +673,36 @@ function formatQuizDetailsFull(data: QuizData): string[] { return lines; } + +/** + * Formats audio details — timeline segments only + */ +function formatAudioDetailsFull(data: AudioData): string[] { + const lines: string[] = []; + + if (data.filename) { + lines.push(` - Filename: ${data.filename}`); + } + + if (data.duration) { + const mins = Math.floor(data.duration / 60); + const secs = Math.floor(data.duration % 60); + lines.push(` - Duration: ${mins}:${secs.toString().padStart(2, "0")}`); + } + + if (data.segments && data.segments.length > 0) { + lines.push(` - Timeline (${data.segments.length} segments):`); + for (const seg of data.segments) { + const lang = seg.language && seg.language !== "English" ? ` [${seg.language}]` : ""; + const emotion = seg.emotion && seg.emotion !== "neutral" ? ` (${seg.emotion})` : ""; + lines.push(` [${seg.timestamp}] ${seg.speaker}${lang}${emotion}: ${seg.content}`); + if (seg.translation) { + lines.push(` Translation: ${seg.translation}`); + } + } + } else { + lines.push(` - (No timeline segments available)`); + } + + return lines; +} diff --git a/src/lib/utils/new-feature.ts b/src/lib/utils/new-feature.ts new file mode 100644 index 00000000..55954373 --- /dev/null +++ b/src/lib/utils/new-feature.ts @@ -0,0 +1,109 @@ +"use client"; + +import { useState, useEffect, useCallback } from "react"; + +const STORAGE_PREFIX = "thinkex_feature_seen_"; + +interface NewFeatureOptions { + /** Unique key identifying this feature */ + featureKey: string; + /** How long (in ms) the badge stays visible after first seen. Default: 14 days */ + ttl?: number; + /** If provided, the badge won't show before this date */ + startDate?: Date; + /** If provided, the badge won't show after this date (regardless of TTL) */ + endDate?: Date; +} + +interface NewFeatureState { + /** Whether the "new" badge should be visible */ + isNew: boolean; + /** Call this to permanently dismiss the badge for this user */ + dismiss: () => void; +} + +const DEFAULT_TTL = 14 * 24 * 60 * 60 * 1000; // 14 days + +function getStorageKey(featureKey: string) { + return `${STORAGE_PREFIX}${featureKey}`; +} + +function isFeatureDismissed(featureKey: string): boolean { + if (typeof window === "undefined") return true; + try { + const raw = localStorage.getItem(getStorageKey(featureKey)); + if (!raw) return false; + const data = JSON.parse(raw) as { dismissedAt: number }; + return !!data.dismissedAt; + } catch { + return false; + } +} + +function dismissFeature(featureKey: string) { + if (typeof window === "undefined") return; + try { + localStorage.setItem( + getStorageKey(featureKey), + JSON.stringify({ dismissedAt: Date.now() }), + ); + } catch { + // localStorage full or unavailable — silently ignore + } +} + +function isWithinWindow( + ttl: number, + startDate?: Date, + endDate?: Date, +): boolean { + const now = Date.now(); + if (startDate && now < startDate.getTime()) return false; + if (endDate && now > endDate.getTime()) return false; + + // If there's a startDate, check TTL from startDate; otherwise always in window + if (startDate) { + return now - startDate.getTime() < ttl; + } + return true; +} + +/** + * Hook that tracks whether a feature should show a "new" badge. + * + * Uses localStorage so the badge is per-device. Once dismissed (either + * manually or after the TTL / endDate expires), it won't reappear. + * + * @example + * const { isNew, dismiss } = useNewFeature({ featureKey: "audio-upload", ttl: 14 * 24 * 60 * 60 * 1000 }); + */ +export function useNewFeature({ + featureKey, + ttl = DEFAULT_TTL, + startDate, + endDate, +}: NewFeatureOptions): NewFeatureState { + const [isNew, setIsNew] = useState(false); + + useEffect(() => { + if (isFeatureDismissed(featureKey)) { + setIsNew(false); + return; + } + const inWindow = isWithinWindow(ttl, startDate, endDate); + setIsNew(inWindow); + }, [featureKey, ttl, startDate, endDate]); + + const dismiss = useCallback(() => { + dismissFeature(featureKey); + setIsNew(false); + }, [featureKey]); + + return { isNew, dismiss }; +} + +/** Imperative helper — useful outside of React components */ +export const newFeatureUtils = { + isDismissed: isFeatureDismissed, + dismiss: dismissFeature, +}; diff --git a/src/lib/workspace-state/grid-layout-helpers.ts b/src/lib/workspace-state/grid-layout-helpers.ts index 8b5077a9..044328f3 100644 --- a/src/lib/workspace-state/grid-layout-helpers.ts +++ b/src/lib/workspace-state/grid-layout-helpers.ts @@ -12,6 +12,7 @@ export const DEFAULT_CARD_DIMENSIONS: Record youtube: { w: 4, h: 10 }, quiz: { w: 1, h: 4 }, image: { w: 4, h: 10 }, + audio: { w: 2, h: 10 }, }; /** diff --git a/src/lib/workspace-state/item-helpers.ts b/src/lib/workspace-state/item-helpers.ts index 41983ad6..389ab77d 100644 --- a/src/lib/workspace-state/item-helpers.ts +++ b/src/lib/workspace-state/item-helpers.ts @@ -1,4 +1,4 @@ -import type { CardType, ItemData, NoteData, PdfData, FlashcardData, FolderData, YouTubeData, ImageData } from "./types"; +import type { CardType, ItemData, NoteData, PdfData, FlashcardData, FolderData, YouTubeData, ImageData, AudioData } from "./types"; /** * Generate a unique item ID @@ -32,6 +32,8 @@ export function defaultDataFor(type: CardType): ItemData { return { url: "" } as YouTubeData; case "image": return { url: "" } as ImageData; + case "audio": + return { fileUrl: "", filename: "", processingStatus: "uploading" } as AudioData; default: return { field1: "" } as NoteData; } diff --git a/src/lib/workspace-state/types.ts b/src/lib/workspace-state/types.ts index d67f6667..9b7857c0 100644 --- a/src/lib/workspace-state/types.ts +++ b/src/lib/workspace-state/types.ts @@ -1,6 +1,6 @@ import type { CardColor } from './colors'; -export type CardType = "note" | "pdf" | "flashcard" | "folder" | "youtube" | "quiz" | "image"; +export type CardType = "note" | "pdf" | "flashcard" | "folder" | "youtube" | "quiz" | "image" | "audio"; /** * Source attribution for notes created from web search or deep research @@ -100,7 +100,31 @@ export interface QuizData { session?: QuizSessionData; // Session state for resuming } -export type ItemData = NoteData | PdfData | FlashcardData | FolderData | YouTubeData | QuizData | ImageData; +// Audio Types +export interface AudioSegment { + speaker: string; + timestamp: string; + content: string; + language?: string; + language_code?: string; + translation?: string; + emotion?: "happy" | "sad" | "angry" | "neutral"; +} + +export interface AudioData { + fileUrl: string; // Supabase/local storage URL + filename: string; // Original filename + fileSize?: number; // File size in bytes + duration?: number; // Duration in seconds + mimeType?: string; // MIME type of the audio file + summary?: string; // Gemini-generated summary + transcript?: string; // Full plain-text transcript + segments?: AudioSegment[];// Timestamped speaker segments + processingStatus: "uploading" | "processing" | "complete" | "failed"; + error?: string; // Error message if processing failed +} + +export type ItemData = NoteData | PdfData | FlashcardData | FolderData | YouTubeData | QuizData | ImageData | AudioData; // ===================================================== // FOLDER TYPES (DEPRECATED)