From 6cfef12b7ffd435e665a571b36dc6d2e3e466d01 Mon Sep 17 00:00:00 2001 From: rishikanthc Date: Sat, 2 May 2026 15:23:16 -0700 Subject: [PATCH] Add browser recording engine --- ...audio-recording-frontend-sprint-tracker.md | 26 +- .../recording/hooks/useBrowserRecorder.ts | 452 ++++++++++++++++++ .../recording/utils/mediaRecorderSupport.ts | 105 ++++ .../recording/utils/recordingDuration.ts | 36 ++ 4 files changed, 609 insertions(+), 10 deletions(-) create mode 100644 web/frontend/src/features/recording/hooks/useBrowserRecorder.ts create mode 100644 web/frontend/src/features/recording/utils/mediaRecorderSupport.ts create mode 100644 web/frontend/src/features/recording/utils/recordingDuration.ts diff --git a/devnotes/v2.0.0/sprint-trackers/in-app-audio-recording-frontend-sprint-tracker.md b/devnotes/v2.0.0/sprint-trackers/in-app-audio-recording-frontend-sprint-tracker.md index 75914928..b9874353 100644 --- a/devnotes/v2.0.0/sprint-trackers/in-app-audio-recording-frontend-sprint-tracker.md +++ b/devnotes/v2.0.0/sprint-trackers/in-app-audio-recording-frontend-sprint-tracker.md @@ -2,7 +2,7 @@ This tracker belongs to `devnotes/v2.0.0/sprint-plans/in-app-audio-recording-frontend-sprint-plan.md`. -Status: Sprint 1 complete. Sprint 2 has not started. +Status: Sprint 2 complete. Sprint 3 has not started. ## Sprint 1: API Contract and Query Foundation @@ -33,21 +33,21 @@ Notes: ## Sprint 2: Browser Recorder Engine -Status: pending +Status: complete Progress: -- [ ] Add `useBrowserRecorder` state machine. -- [ ] Add runtime MIME fallback selection. -- [ ] Add supported audio constraint negotiation. -- [ ] Add sequential chunk upload queue and retry handling. -- [ ] Add monotonic active-duration timer. -- [ ] Add unload protection while recording or chunks are unsent. +- [x] Add `useBrowserRecorder` state machine. +- [x] Add runtime MIME fallback selection. +- [x] Add supported audio constraint negotiation. +- [x] Add sequential chunk upload queue and retry handling. +- [x] Add monotonic active-duration timer. +- [x] Add unload protection while recording or chunks are unsent. Verification: -- [ ] Hook/state machine tests or browser smoke notes. -- [ ] `npm run type-check` from `web/frontend`. +- [x] Hook/state machine implementation reviewed at the API boundary; browser smoke is deferred until Sprint 3 mounts the hook in the recorder dialog. +- [x] `npm run type-check` from `web/frontend`. Artifacts: @@ -55,6 +55,12 @@ Artifacts: - `web/frontend/src/features/recording/utils/mediaRecorderSupport.ts` - `web/frontend/src/features/recording/utils/recordingDuration.ts` +Notes: + +- `useBrowserRecorder` owns the local browser workflow state, stream lifecycle, `MediaRecorder` instance, chunk queue, retry queue, timer, unload guard, and backend create/upload/stop/cancel mutations. +- MIME type and audio constraints are runtime-selected so Chrome, Firefox, and Safari can choose their supported encoder and device features. +- Cancel/unmount disables chunk acceptance before stopping tracks so explicit cancellation does not upload a trailing `dataavailable` blob. + ## Sprint 3: Recorder Dialog and Header Entry Status: pending diff --git a/web/frontend/src/features/recording/hooks/useBrowserRecorder.ts b/web/frontend/src/features/recording/hooks/useBrowserRecorder.ts new file mode 100644 index 00000000..0110e019 --- /dev/null +++ b/web/frontend/src/features/recording/hooks/useBrowserRecorder.ts @@ -0,0 +1,452 @@ +import { useCallback, useEffect, useRef, useState } from "react"; +import type { RecordingSession } from "@/features/recording/api/recordingsApi"; +import { + useCancelRecording, + useCreateRecording, + useStopRecording, + useUploadRecordingChunk, +} from "@/features/recording/hooks/useRecordingSession"; +import { + buildMicrophoneConstraints, + getBrowserRecordingSupport, + mediaRecorderOptionsFor, + selectRecordingMimeType, + type BrowserRecordingSupport, + type MicrophoneConstraintSelection, + type RecordingMimeSelection, +} from "@/features/recording/utils/mediaRecorderSupport"; +import { + createRecordingDurationState, + elapsedRecordingDurationMs, + pauseRecordingDuration, + resumeRecordingDuration, + stopRecordingDuration, + type RecordingDurationState, +} from "@/features/recording/utils/recordingDuration"; + +export type BrowserRecorderStatus = + | "idle" + | "unsupported" + | "permission-denied" + | "permission-ready" + | "recording" + | "paused" + | "stopping" + | "finalizing" + | "ready" + | "failed" + | "canceled"; + +export type BrowserRecorderStartOptions = { + title: string; + deviceId?: string; + chunkDurationMs?: number; + autoTranscribe?: boolean; + profileId?: string; + language?: string; + diarization?: boolean; +}; + +export type BrowserRecorderState = { + status: BrowserRecorderStatus; + session: RecordingSession | null; + error: string | null; + elapsedMs: number; + selectedMimeType: string; + appliedSettings: MediaTrackSettings | null; + requestedConstraints: MicrophoneConstraintSelection["requested"] | null; + pendingChunks: number; + uploadedChunks: number; + failedChunkIndex: number | null; +}; + +type PendingChunk = { + index: number; + blob: Blob; + mimeType: string; + durationMs?: number; +}; + +const defaultChunkDurationMs = 5000; + +export function useBrowserRecorder() { + const createRecordingMutation = useCreateRecording(); + const uploadChunkMutation = useUploadRecordingChunk(); + const stopRecordingMutation = useStopRecording(); + const cancelRecordingMutation = useCancelRecording(); + + const [state, setState] = useState(() => ({ + status: getBrowserRecordingSupport().supported ? "idle" : "unsupported", + session: null, + error: supportErrorMessage(getBrowserRecordingSupport()), + elapsedMs: 0, + selectedMimeType: "", + appliedSettings: null, + requestedConstraints: null, + pendingChunks: 0, + uploadedChunks: 0, + failedChunkIndex: null, + })); + + const mediaRecorderRef = useRef(null); + const streamRef = useRef(null); + const sessionRef = useRef(null); + const chunkIndexRef = useRef(0); + const chunkStartedAtRef = useRef(null); + const durationRef = useRef(null); + const uploadChainRef = useRef(Promise.resolve()); + const failedChunksRef = useRef([]); + const stopResolverRef = useRef<(() => void) | null>(null); + const acceptingChunksRef = useRef(false); + + const setError = useCallback((error: unknown, fallback: string) => { + setState((current) => ({ + ...current, + status: isPermissionError(error) ? "permission-denied" : "failed", + error: error instanceof Error ? error.message : fallback, + })); + }, []); + + const stopStream = useCallback(() => { + streamRef.current?.getTracks().forEach((track) => track.stop()); + streamRef.current = null; + }, []); + + const reset = useCallback(() => { + mediaRecorderRef.current = null; + sessionRef.current = null; + chunkIndexRef.current = 0; + chunkStartedAtRef.current = null; + durationRef.current = null; + acceptingChunksRef.current = false; + failedChunksRef.current = []; + uploadChainRef.current = Promise.resolve(); + stopResolverRef.current = null; + stopStream(); + setState({ + status: getBrowserRecordingSupport().supported ? "idle" : "unsupported", + session: null, + error: supportErrorMessage(getBrowserRecordingSupport()), + elapsedMs: 0, + selectedMimeType: "", + appliedSettings: null, + requestedConstraints: null, + pendingChunks: 0, + uploadedChunks: 0, + failedChunkIndex: null, + }); + }, [stopStream]); + + const enqueueChunkUpload = useCallback((chunk: PendingChunk) => { + setState((current) => ({ + ...current, + pendingChunks: current.pendingChunks + 1, + failedChunkIndex: null, + })); + + uploadChainRef.current = uploadChainRef.current + .then(async () => { + const recording = sessionRef.current; + if (!recording) return; + const sha256 = await sha256Blob(chunk.blob); + await uploadChunkMutation.mutateAsync({ + recordingId: recording.id, + chunkIndex: chunk.index, + chunk: chunk.blob, + mimeType: chunk.mimeType, + sha256, + durationMs: chunk.durationMs, + }); + setState((current) => ({ + ...current, + pendingChunks: Math.max(0, current.pendingChunks - 1), + uploadedChunks: current.uploadedChunks + 1, + failedChunkIndex: null, + })); + }) + .catch((error: unknown) => { + failedChunksRef.current = [...failedChunksRef.current, chunk]; + setState((current) => ({ + ...current, + status: "failed", + error: error instanceof Error ? error.message : "Failed to upload recording chunk", + pendingChunks: Math.max(0, current.pendingChunks - 1), + failedChunkIndex: chunk.index, + })); + }); + }, [uploadChunkMutation]); + + const start = useCallback(async (options: BrowserRecorderStartOptions) => { + const support = getBrowserRecordingSupport(); + if (!support.supported) { + setState((current) => ({ + ...current, + status: "unsupported", + error: supportErrorMessage(support), + })); + return; + } + + try { + const mimeSelection = selectRecordingMimeType(); + const constraints = buildMicrophoneConstraints(options.deviceId); + const stream = await navigator.mediaDevices.getUserMedia({ audio: constraints.audio }); + const audioTrack = stream.getAudioTracks()[0]; + const appliedSettings = audioTrack?.getSettings() || null; + const recorder = createMediaRecorder(stream, mimeSelection); + const chunkDurationMs = options.chunkDurationMs || defaultChunkDurationMs; + const session = await createRecordingMutation.mutateAsync({ + title: options.title, + source_kind: "microphone", + mime_type: recorder.mimeType || mimeSelection.mimeType || "application/octet-stream", + codec: mimeSelection.codec, + chunk_duration_ms: chunkDurationMs, + auto_transcribe: options.autoTranscribe ?? false, + profile_id: options.profileId, + options: { + language: options.language, + diarization: options.diarization, + }, + }); + + streamRef.current = stream; + mediaRecorderRef.current = recorder; + sessionRef.current = session; + chunkIndexRef.current = 0; + failedChunksRef.current = []; + uploadChainRef.current = Promise.resolve(); + durationRef.current = createRecordingDurationState(performance.now()); + chunkStartedAtRef.current = performance.now(); + acceptingChunksRef.current = true; + + recorder.addEventListener("dataavailable", (event) => { + if (!acceptingChunksRef.current || !event.data || event.data.size === 0) return; + const now = performance.now(); + const durationMs = chunkStartedAtRef.current === null ? undefined : Math.max(0, now - chunkStartedAtRef.current); + chunkStartedAtRef.current = now; + enqueueChunkUpload({ + index: chunkIndexRef.current, + blob: event.data, + mimeType: event.data.type || recorder.mimeType || mimeSelection.mimeType || "application/octet-stream", + durationMs, + }); + chunkIndexRef.current += 1; + }); + + recorder.addEventListener("stop", () => { + stopResolverRef.current?.(); + stopResolverRef.current = null; + }); + + recorder.start(chunkDurationMs); + setState({ + status: "recording", + session, + error: null, + elapsedMs: 0, + selectedMimeType: recorder.mimeType || mimeSelection.mimeType, + appliedSettings, + requestedConstraints: constraints.requested, + pendingChunks: 0, + uploadedChunks: 0, + failedChunkIndex: null, + }); + } catch (error) { + stopStream(); + setError(error, "Failed to start recording"); + } + }, [createRecordingMutation, enqueueChunkUpload, setError, stopStream]); + + const pause = useCallback(() => { + const recorder = mediaRecorderRef.current; + if (!recorder || recorder.state !== "recording" || !durationRef.current) return; + recorder.pause(); + durationRef.current = pauseRecordingDuration(durationRef.current, performance.now()); + setState((current) => ({ + ...current, + status: "paused", + elapsedMs: durationRef.current ? elapsedRecordingDurationMs(durationRef.current, performance.now()) : current.elapsedMs, + })); + }, []); + + const resume = useCallback(() => { + const recorder = mediaRecorderRef.current; + if (!recorder || recorder.state !== "paused" || !durationRef.current) return; + recorder.resume(); + const now = performance.now(); + durationRef.current = resumeRecordingDuration(durationRef.current, now); + chunkStartedAtRef.current = now; + setState((current) => ({ + ...current, + status: "recording", + error: null, + })); + }, []); + + const stop = useCallback(async () => { + const recorder = mediaRecorderRef.current; + const session = sessionRef.current; + if (!recorder || !session || !durationRef.current) return; + + try { + setState((current) => ({ ...current, status: "stopping", error: null })); + durationRef.current = stopRecordingDuration(durationRef.current, performance.now()); + const durationMs = elapsedRecordingDurationMs(durationRef.current, performance.now()); + + if (recorder.state !== "inactive") { + const stopped = new Promise((resolve) => { + stopResolverRef.current = resolve; + }); + recorder.requestData(); + recorder.stop(); + await stopped; + } + acceptingChunksRef.current = false; + + await uploadChainRef.current; + if (failedChunksRef.current.length > 0) { + throw new Error("Some recording chunks failed to upload"); + } + + setState((current) => ({ ...current, status: "finalizing", elapsedMs: durationMs })); + const stoppedSession = await stopRecordingMutation.mutateAsync({ + recordingId: session.id, + payload: { + final_chunk_index: Math.max(0, chunkIndexRef.current - 1), + duration_ms: durationMs, + }, + }); + + sessionRef.current = stoppedSession; + stopStream(); + setState((current) => ({ + ...current, + status: stoppedSession.status === "ready" ? "ready" : "finalizing", + session: stoppedSession, + elapsedMs: durationMs, + })); + } catch (error) { + setError(error, "Failed to stop recording"); + } + }, [setError, stopRecordingMutation, stopStream]); + + const cancel = useCallback(async () => { + const recorder = mediaRecorderRef.current; + const session = sessionRef.current; + + try { + acceptingChunksRef.current = false; + if (recorder && recorder.state !== "inactive") { + recorder.stop(); + } + stopStream(); + if (session) { + await cancelRecordingMutation.mutateAsync(session.id); + } + setState((current) => ({ + ...current, + status: "canceled", + error: null, + })); + } catch (error) { + setError(error, "Failed to cancel recording"); + } + }, [cancelRecordingMutation, setError, stopStream]); + + const retryPendingChunk = useCallback(() => { + const chunks = failedChunksRef.current; + if (chunks.length === 0) return; + failedChunksRef.current = []; + setState((current) => ({ + ...current, + status: current.session?.status === "stopping" ? "stopping" : "recording", + error: null, + failedChunkIndex: null, + })); + chunks.forEach(enqueueChunkUpload); + }, [enqueueChunkUpload]); + + useEffect(() => { + if (state.status !== "recording") return; + const timer = window.setInterval(() => { + const duration = durationRef.current; + if (!duration) return; + setState((current) => ({ + ...current, + elapsedMs: elapsedRecordingDurationMs(duration, performance.now()), + })); + }, 250); + + return () => window.clearInterval(timer); + }, [state.status]); + + useEffect(() => { + const shouldWarn = state.status === "recording" || state.status === "paused" || state.pendingChunks > 0; + if (!shouldWarn) return; + + const handleBeforeUnload = (event: BeforeUnloadEvent) => { + event.preventDefault(); + event.returnValue = ""; + }; + + window.addEventListener("beforeunload", handleBeforeUnload); + return () => window.removeEventListener("beforeunload", handleBeforeUnload); + }, [state.pendingChunks, state.status]); + + useEffect(() => { + return () => { + acceptingChunksRef.current = false; + const recorder = mediaRecorderRef.current; + if (recorder && recorder.state !== "inactive") { + recorder.stop(); + } + mediaRecorderRef.current = null; + stopStream(); + }; + }, [stopStream]); + + return { + state, + start, + pause, + resume, + stop, + cancel, + retryPendingChunk, + reset, + }; +} + +function createMediaRecorder(stream: MediaStream, selection: RecordingMimeSelection) { + const options = mediaRecorderOptionsFor(selection); + try { + return options ? new MediaRecorder(stream, options) : new MediaRecorder(stream); + } catch { + return new MediaRecorder(stream); + } +} + +async function sha256Blob(blob: Blob): Promise { + if (!crypto.subtle) return undefined; + const buffer = await blob.arrayBuffer(); + const hash = await crypto.subtle.digest("SHA-256", buffer); + return Array.from(new Uint8Array(hash)) + .map((byte) => byte.toString(16).padStart(2, "0")) + .join(""); +} + +function isPermissionError(error: unknown) { + return error instanceof DOMException && (error.name === "NotAllowedError" || error.name === "SecurityError"); +} + +function supportErrorMessage(support: BrowserRecordingSupport) { + switch (support.reason) { + case "missing-media-devices": + case "missing-get-user-media": + return "This browser does not support microphone recording."; + case "missing-media-recorder": + return "This browser does not support MediaRecorder."; + default: + return null; + } +} diff --git a/web/frontend/src/features/recording/utils/mediaRecorderSupport.ts b/web/frontend/src/features/recording/utils/mediaRecorderSupport.ts new file mode 100644 index 00000000..a2a93d58 --- /dev/null +++ b/web/frontend/src/features/recording/utils/mediaRecorderSupport.ts @@ -0,0 +1,105 @@ +export type RecordingMimeSelection = { + mimeType: string; + codec?: string; +}; + +export type MicrophoneConstraintSelection = { + audio: MediaTrackConstraints; + requested: { + echoCancellation: boolean; + noiseSuppression: boolean; + autoGainControl: boolean; + channelCount: boolean; + sampleRate: boolean; + deviceId: boolean; + }; +}; + +export type BrowserRecordingSupport = { + supported: boolean; + reason?: "missing-media-devices" | "missing-get-user-media" | "missing-media-recorder"; +}; + +const preferredAudioMimeTypes = [ + "audio/webm;codecs=opus", + "audio/webm", + "audio/ogg;codecs=opus", + "audio/ogg", + "audio/mp4;codecs=mp4a.40.2", + "audio/mp4", + "audio/aac", +] as const; + +export function getBrowserRecordingSupport(): BrowserRecordingSupport { + if (!navigator.mediaDevices) { + return { supported: false, reason: "missing-media-devices" }; + } + if (!navigator.mediaDevices.getUserMedia) { + return { supported: false, reason: "missing-get-user-media" }; + } + if (typeof MediaRecorder === "undefined") { + return { supported: false, reason: "missing-media-recorder" }; + } + return { supported: true }; +} + +export function selectRecordingMimeType(): RecordingMimeSelection { + if (typeof MediaRecorder === "undefined" || typeof MediaRecorder.isTypeSupported !== "function") { + return { mimeType: "" }; + } + + const mimeType = preferredAudioMimeTypes.find((candidate) => MediaRecorder.isTypeSupported(candidate)) || ""; + return { + mimeType, + codec: codecFromMimeType(mimeType), + }; +} + +export function buildMicrophoneConstraints(deviceId?: string): MicrophoneConstraintSelection { + const supported = navigator.mediaDevices?.getSupportedConstraints?.() || {}; + const audio: MediaTrackConstraints = {}; + + if (deviceId) { + audio.deviceId = { exact: deviceId }; + } + if (supported.echoCancellation) { + audio.echoCancellation = true; + } + if (supported.noiseSuppression) { + audio.noiseSuppression = true; + } + if (supported.autoGainControl) { + audio.autoGainControl = true; + } + if (supported.channelCount) { + audio.channelCount = { ideal: 1 }; + } + if (supported.sampleRate) { + audio.sampleRate = { ideal: 48000 }; + } + + return { + audio, + requested: { + echoCancellation: Boolean(supported.echoCancellation), + noiseSuppression: Boolean(supported.noiseSuppression), + autoGainControl: Boolean(supported.autoGainControl), + channelCount: Boolean(supported.channelCount), + sampleRate: Boolean(supported.sampleRate), + deviceId: Boolean(deviceId), + }, + }; +} + +export function mediaRecorderOptionsFor(selection: RecordingMimeSelection): MediaRecorderOptions | undefined { + if (!selection.mimeType) return undefined; + return { + mimeType: selection.mimeType, + audioBitsPerSecond: 128_000, + }; +} + +function codecFromMimeType(mimeType: string) { + const codecMatch = mimeType.match(/codecs=([^;]+)/i); + return codecMatch?.[1]?.trim(); +} diff --git a/web/frontend/src/features/recording/utils/recordingDuration.ts b/web/frontend/src/features/recording/utils/recordingDuration.ts new file mode 100644 index 00000000..5865ce3d --- /dev/null +++ b/web/frontend/src/features/recording/utils/recordingDuration.ts @@ -0,0 +1,36 @@ +export type RecordingDurationState = { + accumulatedMs: number; + activeStartedAt: number | null; +}; + +export function createRecordingDurationState(now: number): RecordingDurationState { + return { + accumulatedMs: 0, + activeStartedAt: now, + }; +} + +export function pauseRecordingDuration(state: RecordingDurationState, now: number): RecordingDurationState { + if (state.activeStartedAt === null) return state; + return { + accumulatedMs: state.accumulatedMs + Math.max(0, now - state.activeStartedAt), + activeStartedAt: null, + }; +} + +export function resumeRecordingDuration(state: RecordingDurationState, now: number): RecordingDurationState { + if (state.activeStartedAt !== null) return state; + return { + ...state, + activeStartedAt: now, + }; +} + +export function stopRecordingDuration(state: RecordingDurationState, now: number): RecordingDurationState { + return pauseRecordingDuration(state, now); +} + +export function elapsedRecordingDurationMs(state: RecordingDurationState, now: number): number { + if (state.activeStartedAt === null) return Math.round(state.accumulatedMs); + return Math.round(state.accumulatedMs + Math.max(0, now - state.activeStartedAt)); +}