From 53a0ee197b2fcdd2de4077ba61eed150d9feabf8 Mon Sep 17 00:00:00 2001 From: shrimbly Date: Tue, 3 Feb 2026 23:37:18 +1300 Subject: [PATCH] fix(quick-003): restore MediaBunny audio decoding with Web Audio fallback Use MediaBunny as the primary decoder following the documented pattern: getPrimaryAudioTrack() + canDecode() check + AudioBufferSink.buffers(). Falls back to Web Audio API's decodeAudioData if MediaBunny cannot parse the format (e.g. unusual codecs). This ensures MediaBunny handles container formats it supports while Web Audio API catches the rest. Co-Authored-By: Claude Opus 4.5 --- src/hooks/useAudioMixing.ts | 145 +++++++++++++++++++++++++++++------- 1 file changed, 120 insertions(+), 25 deletions(-) diff --git a/src/hooks/useAudioMixing.ts b/src/hooks/useAudioMixing.ts index 74629bdc..cb424edc 100644 --- a/src/hooks/useAudioMixing.ts +++ b/src/hooks/useAudioMixing.ts @@ -1,6 +1,12 @@ 'use client'; import { useCallback } from 'react'; +import { + Input, + AudioBufferSink, + BlobSource, + ALL_FORMATS, +} from 'mediabunny'; const MINIMUM_AUDIO_DURATION = 0.1; @@ -29,9 +35,68 @@ interface UseAudioMixingReturn { ) => Promise; } +/** + * Decode audio blob using MediaBunny. Supports MP3, WAV, OGG and container formats. + * Returns an array of decoded AudioBuffers. + */ +async function decodeWithMediaBunny( + audioBlob: Blob, + onProgress?: (progress: AudioMixProgress) => void, +): Promise { + onProgress?.({ message: 'Reading audio tracks...', progress: 20 }); + + const blobSource = new BlobSource(audioBlob); + const input = new Input({ source: blobSource, formats: ALL_FORMATS }); + + const audioTrack = await input.getPrimaryAudioTrack(); + if (!audioTrack) { + throw new Error('No audio tracks found in file'); + } + + const decodable = await audioTrack.canDecode(); + if (!decodable) { + throw new Error('Audio codec not supported by this browser'); + } + + const sink = new AudioBufferSink(audioTrack); + const audioDuration = await input.computeDuration(); + + onProgress?.({ message: 'Decoding audio...', progress: 30 }); + const decodedBuffers: AudioBuffer[] = []; + for await (const wrappedBuffer of sink.buffers(0, audioDuration)) { + if (wrappedBuffer?.buffer) { + decodedBuffers.push(wrappedBuffer.buffer); + } + } + + if (decodedBuffers.length === 0) { + throw new Error('Failed to decode audio'); + } + + return decodedBuffers; +} + +/** + * Decode audio blob using the Web Audio API as a fallback. + */ +async function decodeWithWebAudio( + audioBlob: Blob, + onProgress?: (progress: AudioMixProgress) => void, +): Promise { + onProgress?.({ message: 'Decoding audio (Web Audio API)...', progress: 30 }); + const arrayBuffer = await audioBlob.arrayBuffer(); + const audioContext = new AudioContext(); + try { + const decoded = await audioContext.decodeAudioData(arrayBuffer); + return [decoded]; + } finally { + await audioContext.close(); + } +} + /** * Standalone async function to prepare audio for mixing with video. - * Uses Web Audio API to decode standalone audio files (MP3, WAV, OGG, etc.). + * Uses MediaBunny for decoding, with Web Audio API as fallback. */ export async function prepareAudioAsync( audioBlob: Blob, @@ -41,19 +106,17 @@ export async function prepareAudioAsync( ): Promise { try { onProgress?.({ message: 'Loading audio file...', progress: 10 }); - const arrayBuffer = await audioBlob.arrayBuffer(); - onProgress?.({ message: 'Decoding audio...', progress: 30 }); - const audioContext = new AudioContext(); - let decoded: AudioBuffer; + let decodedBuffers: AudioBuffer[]; try { - decoded = await audioContext.decodeAudioData(arrayBuffer); - } finally { - await audioContext.close(); + decodedBuffers = await decodeWithMediaBunny(audioBlob, onProgress); + } catch (mediaBunnyError) { + console.warn('MediaBunny audio decode failed, falling back to Web Audio API:', mediaBunnyError); + decodedBuffers = await decodeWithWebAudio(audioBlob, onProgress); } - const sampleRate = decoded.sampleRate; - const channels = decoded.numberOfChannels; + const sampleRate = decodedBuffers[0].sampleRate; + const channels = decodedBuffers[0].numberOfChannels; const targetDuration = Math.max(MINIMUM_AUDIO_DURATION, videoDuration); const totalSamples = Math.max(1, Math.floor(targetDuration * sampleRate)); @@ -69,33 +132,65 @@ export async function prepareAudioAsync( const offsetSamples = Math.floor(offsetSeconds * sampleRate); let writeOffset = 0; - let sourceOffset = 0; + let sourceSkipSamples = 0; if (offsetSamples > 0) { writeOffset = Math.min(offsetSamples, totalSamples); } else if (offsetSamples < 0) { - sourceOffset = Math.abs(offsetSamples); + sourceSkipSamples = Math.abs(offsetSamples); } - // Copy decoded audio into the target buffer - const copyLength = Math.min(decoded.length - sourceOffset, totalSamples - writeOffset); - if (copyLength > 0) { + // Skip offset samples across decoded buffers + let samplesToSkip = sourceSkipSamples; + let bufferIndex = 0; + let bufferOffset = 0; + + while (samplesToSkip > 0 && bufferIndex < decodedBuffers.length) { + const buffer = decodedBuffers[bufferIndex]; + const availableInBuffer = buffer.length - bufferOffset; + if (samplesToSkip >= availableInBuffer) { + samplesToSkip -= availableInBuffer; + bufferIndex++; + bufferOffset = 0; + } else { + bufferOffset = samplesToSkip; + samplesToSkip = 0; + } + } + + // Copy decoded audio into the merged buffer + while (writeOffset < totalSamples && bufferIndex < decodedBuffers.length) { + const buffer = decodedBuffers[bufferIndex]; + const remainingSamples = totalSamples - writeOffset; + const availableInBuffer = buffer.length - bufferOffset; + const writeLength = Math.min(availableInBuffer, remainingSamples); + for (let channel = 0; channel < channels; channel++) { - const sourceData = decoded.getChannelData(channel).subarray(sourceOffset, sourceOffset + copyLength); - mergedBuffer.getChannelData(channel).set(sourceData, writeOffset); + const channelData = buffer.getChannelData(channel).subarray(bufferOffset, bufferOffset + writeLength); + mergedBuffer.getChannelData(channel).set(channelData, writeOffset); + } + + writeOffset += writeLength; + bufferOffset += writeLength; + + if (bufferOffset >= buffer.length) { + bufferIndex++; + bufferOffset = 0; } - writeOffset += copyLength; } // Loop audio to fill remaining duration if needed - while (writeOffset < totalSamples) { - const remaining = totalSamples - writeOffset; - const loopLength = Math.min(decoded.length, remaining); - for (let channel = 0; channel < channels; channel++) { - const sourceData = decoded.getChannelData(channel).subarray(0, loopLength); - mergedBuffer.getChannelData(channel).set(sourceData, writeOffset); + while (writeOffset < totalSamples && decodedBuffers.length > 0) { + for (const buffer of decodedBuffers) { + const remainingSamples = totalSamples - writeOffset; + if (remainingSamples <= 0) break; + const writeLength = Math.min(buffer.length, remainingSamples); + for (let channel = 0; channel < channels; channel++) { + const channelData = buffer.getChannelData(channel).subarray(0, writeLength); + mergedBuffer.getChannelData(channel).set(channelData, writeOffset); + } + writeOffset += writeLength; } - writeOffset += loopLength; } applyFades(mergedBuffer, options);