Browse Source

fix(quick-003): restore MediaBunny audio decoding with Web Audio fallback

Use MediaBunny as the primary decoder following the documented pattern:
getPrimaryAudioTrack() + canDecode() check + AudioBufferSink.buffers().
Falls back to Web Audio API's decodeAudioData if MediaBunny cannot parse
the format (e.g. unusual codecs). This ensures MediaBunny handles
container formats it supports while Web Audio API catches the rest.

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
handoff-20260429-1057
shrimbly 5 months ago
parent
commit
53a0ee197b
  1. 143
      src/hooks/useAudioMixing.ts

143
src/hooks/useAudioMixing.ts

@ -1,6 +1,12 @@
'use client'; 'use client';
import { useCallback } from 'react'; import { useCallback } from 'react';
import {
Input,
AudioBufferSink,
BlobSource,
ALL_FORMATS,
} from 'mediabunny';
const MINIMUM_AUDIO_DURATION = 0.1; const MINIMUM_AUDIO_DURATION = 0.1;
@ -29,9 +35,68 @@ interface UseAudioMixingReturn {
) => Promise<AudioData | null>; ) => Promise<AudioData | null>;
} }
/**
* Decode audio blob using MediaBunny. Supports MP3, WAV, OGG and container formats.
* Returns an array of decoded AudioBuffers.
*/
async function decodeWithMediaBunny(
audioBlob: Blob,
onProgress?: (progress: AudioMixProgress) => void,
): Promise<AudioBuffer[]> {
onProgress?.({ message: 'Reading audio tracks...', progress: 20 });
const blobSource = new BlobSource(audioBlob);
const input = new Input({ source: blobSource, formats: ALL_FORMATS });
const audioTrack = await input.getPrimaryAudioTrack();
if (!audioTrack) {
throw new Error('No audio tracks found in file');
}
const decodable = await audioTrack.canDecode();
if (!decodable) {
throw new Error('Audio codec not supported by this browser');
}
const sink = new AudioBufferSink(audioTrack);
const audioDuration = await input.computeDuration();
onProgress?.({ message: 'Decoding audio...', progress: 30 });
const decodedBuffers: AudioBuffer[] = [];
for await (const wrappedBuffer of sink.buffers(0, audioDuration)) {
if (wrappedBuffer?.buffer) {
decodedBuffers.push(wrappedBuffer.buffer);
}
}
if (decodedBuffers.length === 0) {
throw new Error('Failed to decode audio');
}
return decodedBuffers;
}
/**
* Decode audio blob using the Web Audio API as a fallback.
*/
async function decodeWithWebAudio(
audioBlob: Blob,
onProgress?: (progress: AudioMixProgress) => void,
): Promise<AudioBuffer[]> {
onProgress?.({ message: 'Decoding audio (Web Audio API)...', progress: 30 });
const arrayBuffer = await audioBlob.arrayBuffer();
const audioContext = new AudioContext();
try {
const decoded = await audioContext.decodeAudioData(arrayBuffer);
return [decoded];
} finally {
await audioContext.close();
}
}
/** /**
* Standalone async function to prepare audio for mixing with video. * Standalone async function to prepare audio for mixing with video.
* Uses Web Audio API to decode standalone audio files (MP3, WAV, OGG, etc.). * Uses MediaBunny for decoding, with Web Audio API as fallback.
*/ */
export async function prepareAudioAsync( export async function prepareAudioAsync(
audioBlob: Blob, audioBlob: Blob,
@ -41,19 +106,17 @@ export async function prepareAudioAsync(
): Promise<AudioData | null> { ): Promise<AudioData | null> {
try { try {
onProgress?.({ message: 'Loading audio file...', progress: 10 }); onProgress?.({ message: 'Loading audio file...', progress: 10 });
const arrayBuffer = await audioBlob.arrayBuffer();
onProgress?.({ message: 'Decoding audio...', progress: 30 }); let decodedBuffers: AudioBuffer[];
const audioContext = new AudioContext();
let decoded: AudioBuffer;
try { try {
decoded = await audioContext.decodeAudioData(arrayBuffer); decodedBuffers = await decodeWithMediaBunny(audioBlob, onProgress);
} finally { } catch (mediaBunnyError) {
await audioContext.close(); console.warn('MediaBunny audio decode failed, falling back to Web Audio API:', mediaBunnyError);
decodedBuffers = await decodeWithWebAudio(audioBlob, onProgress);
} }
const sampleRate = decoded.sampleRate; const sampleRate = decodedBuffers[0].sampleRate;
const channels = decoded.numberOfChannels; const channels = decodedBuffers[0].numberOfChannels;
const targetDuration = Math.max(MINIMUM_AUDIO_DURATION, videoDuration); const targetDuration = Math.max(MINIMUM_AUDIO_DURATION, videoDuration);
const totalSamples = Math.max(1, Math.floor(targetDuration * sampleRate)); const totalSamples = Math.max(1, Math.floor(targetDuration * sampleRate));
@ -69,33 +132,65 @@ export async function prepareAudioAsync(
const offsetSamples = Math.floor(offsetSeconds * sampleRate); const offsetSamples = Math.floor(offsetSeconds * sampleRate);
let writeOffset = 0; let writeOffset = 0;
let sourceOffset = 0; let sourceSkipSamples = 0;
if (offsetSamples > 0) { if (offsetSamples > 0) {
writeOffset = Math.min(offsetSamples, totalSamples); writeOffset = Math.min(offsetSamples, totalSamples);
} else if (offsetSamples < 0) { } else if (offsetSamples < 0) {
sourceOffset = Math.abs(offsetSamples); sourceSkipSamples = Math.abs(offsetSamples);
}
// Skip offset samples across decoded buffers
let samplesToSkip = sourceSkipSamples;
let bufferIndex = 0;
let bufferOffset = 0;
while (samplesToSkip > 0 && bufferIndex < decodedBuffers.length) {
const buffer = decodedBuffers[bufferIndex];
const availableInBuffer = buffer.length - bufferOffset;
if (samplesToSkip >= availableInBuffer) {
samplesToSkip -= availableInBuffer;
bufferIndex++;
bufferOffset = 0;
} else {
bufferOffset = samplesToSkip;
samplesToSkip = 0;
}
} }
// Copy decoded audio into the target buffer // Copy decoded audio into the merged buffer
const copyLength = Math.min(decoded.length - sourceOffset, totalSamples - writeOffset); while (writeOffset < totalSamples && bufferIndex < decodedBuffers.length) {
if (copyLength > 0) { const buffer = decodedBuffers[bufferIndex];
const remainingSamples = totalSamples - writeOffset;
const availableInBuffer = buffer.length - bufferOffset;
const writeLength = Math.min(availableInBuffer, remainingSamples);
for (let channel = 0; channel < channels; channel++) { for (let channel = 0; channel < channels; channel++) {
const sourceData = decoded.getChannelData(channel).subarray(sourceOffset, sourceOffset + copyLength); const channelData = buffer.getChannelData(channel).subarray(bufferOffset, bufferOffset + writeLength);
mergedBuffer.getChannelData(channel).set(sourceData, writeOffset); mergedBuffer.getChannelData(channel).set(channelData, writeOffset);
}
writeOffset += writeLength;
bufferOffset += writeLength;
if (bufferOffset >= buffer.length) {
bufferIndex++;
bufferOffset = 0;
} }
writeOffset += copyLength;
} }
// Loop audio to fill remaining duration if needed // Loop audio to fill remaining duration if needed
while (writeOffset < totalSamples) { while (writeOffset < totalSamples && decodedBuffers.length > 0) {
const remaining = totalSamples - writeOffset; for (const buffer of decodedBuffers) {
const loopLength = Math.min(decoded.length, remaining); const remainingSamples = totalSamples - writeOffset;
if (remainingSamples <= 0) break;
const writeLength = Math.min(buffer.length, remainingSamples);
for (let channel = 0; channel < channels; channel++) { for (let channel = 0; channel < channels; channel++) {
const sourceData = decoded.getChannelData(channel).subarray(0, loopLength); const channelData = buffer.getChannelData(channel).subarray(0, writeLength);
mergedBuffer.getChannelData(channel).set(sourceData, writeOffset); mergedBuffer.getChannelData(channel).set(channelData, writeOffset);
}
writeOffset += writeLength;
} }
writeOffset += loopLength;
} }
applyFades(mergedBuffer, options); applyFades(mergedBuffer, options);

Loading…
Cancel
Save