Browse Source

feat: add audio model detection to Replicate & WaveSpeed providers

Audio models (music, TTS, speech) were misclassified as text-to-image
because capability inference only checked for 3D and video keywords.
This caused audio models to not appear in GenerateAudio's model browser
and to create nanoBanana nodes instead of generateAudio nodes.

- Add audio keyword detection to inferReplicateCapabilities()
- Add audio keyword detection to inferWaveSpeedCapabilities()
- Add audio output handling to replicate.ts provider
- Add audio output handling to wavespeed.ts provider

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
handoff-20260429-1057
shrimbly 5 months ago
parent
commit
6d4d19948b
  1. 18
      src/app/api/generate/providers/replicate.ts
  2. 22
      src/app/api/generate/providers/wavespeed.ts
  3. 35
      src/app/api/models/route.ts

18
src/app/api/generate/providers/replicate.ts

@ -280,6 +280,8 @@ export async function generateWithReplicate(
// Determine MIME type from response
const contentType = mediaResponse.headers.get("content-type") || "image/png";
const isVideo = contentType.startsWith("video/");
const isAudio = contentType.startsWith("audio/") ||
input.model.capabilities.some(c => c.includes("audio"));
const mediaArrayBuffer = await mediaResponse.arrayBuffer();
const mediaSizeBytes = mediaArrayBuffer.byteLength;
@ -303,6 +305,22 @@ export async function generateWithReplicate(
}
const mediaBase64 = Buffer.from(mediaArrayBuffer).toString("base64");
if (isAudio) {
const audioContentType = contentType.startsWith("audio/") ? contentType : "audio/mpeg";
console.log(`[API:${requestId}] SUCCESS - Returning audio`);
return {
success: true,
outputs: [
{
type: "audio",
data: `data:${audioContentType};base64,${mediaBase64}`,
url: mediaUrl,
},
],
};
}
console.log(`[API:${requestId}] SUCCESS - Returning ${isVideo ? "video" : "image"}`);
return {

22
src/app/api/generate/providers/wavespeed.ts

@ -373,10 +373,11 @@ export async function generateWithWaveSpeed(
const outputSizeMB = outputArrayBuffer.byteLength / (1024 * 1024);
const rawContentType = outputResponse.headers.get("content-type");
const isAudioModel = input.model.capabilities.some(c => c.includes("audio"));
const contentType =
(rawContentType && (rawContentType.startsWith("video/") || rawContentType.startsWith("image/")))
(rawContentType && (rawContentType.startsWith("video/") || rawContentType.startsWith("image/") || rawContentType.startsWith("audio/")))
? rawContentType
: (isVideoModel ? "video/mp4" : "image/png");
: (isVideoModel ? "video/mp4" : isAudioModel ? "audio/mpeg" : "image/png");
console.log(`[API:${requestId}] Output: ${contentType}, ${outputSizeMB.toFixed(2)}MB`);
@ -396,6 +397,23 @@ export async function generateWithWaveSpeed(
}
const outputBase64 = Buffer.from(outputArrayBuffer).toString("base64");
const isAudio = contentType.startsWith("audio/") || isAudioModel;
if (isAudio) {
const audioContentType = contentType.startsWith("audio/") ? contentType : "audio/mpeg";
console.log(`[API:${requestId}] SUCCESS - Returning audio`);
return {
success: true,
outputs: [
{
type: "audio",
data: `data:${audioContentType};base64,${outputBase64}`,
url: outputUrl,
},
],
};
}
console.log(`[API:${requestId}] SUCCESS - Returning ${isVideoModel ? "video" : "image"}`);
return {

35
src/app/api/models/route.ts

@ -491,6 +491,23 @@ function inferReplicateCapabilities(model: ReplicateModel): ModelCapability[] {
return capabilities;
}
// Check for audio-related keywords
const isAudioModel =
searchText.includes("music") ||
searchText.includes("audio") ||
searchText.includes("tts") ||
searchText.includes("text-to-speech") ||
searchText.includes("speech") ||
searchText.includes("sound effect") ||
searchText.includes("voice") ||
searchText.includes("bark") ||
searchText.includes("xtts");
if (isAudioModel) {
capabilities.push("text-to-audio");
return capabilities;
}
// Check for video-related keywords
const isVideoModel =
searchText.includes("video") ||
@ -651,6 +668,24 @@ function inferWaveSpeedCapabilities(model: WaveSpeedModel): ModelCapability[] {
return capabilities;
}
// Check for audio-related keywords
const isAudioModel =
searchText.includes("music") ||
searchText.includes("audio") ||
searchText.includes("tts") ||
searchText.includes("text-to-speech") ||
searchText.includes("speech") ||
searchText.includes("sound effect") ||
searchText.includes("voice") ||
category.includes("audio") ||
category.includes("music") ||
category.includes("speech");
if (isAudioModel) {
capabilities.push("text-to-audio");
return capabilities;
}
// Check for video-related keywords
const isVideoModel =
searchText.includes("video") ||

Loading…
Cancel
Save