Browse Source

fix: preserve embedded audio when stitching videos

The video stitch node was only copying video tracks from source clips,
silently discarding any embedded audio. Now extracts and concatenates
audio from source videos when no external audio node is connected.

Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
handoff-20260429-1057
shrimbly 3 months ago
parent
commit
99197fc416
  1. 9
      src/components/ConnectionDropMenu.tsx
  2. 2
      src/components/WorkflowCanvas.tsx
  3. 28
      src/components/edges/EditableEdge.tsx
  4. 4
      src/components/nodes/EaseCurveNode.tsx
  5. 12
      src/components/nodes/OutputGalleryNode.tsx
  6. 4
      src/components/nodes/VideoFrameGrabNode.tsx
  7. 75
      src/hooks/useStitchVideos.ts
  8. 29
      src/store/execution/simpleNodeExecutors.ts
  9. 3
      src/store/utils/nodeDefaults.ts
  10. 3
      src/types/nodes.ts
  11. 86
      src/utils/mediaStorage.ts

9
src/components/ConnectionDropMenu.tsx

@ -411,6 +411,15 @@ const VIDEO_TARGET_OPTIONS: MenuOption[] = [
</svg>
),
},
{
type: "outputGallery",
label: "Output Gallery",
icon: (
<svg className="w-4 h-4" fill="none" viewBox="0 0 24 24" stroke="currentColor" strokeWidth={1.5}>
<path strokeLinecap="round" strokeLinejoin="round" d="M3.75 6A2.25 2.25 0 016 3.75h2.25A2.25 2.25 0 0110.5 6v2.25a2.25 2.25 0 01-2.25 2.25H6a2.25 2.25 0 01-2.25-2.25V6zM3.75 15.75A2.25 2.25 0 016 13.5h2.25a2.25 2.25 0 012.25 2.25V18a2.25 2.25 0 01-2.25 2.25H6A2.25 2.25 0 013.75 18v-2.25zM13.5 6a2.25 2.25 0 012.25-2.25H18A2.25 2.25 0 0120.25 6v2.25A2.25 2.25 0 0118 10.5h-2.25a2.25 2.25 0 01-2.25-2.25V6zM13.5 15.75a2.25 2.25 0 012.25-2.25H18a2.25 2.25 0 012.25 2.25V18A2.25 2.25 0 0118 20.25h-2.25A2.25 2.25 0 0113.5 18v-2.25z" />
</svg>
),
},
{
type: "router",
label: "Router",

2
src/components/WorkflowCanvas.tsx

@ -536,7 +536,7 @@ export function WorkflowCanvas() {
if (!targetNode) return false;
const targetNodeType = targetNode.type;
if (targetNodeType === "generateVideo" || targetNodeType === "videoStitch" || targetNodeType === "easeCurve" || targetNodeType === "videoTrim" || targetNodeType === "videoFrameGrab" || targetNodeType === "videoInput" || targetNodeType === "output" || targetNodeType === "router") {
if (targetNodeType === "generateVideo" || targetNodeType === "videoStitch" || targetNodeType === "easeCurve" || targetNodeType === "videoTrim" || targetNodeType === "videoFrameGrab" || targetNodeType === "videoInput" || targetNodeType === "output" || targetNodeType === "outputGallery" || targetNodeType === "router") {
// For output node, we allow video even though its handle is typed as "image"
// because output node can display both images and videos
return true;

28
src/components/edges/EditableEdge.tsx

@ -103,6 +103,32 @@ export function EditableEdge({
// Calculate the path based on edge style
const [edgePath, labelX, labelY] = useMemo(() => {
// Loop edges: smooth arc that exits/enters along handle directions, bowed below nodes
if (edgeData?.isLoop) {
const dist = Math.sqrt((targetX - sourceX) ** 2 + (targetY - sourceY) ** 2);
const extent = Math.max(100, dist * 0.4);
const drop = Math.max(120, dist * 0.4);
// Direction vectors matching handle positions
const dir: Record<string, [number, number]> = {
top: [0, -1], bottom: [0, 1], left: [-1, 0], right: [1, 0],
};
const [sdx, sdy] = dir[sourcePosition] ?? [1, 0];
const [tdx, tdy] = dir[targetPosition] ?? [-1, 0];
// Follow handle direction + push arc below the nodes
const cp1x = sourceX + sdx * extent;
const cp1y = sourceY + sdy * extent + drop;
const cp2x = targetX + tdx * extent;
const cp2y = targetY + tdy * extent + drop;
const path = `M${sourceX},${sourceY} C${cp1x},${cp1y} ${cp2x},${cp2y} ${targetX},${targetY}`;
// Label at bezier midpoint (t=0.5)
const lx = 0.125 * sourceX + 0.375 * cp1x + 0.375 * cp2x + 0.125 * targetX;
const ly = 0.125 * sourceY + 0.375 * cp1y + 0.375 * cp2y + 0.125 * targetY;
return [path, lx, ly] as [string, number, number];
}
if (edgeStyle === "curved") {
return getBezierPath({
sourceX,
@ -125,7 +151,7 @@ export function EditableEdge({
offset: offsetX,
});
}
}, [edgeStyle, sourceX, sourceY, sourcePosition, targetX, targetY, targetPosition, offsetX]);
}, [edgeStyle, edgeData?.isLoop, sourceX, sourceY, sourcePosition, targetX, targetY, targetPosition, offsetX]);
// Calculate handle positions on the path segments (only for angular mode)
const handlePositions = useMemo(() => {

4
src/components/nodes/EaseCurveNode.tsx

@ -11,7 +11,6 @@ import { useVideoAutoplay } from "@/hooks/useVideoAutoplay";
type EaseCurveNodeType = Node<EaseCurveNodeData, "easeCurve">;
const VIDEO_HEIGHT = 320;
export function EaseCurveNode({ id, data, selected }: NodeProps<EaseCurveNodeType>) {
const nodeData = data;
@ -120,7 +119,6 @@ export function EaseCurveNode({ id, data, selected }: NodeProps<EaseCurveNodeTyp
selected={selected}
fullBleed
minWidth={340}
minHeight={VIDEO_HEIGHT}
>
{renderHandles()}
<div className="flex-1 flex flex-col items-center justify-center gap-2 text-center px-4">
@ -151,7 +149,6 @@ export function EaseCurveNode({ id, data, selected }: NodeProps<EaseCurveNodeTyp
selected={selected}
fullBleed
minWidth={340}
minHeight={VIDEO_HEIGHT}
>
{renderHandles()}
<div className="flex-1 flex items-center justify-center">
@ -175,7 +172,6 @@ export function EaseCurveNode({ id, data, selected }: NodeProps<EaseCurveNodeTyp
isExecuting={isRunning}
hasError={nodeData.status === "error"}
minWidth={340}
minHeight={VIDEO_HEIGHT}
aspectFitMedia={nodeData.outputVideo}
>
{renderHandles()}

12
src/components/nodes/OutputGalleryNode.tsx

@ -112,14 +112,18 @@ export function OutputGalleryNode({ id, data, selected }: NodeProps<OutputGaller
const images = nodeData.images || [];
const videos = nodeData.videos || [];
for (let i = 0; i < images.length; i++) {
const nodeId = addNode("imageInput", { x: startX, y: currentY }, { image: images[i], filename: `gallery-image-${i + 1}.png` });
// Reverse so oldest items (end of array) appear at top, newest at bottom
const reversedImages = [...images].reverse();
const reversedVideos = [...videos].reverse();
for (let i = 0; i < reversedImages.length; i++) {
const nodeId = addNode("imageInput", { x: startX, y: currentY }, { image: reversedImages[i], filename: `gallery-image-${i + 1}.png` });
newNodeIds.push(nodeId);
currentY += defaultNodeDimensions.imageInput.height + gap;
}
for (let i = 0; i < videos.length; i++) {
const nodeId = addNode("videoInput", { x: startX, y: currentY }, { video: videos[i], filename: `gallery-video-${i + 1}.mp4` });
for (let i = 0; i < reversedVideos.length; i++) {
const nodeId = addNode("videoInput", { x: startX, y: currentY }, { video: reversedVideos[i], filename: `gallery-video-${i + 1}.mp4` });
newNodeIds.push(nodeId);
currentY += defaultNodeDimensions.videoInput.height + gap;
}

4
src/components/nodes/VideoFrameGrabNode.tsx

@ -109,8 +109,7 @@ export function VideoFrameGrabNode({ id, data, selected }: NodeProps<VideoFrameG
)}
</div>
{/* Frame position toggle (only when source video connected) */}
{hasSourceVideo && (
{/* Frame position toggle */}
<div className="nodrag nowheel shrink-0 flex gap-1 px-1">
<button
onClick={() => updateNodeData(id, { framePosition: "first", outputImage: null })}
@ -133,7 +132,6 @@ export function VideoFrameGrabNode({ id, data, selected }: NodeProps<VideoFrameG
Last
</button>
</div>
)}
{/* Extract Frame button */}
<div className="shrink-0 flex justify-end px-1">

75
src/hooks/useStitchVideos.ts

@ -7,6 +7,7 @@ import {
VideoSampleSink,
VideoSampleSource,
AudioBufferSource,
AudioBufferSink,
BlobSource,
ALL_FORMATS,
BufferTarget,
@ -211,6 +212,68 @@ export async function stitchVideosAsync(
rotation: aggregateRotation,
});
// Extract embedded audio from source videos when no external audio override is provided
let effectiveAudioData = audioData ?? null;
if (!effectiveAudioData) {
updateProgress('processing', 'Extracting audio from source videos...', 8);
const allAudioBuffers: AudioBuffer[] = [];
let referenceSampleRate: number | null = null;
let referenceChannels: number | null = null;
for (let i = 0; i < videoBlobs.length; i++) {
try {
const blobSource = new BlobSource(videoBlobs[i]);
const input = new Input({ source: blobSource, formats: ALL_FORMATS });
try {
const audioTracks = await input.getAudioTracks();
if (audioTracks.length > 0) {
const audioTrack = audioTracks[0];
const sink = new AudioBufferSink(audioTrack);
const duration = await input.computeDuration();
for await (const wrapped of sink.buffers(0, duration)) {
if (referenceSampleRate === null) {
referenceSampleRate = wrapped.buffer.sampleRate;
referenceChannels = wrapped.buffer.numberOfChannels;
}
if (
wrapped.buffer.sampleRate === referenceSampleRate &&
wrapped.buffer.numberOfChannels === referenceChannels
) {
allAudioBuffers.push(wrapped.buffer);
}
}
}
} finally {
input.dispose();
}
} catch (err) {
console.warn(`Failed to extract audio from video ${i + 1}:`, err);
}
}
if (allAudioBuffers.length > 0 && referenceSampleRate && referenceChannels) {
const totalSamples = allAudioBuffers.reduce((sum, b) => sum + b.length, 0);
const concatenated = new AudioBuffer({
length: Math.max(1, totalSamples),
numberOfChannels: referenceChannels,
sampleRate: referenceSampleRate,
});
let offset = 0;
for (const buffer of allAudioBuffers) {
for (let ch = 0; ch < referenceChannels; ch++) {
concatenated.getChannelData(ch).set(buffer.getChannelData(ch), offset);
}
offset += buffer.length;
}
effectiveAudioData = {
buffer: concatenated,
duration: totalSamples / referenceSampleRate,
};
}
}
// Create output
updateProgress('processing', 'Creating output container...', 10);
@ -226,19 +289,19 @@ export async function stitchVideosAsync(
output.addVideoTrack(videoSource, { rotation: aggregateRotation, frameRate: MAX_OUTPUT_FPS });
// Add audio track if provided
// Add audio track if provided (external audio input or extracted from source videos)
let audioSource: AudioBufferSource | null = null;
let pendingAudioBuffer: AudioBuffer | null = null;
let outputStarted = false;
if (audioData) {
if (effectiveAudioData) {
updateProgress('processing', 'Detecting supported audio codec...', 8);
// Detect the best supported audio codec for MP4
// Try common codecs in order of preference: aac, mp3 (no opus - Twitter doesn't support it)
const audioCodec = await getFirstEncodableAudioCodec(['aac', 'mp3'], {
numberOfChannels: audioData.buffer.numberOfChannels,
sampleRate: audioData.buffer.sampleRate,
numberOfChannels: effectiveAudioData.buffer.numberOfChannels,
sampleRate: effectiveAudioData.buffer.sampleRate,
bitrate: 128000,
});
@ -255,7 +318,7 @@ export async function stitchVideosAsync(
});
output.addAudioTrack(audioSource);
pendingAudioBuffer = audioData.buffer;
pendingAudioBuffer = effectiveAudioData.buffer;
}
}
@ -267,7 +330,7 @@ export async function stitchVideosAsync(
// Writing audio after all video causes broken interleaving (Discord won't play audio).
if (audioSource && pendingAudioBuffer) {
updateProgress('processing', 'Encoding audio track...', 12);
const trimTarget = probedVideoDuration > 0 ? probedVideoDuration : audioData!.duration;
const trimTarget = probedVideoDuration > 0 ? probedVideoDuration : effectiveAudioData!.duration;
const trimmedBuffer = trimAudioBuffer(pendingAudioBuffer, trimTarget);
await audioSource.add(trimmedBuffer);
await audioSource.close();

29
src/store/execution/simpleNodeExecutors.ts

@ -290,21 +290,34 @@ export async function executeOutput(ctx: NodeExecutionContext): Promise<void> {
}
/**
* OutputGallery node: accumulates images from upstream nodes.
* OutputGallery node: accumulates images and videos from upstream nodes.
*/
export async function executeOutputGallery(ctx: NodeExecutionContext): Promise<void> {
const { node, getConnectedInputs, updateNodeData, getFreshNode } = ctx;
const { images } = getConnectedInputs(node.id);
const { images, videos } = getConnectedInputs(node.id);
// Use fresh node data — the stale `node` from topological sort may be missing
// images pushed by appendOutputGalleryImage during upstream batch execution.
const freshNode = getFreshNode(node.id);
const galleryImages = ((freshNode?.data ?? node.data) as OutputGalleryNodeData).images || [];
const existing = new Set(galleryImages);
const newImages = images.filter((img) => !existing.has(img));
const freshData = (freshNode?.data ?? node.data) as OutputGalleryNodeData;
const galleryImages = freshData.images || [];
const galleryVideos = freshData.videos || [];
const updates: Partial<OutputGalleryNodeData> = {};
const existingImages = new Set(galleryImages);
const newImages = images.filter((img) => !existingImages.has(img));
if (newImages.length > 0) {
updateNodeData(node.id, {
images: [...newImages, ...galleryImages],
});
updates.images = [...newImages, ...galleryImages];
}
const existingVideos = new Set(galleryVideos);
const newVideos = videos.filter((v) => !existingVideos.has(v));
if (newVideos.length > 0) {
updates.videos = [...newVideos, ...galleryVideos];
}
if (Object.keys(updates).length > 0) {
updateNodeData(node.id, updates);
}
}

3
src/store/utils/nodeDefaults.ts

@ -53,7 +53,7 @@ export const defaultNodeDimensions: Record<NodeType, { width: number; height: nu
outputGallery: { width: 320, height: 360 },
imageCompare: { width: 400, height: 360 },
videoStitch: { width: 400, height: 280 },
easeCurve: { width: 340, height: 480 },
easeCurve: { width: 340, height: 280 },
videoTrim: { width: 360, height: 360 },
videoFrameGrab: { width: 320, height: 320 },
router: { width: 200, height: 80 },
@ -258,6 +258,7 @@ export const createDefaultNodeData = (type: NodeType): WorkflowNodeData => {
case "outputGallery":
return {
images: [],
videos: [],
} as OutputGalleryNodeData;
case "imageCompare":
return {

3
src/types/nodes.ts

@ -308,6 +308,9 @@ export interface OutputNodeData extends BaseNodeData {
*/
export interface OutputGalleryNodeData extends BaseNodeData {
images: string[]; // Array of base64 data URLs from connected nodes
imageRefs?: string[]; // External storage refs for images
videos: string[]; // Array of video URLs from connected nodes
videoRefs?: string[]; // External storage refs for videos
}
/**

86
src/utils/mediaStorage.ts

@ -436,9 +436,57 @@ async function externalizeNodeMedia(
case "outputGallery": {
const d = data as import("@/types").OutputGalleryNodeData;
// OutputGallery content is regenerated on each workflow run
// Clear images array to keep workflow file small
newData = { ...d, images: [] };
const galleryImageRefs: string[] = d.imageRefs ? [...d.imageRefs] : [];
const galleryVideoRefs: string[] = d.videoRefs ? [...d.videoRefs] : [];
// Externalize gallery images
for (let i = 0; i < (d.images?.length || 0); i++) {
const img = d.images[i];
const existingRef = galleryImageRefs[i];
if (existingRef && isDataUrl(img)) {
// Already has ref, just keep it
} else if (isDataUrl(img)) {
galleryImageRefs[i] = await saveImageAndGetId(img, workflowPath, savedImageIds, "generations");
} else if (isHttpUrl(img)) {
// Keep HTTP URLs inline (small, no ref needed)
galleryImageRefs[i] = ""; // placeholder to keep indices aligned
}
}
// Externalize gallery videos
for (let i = 0; i < (d.videos?.length || 0); i++) {
const vid = d.videos[i];
const existingRef = galleryVideoRefs[i];
if (existingRef && isDataUrl(vid)) {
// Already has ref, just keep it
} else if (isDataUrl(vid)) {
const ref = await saveVideoAndGetRef(vid, workflowPath, savedMediaIds);
if (ref) galleryVideoRefs[i] = ref;
} else if (isHttpUrl(vid)) {
// Keep HTTP URLs inline (small, no ref needed)
galleryVideoRefs[i] = ""; // placeholder to keep indices aligned
}
}
// Build cleaned arrays: clear base64 data where refs exist, keep HTTP URLs
const cleanedImages = d.images.map((img, i) =>
galleryImageRefs[i] && isDataUrl(img) ? "" : img
);
const cleanedVideos = d.videos.map((vid, i) =>
galleryVideoRefs[i] && isDataUrl(vid) ? "" : vid
);
// Filter out empty placeholders from refs
const hasImageRefs = galleryImageRefs.some(r => r && r !== "");
const hasVideoRefs = galleryVideoRefs.some(r => r && r !== "");
newData = {
...d,
images: cleanedImages,
imageRefs: hasImageRefs ? galleryImageRefs : undefined,
videos: cleanedVideos,
videoRefs: hasVideoRefs ? galleryVideoRefs : undefined,
};
break;
}
@ -990,8 +1038,36 @@ async function hydrateNodeMedia(
}
case "outputGallery": {
// OutputGallery content is not persisted - it's regenerated on each workflow run
newData = data;
const d = data as import("@/types").OutputGalleryNodeData;
const images = [...(d.images || [])];
const videos = [...(d.videos || [])];
// Hydrate images from refs
if (d.imageRefs && d.imageRefs.length > 0) {
for (let i = 0; i < d.imageRefs.length; i++) {
const ref = d.imageRefs[i];
if (ref && ref !== "" && (!images[i] || images[i] === "")) {
images[i] = await loadMediaById(ref, workflowPath, loadedMedia, "image");
}
}
}
// Hydrate videos from refs
if (d.videoRefs && d.videoRefs.length > 0) {
for (let i = 0; i < d.videoRefs.length; i++) {
const ref = d.videoRefs[i];
if (ref && ref !== "" && (!videos[i] || videos[i] === "")) {
videos[i] = await loadMediaById(ref, workflowPath, loadedMedia, "video");
}
}
}
// Filter out any empty entries that failed to hydrate
newData = {
...d,
images: images.filter(img => img && img !== ""),
videos: videos.filter(vid => vid && vid !== ""),
};
break;
}

Loading…
Cancel
Save