From 99197fc416969f24dd76a3328fb380beeead7cb5 Mon Sep 17 00:00:00 2001 From: shrimbly Date: Sat, 4 Apr 2026 20:36:04 +1300 Subject: [PATCH] fix: preserve embedded audio when stitching videos The video stitch node was only copying video tracks from source clips, silently discarding any embedded audio. Now extracts and concatenates audio from source videos when no external audio node is connected. Co-Authored-By: Claude Sonnet 4.5 --- src/components/ConnectionDropMenu.tsx | 9 +++ src/components/WorkflowCanvas.tsx | 2 +- src/components/edges/EditableEdge.tsx | 28 ++++++- src/components/nodes/EaseCurveNode.tsx | 4 - src/components/nodes/OutputGalleryNode.tsx | 12 ++- src/components/nodes/VideoFrameGrabNode.tsx | 48 ++++++------ src/hooks/useStitchVideos.ts | 75 ++++++++++++++++-- src/store/execution/simpleNodeExecutors.ts | 29 +++++-- src/store/utils/nodeDefaults.ts | 3 +- src/types/nodes.ts | 3 + src/utils/mediaStorage.ts | 86 +++++++++++++++++++-- 11 files changed, 244 insertions(+), 55 deletions(-) diff --git a/src/components/ConnectionDropMenu.tsx b/src/components/ConnectionDropMenu.tsx index ca8ff240..6fc0f706 100644 --- a/src/components/ConnectionDropMenu.tsx +++ b/src/components/ConnectionDropMenu.tsx @@ -411,6 +411,15 @@ const VIDEO_TARGET_OPTIONS: MenuOption[] = [ ), }, + { + type: "outputGallery", + label: "Output Gallery", + icon: ( + + + + ), + }, { type: "router", label: "Router", diff --git a/src/components/WorkflowCanvas.tsx b/src/components/WorkflowCanvas.tsx index 343de008..1eaa2b87 100644 --- a/src/components/WorkflowCanvas.tsx +++ b/src/components/WorkflowCanvas.tsx @@ -536,7 +536,7 @@ export function WorkflowCanvas() { if (!targetNode) return false; const targetNodeType = targetNode.type; - if (targetNodeType === "generateVideo" || targetNodeType === "videoStitch" || targetNodeType === "easeCurve" || targetNodeType === "videoTrim" || targetNodeType === "videoFrameGrab" || targetNodeType === "videoInput" || targetNodeType === "output" || targetNodeType === "router") { + if (targetNodeType === "generateVideo" || targetNodeType === "videoStitch" || targetNodeType === "easeCurve" || targetNodeType === "videoTrim" || targetNodeType === "videoFrameGrab" || targetNodeType === "videoInput" || targetNodeType === "output" || targetNodeType === "outputGallery" || targetNodeType === "router") { // For output node, we allow video even though its handle is typed as "image" // because output node can display both images and videos return true; diff --git a/src/components/edges/EditableEdge.tsx b/src/components/edges/EditableEdge.tsx index 1b4a13ad..7d78d9e9 100644 --- a/src/components/edges/EditableEdge.tsx +++ b/src/components/edges/EditableEdge.tsx @@ -103,6 +103,32 @@ export function EditableEdge({ // Calculate the path based on edge style const [edgePath, labelX, labelY] = useMemo(() => { + // Loop edges: smooth arc that exits/enters along handle directions, bowed below nodes + if (edgeData?.isLoop) { + const dist = Math.sqrt((targetX - sourceX) ** 2 + (targetY - sourceY) ** 2); + const extent = Math.max(100, dist * 0.4); + const drop = Math.max(120, dist * 0.4); + + // Direction vectors matching handle positions + const dir: Record = { + top: [0, -1], bottom: [0, 1], left: [-1, 0], right: [1, 0], + }; + const [sdx, sdy] = dir[sourcePosition] ?? [1, 0]; + const [tdx, tdy] = dir[targetPosition] ?? [-1, 0]; + + // Follow handle direction + push arc below the nodes + const cp1x = sourceX + sdx * extent; + const cp1y = sourceY + sdy * extent + drop; + const cp2x = targetX + tdx * extent; + const cp2y = targetY + tdy * extent + drop; + + const path = `M${sourceX},${sourceY} C${cp1x},${cp1y} ${cp2x},${cp2y} ${targetX},${targetY}`; + // Label at bezier midpoint (t=0.5) + const lx = 0.125 * sourceX + 0.375 * cp1x + 0.375 * cp2x + 0.125 * targetX; + const ly = 0.125 * sourceY + 0.375 * cp1y + 0.375 * cp2y + 0.125 * targetY; + return [path, lx, ly] as [string, number, number]; + } + if (edgeStyle === "curved") { return getBezierPath({ sourceX, @@ -125,7 +151,7 @@ export function EditableEdge({ offset: offsetX, }); } - }, [edgeStyle, sourceX, sourceY, sourcePosition, targetX, targetY, targetPosition, offsetX]); + }, [edgeStyle, edgeData?.isLoop, sourceX, sourceY, sourcePosition, targetX, targetY, targetPosition, offsetX]); // Calculate handle positions on the path segments (only for angular mode) const handlePositions = useMemo(() => { diff --git a/src/components/nodes/EaseCurveNode.tsx b/src/components/nodes/EaseCurveNode.tsx index 9e1b7c32..1548ca04 100644 --- a/src/components/nodes/EaseCurveNode.tsx +++ b/src/components/nodes/EaseCurveNode.tsx @@ -11,7 +11,6 @@ import { useVideoAutoplay } from "@/hooks/useVideoAutoplay"; type EaseCurveNodeType = Node; -const VIDEO_HEIGHT = 320; export function EaseCurveNode({ id, data, selected }: NodeProps) { const nodeData = data; @@ -120,7 +119,6 @@ export function EaseCurveNode({ id, data, selected }: NodeProps {renderHandles()}
@@ -151,7 +149,6 @@ export function EaseCurveNode({ id, data, selected }: NodeProps {renderHandles()}
@@ -175,7 +172,6 @@ export function EaseCurveNode({ id, data, selected }: NodeProps {renderHandles()} diff --git a/src/components/nodes/OutputGalleryNode.tsx b/src/components/nodes/OutputGalleryNode.tsx index d6d3a1db..b6f73101 100644 --- a/src/components/nodes/OutputGalleryNode.tsx +++ b/src/components/nodes/OutputGalleryNode.tsx @@ -112,14 +112,18 @@ export function OutputGalleryNode({ id, data, selected }: NodeProps - {/* Frame position toggle (only when source video connected) */} - {hasSourceVideo && ( -
- - -
- )} + {/* Frame position toggle */} +
+ + +
{/* Extract Frame button */}
diff --git a/src/hooks/useStitchVideos.ts b/src/hooks/useStitchVideos.ts index 13f0ad5d..c41b2b31 100644 --- a/src/hooks/useStitchVideos.ts +++ b/src/hooks/useStitchVideos.ts @@ -7,6 +7,7 @@ import { VideoSampleSink, VideoSampleSource, AudioBufferSource, + AudioBufferSink, BlobSource, ALL_FORMATS, BufferTarget, @@ -211,6 +212,68 @@ export async function stitchVideosAsync( rotation: aggregateRotation, }); + // Extract embedded audio from source videos when no external audio override is provided + let effectiveAudioData = audioData ?? null; + if (!effectiveAudioData) { + updateProgress('processing', 'Extracting audio from source videos...', 8); + const allAudioBuffers: AudioBuffer[] = []; + let referenceSampleRate: number | null = null; + let referenceChannels: number | null = null; + + for (let i = 0; i < videoBlobs.length; i++) { + try { + const blobSource = new BlobSource(videoBlobs[i]); + const input = new Input({ source: blobSource, formats: ALL_FORMATS }); + try { + const audioTracks = await input.getAudioTracks(); + if (audioTracks.length > 0) { + const audioTrack = audioTracks[0]; + const sink = new AudioBufferSink(audioTrack); + const duration = await input.computeDuration(); + for await (const wrapped of sink.buffers(0, duration)) { + if (referenceSampleRate === null) { + referenceSampleRate = wrapped.buffer.sampleRate; + referenceChannels = wrapped.buffer.numberOfChannels; + } + if ( + wrapped.buffer.sampleRate === referenceSampleRate && + wrapped.buffer.numberOfChannels === referenceChannels + ) { + allAudioBuffers.push(wrapped.buffer); + } + } + } + } finally { + input.dispose(); + } + } catch (err) { + console.warn(`Failed to extract audio from video ${i + 1}:`, err); + } + } + + if (allAudioBuffers.length > 0 && referenceSampleRate && referenceChannels) { + const totalSamples = allAudioBuffers.reduce((sum, b) => sum + b.length, 0); + const concatenated = new AudioBuffer({ + length: Math.max(1, totalSamples), + numberOfChannels: referenceChannels, + sampleRate: referenceSampleRate, + }); + + let offset = 0; + for (const buffer of allAudioBuffers) { + for (let ch = 0; ch < referenceChannels; ch++) { + concatenated.getChannelData(ch).set(buffer.getChannelData(ch), offset); + } + offset += buffer.length; + } + + effectiveAudioData = { + buffer: concatenated, + duration: totalSamples / referenceSampleRate, + }; + } + } + // Create output updateProgress('processing', 'Creating output container...', 10); @@ -226,19 +289,19 @@ export async function stitchVideosAsync( output.addVideoTrack(videoSource, { rotation: aggregateRotation, frameRate: MAX_OUTPUT_FPS }); - // Add audio track if provided + // Add audio track if provided (external audio input or extracted from source videos) let audioSource: AudioBufferSource | null = null; let pendingAudioBuffer: AudioBuffer | null = null; let outputStarted = false; - if (audioData) { + if (effectiveAudioData) { updateProgress('processing', 'Detecting supported audio codec...', 8); // Detect the best supported audio codec for MP4 // Try common codecs in order of preference: aac, mp3 (no opus - Twitter doesn't support it) const audioCodec = await getFirstEncodableAudioCodec(['aac', 'mp3'], { - numberOfChannels: audioData.buffer.numberOfChannels, - sampleRate: audioData.buffer.sampleRate, + numberOfChannels: effectiveAudioData.buffer.numberOfChannels, + sampleRate: effectiveAudioData.buffer.sampleRate, bitrate: 128000, }); @@ -255,7 +318,7 @@ export async function stitchVideosAsync( }); output.addAudioTrack(audioSource); - pendingAudioBuffer = audioData.buffer; + pendingAudioBuffer = effectiveAudioData.buffer; } } @@ -267,7 +330,7 @@ export async function stitchVideosAsync( // Writing audio after all video causes broken interleaving (Discord won't play audio). if (audioSource && pendingAudioBuffer) { updateProgress('processing', 'Encoding audio track...', 12); - const trimTarget = probedVideoDuration > 0 ? probedVideoDuration : audioData!.duration; + const trimTarget = probedVideoDuration > 0 ? probedVideoDuration : effectiveAudioData!.duration; const trimmedBuffer = trimAudioBuffer(pendingAudioBuffer, trimTarget); await audioSource.add(trimmedBuffer); await audioSource.close(); diff --git a/src/store/execution/simpleNodeExecutors.ts b/src/store/execution/simpleNodeExecutors.ts index 7ba3bafe..a1e6e844 100644 --- a/src/store/execution/simpleNodeExecutors.ts +++ b/src/store/execution/simpleNodeExecutors.ts @@ -290,21 +290,34 @@ export async function executeOutput(ctx: NodeExecutionContext): Promise { } /** - * OutputGallery node: accumulates images from upstream nodes. + * OutputGallery node: accumulates images and videos from upstream nodes. */ export async function executeOutputGallery(ctx: NodeExecutionContext): Promise { const { node, getConnectedInputs, updateNodeData, getFreshNode } = ctx; - const { images } = getConnectedInputs(node.id); + const { images, videos } = getConnectedInputs(node.id); // Use fresh node data — the stale `node` from topological sort may be missing // images pushed by appendOutputGalleryImage during upstream batch execution. const freshNode = getFreshNode(node.id); - const galleryImages = ((freshNode?.data ?? node.data) as OutputGalleryNodeData).images || []; - const existing = new Set(galleryImages); - const newImages = images.filter((img) => !existing.has(img)); + const freshData = (freshNode?.data ?? node.data) as OutputGalleryNodeData; + const galleryImages = freshData.images || []; + const galleryVideos = freshData.videos || []; + + const updates: Partial = {}; + + const existingImages = new Set(galleryImages); + const newImages = images.filter((img) => !existingImages.has(img)); if (newImages.length > 0) { - updateNodeData(node.id, { - images: [...newImages, ...galleryImages], - }); + updates.images = [...newImages, ...galleryImages]; + } + + const existingVideos = new Set(galleryVideos); + const newVideos = videos.filter((v) => !existingVideos.has(v)); + if (newVideos.length > 0) { + updates.videos = [...newVideos, ...galleryVideos]; + } + + if (Object.keys(updates).length > 0) { + updateNodeData(node.id, updates); } } diff --git a/src/store/utils/nodeDefaults.ts b/src/store/utils/nodeDefaults.ts index 3a2f9c27..3cc3a16f 100644 --- a/src/store/utils/nodeDefaults.ts +++ b/src/store/utils/nodeDefaults.ts @@ -53,7 +53,7 @@ export const defaultNodeDimensions: Record { case "outputGallery": return { images: [], + videos: [], } as OutputGalleryNodeData; case "imageCompare": return { diff --git a/src/types/nodes.ts b/src/types/nodes.ts index 18568bce..3ca67905 100644 --- a/src/types/nodes.ts +++ b/src/types/nodes.ts @@ -308,6 +308,9 @@ export interface OutputNodeData extends BaseNodeData { */ export interface OutputGalleryNodeData extends BaseNodeData { images: string[]; // Array of base64 data URLs from connected nodes + imageRefs?: string[]; // External storage refs for images + videos: string[]; // Array of video URLs from connected nodes + videoRefs?: string[]; // External storage refs for videos } /** diff --git a/src/utils/mediaStorage.ts b/src/utils/mediaStorage.ts index 7c1da3c5..585417d0 100644 --- a/src/utils/mediaStorage.ts +++ b/src/utils/mediaStorage.ts @@ -436,9 +436,57 @@ async function externalizeNodeMedia( case "outputGallery": { const d = data as import("@/types").OutputGalleryNodeData; - // OutputGallery content is regenerated on each workflow run - // Clear images array to keep workflow file small - newData = { ...d, images: [] }; + const galleryImageRefs: string[] = d.imageRefs ? [...d.imageRefs] : []; + const galleryVideoRefs: string[] = d.videoRefs ? [...d.videoRefs] : []; + + // Externalize gallery images + for (let i = 0; i < (d.images?.length || 0); i++) { + const img = d.images[i]; + const existingRef = galleryImageRefs[i]; + if (existingRef && isDataUrl(img)) { + // Already has ref, just keep it + } else if (isDataUrl(img)) { + galleryImageRefs[i] = await saveImageAndGetId(img, workflowPath, savedImageIds, "generations"); + } else if (isHttpUrl(img)) { + // Keep HTTP URLs inline (small, no ref needed) + galleryImageRefs[i] = ""; // placeholder to keep indices aligned + } + } + + // Externalize gallery videos + for (let i = 0; i < (d.videos?.length || 0); i++) { + const vid = d.videos[i]; + const existingRef = galleryVideoRefs[i]; + if (existingRef && isDataUrl(vid)) { + // Already has ref, just keep it + } else if (isDataUrl(vid)) { + const ref = await saveVideoAndGetRef(vid, workflowPath, savedMediaIds); + if (ref) galleryVideoRefs[i] = ref; + } else if (isHttpUrl(vid)) { + // Keep HTTP URLs inline (small, no ref needed) + galleryVideoRefs[i] = ""; // placeholder to keep indices aligned + } + } + + // Build cleaned arrays: clear base64 data where refs exist, keep HTTP URLs + const cleanedImages = d.images.map((img, i) => + galleryImageRefs[i] && isDataUrl(img) ? "" : img + ); + const cleanedVideos = d.videos.map((vid, i) => + galleryVideoRefs[i] && isDataUrl(vid) ? "" : vid + ); + + // Filter out empty placeholders from refs + const hasImageRefs = galleryImageRefs.some(r => r && r !== ""); + const hasVideoRefs = galleryVideoRefs.some(r => r && r !== ""); + + newData = { + ...d, + images: cleanedImages, + imageRefs: hasImageRefs ? galleryImageRefs : undefined, + videos: cleanedVideos, + videoRefs: hasVideoRefs ? galleryVideoRefs : undefined, + }; break; } @@ -990,8 +1038,36 @@ async function hydrateNodeMedia( } case "outputGallery": { - // OutputGallery content is not persisted - it's regenerated on each workflow run - newData = data; + const d = data as import("@/types").OutputGalleryNodeData; + const images = [...(d.images || [])]; + const videos = [...(d.videos || [])]; + + // Hydrate images from refs + if (d.imageRefs && d.imageRefs.length > 0) { + for (let i = 0; i < d.imageRefs.length; i++) { + const ref = d.imageRefs[i]; + if (ref && ref !== "" && (!images[i] || images[i] === "")) { + images[i] = await loadMediaById(ref, workflowPath, loadedMedia, "image"); + } + } + } + + // Hydrate videos from refs + if (d.videoRefs && d.videoRefs.length > 0) { + for (let i = 0; i < d.videoRefs.length; i++) { + const ref = d.videoRefs[i]; + if (ref && ref !== "" && (!videos[i] || videos[i] === "")) { + videos[i] = await loadMediaById(ref, workflowPath, loadedMedia, "video"); + } + } + } + + // Filter out any empty entries that failed to hydrate + newData = { + ...d, + images: images.filter(img => img && img !== ""), + videos: videos.filter(vid => vid && vid !== ""), + }; break; }