From a064ddc10641ebb6f018636f41c627923f976125 Mon Sep 17 00:00:00 2001 From: shrimbly Date: Tue, 31 Mar 2026 06:19:00 +1300 Subject: [PATCH] feat: add VideoInput node for uploading/loading video files Adds a new videoInput node type that allows users to drag-and-drop or browse for video files (MP4, WebM, QuickTime), preview them inline with native video controls, and feed them into downstream video processing nodes. Includes full media externalization/hydration support, Shift+Y keyboard shortcut, and ConnectionDropMenu integration. Co-Authored-By: Claude Opus 4.6 --- src/components/ConnectionDropMenu.tsx | 22 ++- src/components/WorkflowCanvas.tsx | 60 ++++++- src/components/nodes/VideoInputNode.tsx | 212 ++++++++++++++++++++++++ src/components/nodes/index.ts | 1 + src/lib/quickstart/validation.ts | 10 ++ src/store/execution/executeNode.ts | 8 + src/store/utils/connectedInputs.ts | 3 + src/store/utils/executionUtils.ts | 1 + src/store/utils/nodeDefaults.ts | 10 ++ src/store/workflowStore.ts | 12 ++ src/types/nodes.ts | 15 ++ src/utils/mediaStorage.ts | 28 ++++ 12 files changed, 377 insertions(+), 5 deletions(-) create mode 100644 src/components/nodes/VideoInputNode.tsx diff --git a/src/components/ConnectionDropMenu.tsx b/src/components/ConnectionDropMenu.tsx index e2730b35..8b3a60f4 100644 --- a/src/components/ConnectionDropMenu.tsx +++ b/src/components/ConnectionDropMenu.tsx @@ -346,8 +346,17 @@ const TEXT_SOURCE_OPTIONS: MenuOption[] = [ }, ]; -// Video can only connect to videoStitch, generateVideo (video-to-video), or output nodes +// Video can connect to videoStitch, videoInput, generateVideo (video-to-video), or output nodes const VIDEO_TARGET_OPTIONS: MenuOption[] = [ + { + type: "videoInput", + label: "Video Input", + icon: ( + + + + ), + }, { type: "videoStitch", label: "Video Stitch", @@ -425,8 +434,17 @@ const VIDEO_TARGET_OPTIONS: MenuOption[] = [ }, ]; -// GenerateVideo and VideoStitch nodes produce video output +// GenerateVideo, VideoStitch, and VideoInput nodes produce video output const VIDEO_SOURCE_OPTIONS: MenuOption[] = [ + { + type: "videoInput", + label: "Video Input", + icon: ( + + + + ), + }, { type: "generateVideo", label: "Generate Video", diff --git a/src/components/WorkflowCanvas.tsx b/src/components/WorkflowCanvas.tsx index 2b2349d1..bb3de3a9 100644 --- a/src/components/WorkflowCanvas.tsx +++ b/src/components/WorkflowCanvas.tsx @@ -25,6 +25,7 @@ import dynamic from "next/dynamic"; import { ImageInputNode, AudioInputNode, + VideoInputNode, AnnotationNode, PromptNode, ArrayNode, @@ -55,7 +56,7 @@ import { MultiSelectToolbar } from "./MultiSelectToolbar"; import { EdgeToolbar } from "./EdgeToolbar"; import { GlobalImageHistory } from "./GlobalImageHistory"; import { GroupBackgroundsPortal, GroupControlsOverlay } from "./GroupsOverlay"; -import { NodeType, NanoBananaNodeData, HandleType } from "@/types"; +import { NodeType, NanoBananaNodeData, HandleType, PromptNodeData, LLMGenerateNodeData, PromptConstructorNodeData, AvailableVariable } from "@/types"; import { defaultNodeDimensions } from "@/store/utils/nodeDefaults"; import { FloatingNodeHeader } from "./nodes/FloatingNodeHeader"; import { ControlPanel } from "./nodes/ControlPanel"; @@ -67,6 +68,9 @@ import { ChatPanel } from "./ChatPanel"; import { EditOperation } from "@/lib/chat/editOperations"; import { stripBinaryData } from "@/lib/chat/contextBuilder"; import { PromptEditorModal } from "./modals/PromptEditorModal"; +import { PromptConstructorEditorModal } from "./modals/PromptConstructorEditorModal"; +import { resolveTextSourcesThroughRouters } from "@/store/utils/connectedInputs"; +import { parseVarTags } from "@/utils/parseVarTags"; import { AnnotationModal } from "./AnnotationModal"; import { browseRegistry } from "@/utils/browseRegistry"; import { useInlineParameters } from "@/hooks/useInlineParameters"; @@ -77,6 +81,7 @@ import { useAnnotationStore } from "@/store/annotationStore"; const nodeTypes: NodeTypes = { imageInput: ImageInputNode, audioInput: AudioInputNode, + videoInput: VideoInputNode, annotation: AnnotationNode, prompt: PromptNode, array: ArrayNode, @@ -137,6 +142,8 @@ const getNodeHandles = (nodeType: string): { inputs: string[]; outputs: string[] return { inputs: ["reference"], outputs: ["image"] }; case "audioInput": return { inputs: ["audio"], outputs: ["audio"] }; + case "videoInput": + return { inputs: ["video"], outputs: ["video"] }; case "annotation": return { inputs: ["image"], outputs: ["image"] }; case "prompt": @@ -365,6 +372,7 @@ export function WorkflowCanvas() { const NODE_TITLES: Record = { imageInput: 'Image Input', audioInput: 'Audio Input', + videoInput: 'Video Input', annotation: 'Annotation', prompt: 'Prompt', array: 'Array', @@ -1448,6 +1456,9 @@ export function WorkflowCanvas() { case "t": nodeType = "generateAudio"; break; + case "y": + nodeType = "videoInput"; + break; } if (nodeType) { @@ -1457,6 +1468,7 @@ export function WorkflowCanvas() { const defaultDimensions: Record = { imageInput: { width: 300, height: 280 }, audioInput: { width: 300, height: 200 }, + videoInput: { width: 300, height: 280 }, annotation: { width: 300, height: 280 }, prompt: { width: 320, height: 220 }, array: { width: 360, height: 360 }, @@ -2043,6 +2055,8 @@ export function WorkflowCanvas() { return "#3b82f6"; case "audioInput": return "#a78bfa"; + case "videoInput": + return "#c084fc"; // purple-400 (video input, distinct from generateVideo's #9333ea) case "annotation": return "#8b5cf6"; case "prompt": @@ -2213,10 +2227,50 @@ export function WorkflowCanvas() { {expandingNode && expandingNode.type === 'promptConstructor' && (() => { const node = getNodeById(expandingNode.id); if (!node) return null; + + // Compute available variables from connected text nodes (same logic as PromptConstructorNode) + const directTextNodes = edges + .filter((e) => e.target === expandingNode.id && e.targetHandle === "text") + .map((e) => nodes.find((n) => n.id === e.source)) + .filter((n): n is typeof nodes[0] => n !== undefined); + const connectedTextNodes = resolveTextSourcesThroughRouters(directTextNodes, nodes, edges); + + const vars: AvailableVariable[] = []; + const usedNames = new Set(); + + connectedTextNodes.forEach((cn) => { + if (cn.type === "prompt") { + const promptData = cn.data as PromptNodeData; + if (promptData.variableName) { + vars.push({ name: promptData.variableName, value: promptData.prompt || "", nodeId: cn.id }); + usedNames.add(promptData.variableName); + } + } + }); + + connectedTextNodes.forEach((cn) => { + let text: string | null = null; + if (cn.type === "prompt") text = (cn.data as PromptNodeData).prompt || null; + else if (cn.type === "llmGenerate") text = (cn.data as LLMGenerateNodeData).outputText || null; + else if (cn.type === "promptConstructor") { + const pcData = cn.data as PromptConstructorNodeData; + text = pcData.outputText ?? pcData.template ?? null; + } + if (text) { + parseVarTags(text).forEach(({ name, value }) => { + if (!usedNames.has(name)) { + vars.push({ name, value, nodeId: `${cn.id}-var-${name}` }); + usedNames.add(name); + } + }); + } + }); + return createPortal( - { updateNodeData(expandingNode.id, { template }); setExpandingNode(null); diff --git a/src/components/nodes/VideoInputNode.tsx b/src/components/nodes/VideoInputNode.tsx new file mode 100644 index 00000000..beae53f3 --- /dev/null +++ b/src/components/nodes/VideoInputNode.tsx @@ -0,0 +1,212 @@ +"use client"; + +import { useCallback, useRef } from "react"; +import { Handle, Position, NodeProps, Node } from "@xyflow/react"; +import { BaseNode } from "./BaseNode"; +import { useWorkflowStore } from "@/store/workflowStore"; +import { VideoInputNodeData } from "@/types"; +import { useVideoBlobUrl } from "@/hooks/useVideoBlobUrl"; + +type VideoInputNodeType = Node; + +const MAX_FILE_SIZE = 200 * 1024 * 1024; // 200MB +const ACCEPTED_FORMATS = "video/mp4,video/webm,video/quicktime"; + +function formatDuration(seconds: number): string { + const mins = Math.floor(seconds / 60); + const secs = Math.floor(seconds % 60); + return `${mins}:${secs.toString().padStart(2, "0")}`; +} + +export function VideoInputNode({ id, data, selected }: NodeProps) { + const nodeData = data; + const updateNodeData = useWorkflowStore((state) => state.updateNodeData); + const fileInputRef = useRef(null); + + // Use blob URL for efficient playback of large base64 videos + const playbackUrl = useVideoBlobUrl(nodeData.video ?? null); + + const handleFileChange = useCallback( + (e: React.ChangeEvent) => { + const file = e.target.files?.[0]; + if (!file) return; + + if (!file.type.match(/^video\//)) { + alert("Unsupported format. Use MP4, WebM, or QuickTime video files."); + return; + } + + if (file.size > MAX_FILE_SIZE) { + alert("Video file too large. Maximum size is 200MB."); + return; + } + + const reader = new FileReader(); + reader.onload = (event) => { + const base64 = event.target?.result as string; + + // Extract duration and dimensions using HTML Video element + const video = document.createElement("video"); + video.preload = "metadata"; + video.onloadedmetadata = () => { + updateNodeData(id, { + video: base64, + videoRef: undefined, + filename: file.name, + format: file.type, + duration: video.duration, + dimensions: { width: video.videoWidth, height: video.videoHeight }, + }); + URL.revokeObjectURL(video.src); + }; + video.onerror = () => { + // Still load the file even if metadata extraction fails + updateNodeData(id, { + video: base64, + videoRef: undefined, + filename: file.name, + format: file.type, + duration: null, + dimensions: null, + }); + URL.revokeObjectURL(video.src); + }; + // Use blob URL for metadata extraction to avoid base64 parsing overhead + const blob = new Blob([Uint8Array.from(atob(base64.split(",")[1]), c => c.charCodeAt(0))], { type: file.type }); + video.src = URL.createObjectURL(blob); + }; + reader.readAsDataURL(file); + }, + [id, updateNodeData] + ); + + const handleDrop = useCallback( + (e: React.DragEvent) => { + e.preventDefault(); + e.stopPropagation(); + + const file = e.dataTransfer.files?.[0]; + if (!file) return; + + const dt = new DataTransfer(); + dt.items.add(file); + if (fileInputRef.current) { + fileInputRef.current.files = dt.files; + fileInputRef.current.dispatchEvent(new Event("change", { bubbles: true })); + } + }, + [] + ); + + const handleDragOver = useCallback((e: React.DragEvent) => { + e.preventDefault(); + e.stopPropagation(); + }, []); + + const handleRemove = useCallback(() => { + updateNodeData(id, { + video: null, + videoRef: undefined, + filename: null, + duration: null, + dimensions: null, + format: null, + }); + }, [id, updateNodeData]); + + return ( + + + + {nodeData.video ? ( +
+ {nodeData.isOptional && ( + + Optional + + )} + {/* Filename and duration */} +
+ + {nodeData.filename} + +
+ {nodeData.duration != null && ( + + {formatDuration(nodeData.duration)} + + )} + {nodeData.dimensions && ( + + {nodeData.dimensions.width}x{nodeData.dimensions.height} + + )} +
+
+ + {/* Video player */} +
+
+ + {/* Remove button */} + +
+ ) : ( +
fileInputRef.current?.click()} + onKeyDown={(e) => { if (e.key === "Enter" || e.key === " ") { e.preventDefault(); fileInputRef.current?.click(); } }} + onDrop={handleDrop} + onDragOver={handleDragOver} + className={`w-full h-full bg-neutral-900/40 flex flex-col items-center justify-center cursor-pointer hover:bg-neutral-800/60 transition-colors ${nodeData.isOptional ? "border-2 border-dashed border-neutral-600" : ""}`} + > + + + + + {nodeData.isOptional ? "Optional" : "Drop video or click"} + +
+ )} + + + + + ); +} diff --git a/src/components/nodes/index.ts b/src/components/nodes/index.ts index 27f0d61c..6d3a6577 100644 --- a/src/components/nodes/index.ts +++ b/src/components/nodes/index.ts @@ -1,5 +1,6 @@ export { ImageInputNode } from "./ImageInputNode"; export { AudioInputNode } from "./AudioInputNode"; +export { VideoInputNode } from "./VideoInputNode"; export { AnnotationNode } from "./AnnotationNode"; export { PromptNode } from "./PromptNode"; export { ArrayNode } from "./ArrayNode"; diff --git a/src/lib/quickstart/validation.ts b/src/lib/quickstart/validation.ts index d1e0a40c..4867e134 100644 --- a/src/lib/quickstart/validation.ts +++ b/src/lib/quickstart/validation.ts @@ -14,6 +14,7 @@ interface ValidationResult { const VALID_NODE_TYPES: NodeType[] = [ "imageInput", "audioInput", + "videoInput", "annotation", "prompt", "array", @@ -43,6 +44,7 @@ const VALID_HANDLE_TYPES = ["image", "text", "audio", "video", "easeCurve", "3d" const DEFAULT_DIMENSIONS: Record = { imageInput: { width: 300, height: 280 }, audioInput: { width: 300, height: 200 }, + videoInput: { width: 300, height: 280 }, annotation: { width: 300, height: 280 }, prompt: { width: 320, height: 220 }, array: { width: 360, height: 360 }, @@ -232,6 +234,14 @@ function createDefaultNodeData(type: NodeType): WorkflowNodeData { duration: null, format: null, }; + case "videoInput": + return { + video: null, + filename: null, + duration: null, + dimensions: null, + format: null, + }; case "annotation": return { sourceImage: null, diff --git a/src/store/execution/executeNode.ts b/src/store/execution/executeNode.ts index c9b3f143..9657fb82 100644 --- a/src/store/execution/executeNode.ts +++ b/src/store/execution/executeNode.ts @@ -53,6 +53,14 @@ export async function executeNode( } break; } + case "videoInput": { + // If video is connected from upstream, use it (connection wins over upload) + const videoInputs = ctx.getConnectedInputs(ctx.node.id); + if (videoInputs.videos.length > 0 && videoInputs.videos[0]) { + ctx.updateNodeData(ctx.node.id, { video: videoInputs.videos[0] }); + } + break; + } case "annotation": await executeAnnotation(ctx); break; diff --git a/src/store/utils/connectedInputs.ts b/src/store/utils/connectedInputs.ts index cf9c7821..9554e3e7 100644 --- a/src/store/utils/connectedInputs.ts +++ b/src/store/utils/connectedInputs.ts @@ -10,6 +10,7 @@ import { WorkflowEdge, ImageInputNodeData, AudioInputNodeData, + VideoInputNodeData, AnnotationNodeData, NanoBananaNodeData, GenerateVideoNodeData, @@ -68,6 +69,8 @@ function getSourceOutput( ): { type: "image" | "text" | "video" | "audio" | "3d"; value: string | null } { if (sourceNode.type === "imageInput") { return { type: "image", value: (sourceNode.data as ImageInputNodeData).image }; + } else if (sourceNode.type === "videoInput") { + return { type: "video", value: (sourceNode.data as VideoInputNodeData).video }; } else if (sourceNode.type === "audioInput") { return { type: "audio", value: (sourceNode.data as AudioInputNodeData).audioFile }; } else if (sourceNode.type === "annotation") { diff --git a/src/store/utils/executionUtils.ts b/src/store/utils/executionUtils.ts index 37587530..aba4225f 100644 --- a/src/store/utils/executionUtils.ts +++ b/src/store/utils/executionUtils.ts @@ -134,6 +134,7 @@ export function clearNodeImageRefs(nodes: WorkflowNode[]): WorkflowNode[] { delete data.sourceImageRef; delete data.outputImageRef; delete data.inputImageRefs; + delete data.videoRef; return { ...node, data: data as WorkflowNodeData } as WorkflowNode; }); diff --git a/src/store/utils/nodeDefaults.ts b/src/store/utils/nodeDefaults.ts index fd450fc6..a0269ae1 100644 --- a/src/store/utils/nodeDefaults.ts +++ b/src/store/utils/nodeDefaults.ts @@ -3,6 +3,7 @@ import { ModelType, ImageInputNodeData, AudioInputNodeData, + VideoInputNodeData, AnnotationNodeData, PromptNodeData, ArrayNodeData, @@ -37,6 +38,7 @@ import { loadGenerateImageDefaults, loadNodeDefaults } from "./localStorage"; export const defaultNodeDimensions: Record = { imageInput: { width: 300, height: 280 }, audioInput: { width: 300, height: 200 }, + videoInput: { width: 300, height: 280 }, annotation: { width: 300, height: 280 }, prompt: { width: 320, height: 220 }, array: { width: 340, height: 260 }, @@ -97,6 +99,14 @@ export const createDefaultNodeData = (type: NodeType): WorkflowNodeData => { duration: null, format: null, } as AudioInputNodeData; + case "videoInput": + return { + video: null, + filename: null, + duration: null, + dimensions: null, + format: null, + } as VideoInputNodeData; case "annotation": return { sourceImage: null, diff --git a/src/store/workflowStore.ts b/src/store/workflowStore.ts index b32d836d..38279c66 100644 --- a/src/store/workflowStore.ts +++ b/src/store/workflowStore.ts @@ -1136,6 +1136,14 @@ const workflowStoreImpl: StateCreator = (set, get) => ({ } break; } + case "videoInput": { + // If video is connected from upstream, use it (connection wins over upload) + const videoInputs = get().getConnectedInputs(node.id); + if (videoInputs.videos.length > 0 && videoInputs.videos[0]) { + get().updateNodeData(node.id, { video: videoInputs.videos[0] }); + } + break; + } case "glbViewer": await executeGlbViewer(executionCtx); break; @@ -1466,6 +1474,7 @@ const workflowStoreImpl: StateCreator = (set, get) => ({ switch (node.type) { case "imageInput": case "audioInput": + case "videoInput": // Data source nodes - no execution needed break; case "glbViewer": @@ -2017,6 +2026,9 @@ const workflowStoreImpl: StateCreator = (set, get) => ({ mergedData.capturedImageRef = extData.capturedImageRef; } // Video refs + if (extData.videoRef && typeof extData.videoRef === 'string') { + mergedData.videoRef = extData.videoRef; + } if (extData.outputVideoRef && typeof extData.outputVideoRef === 'string') { mergedData.outputVideoRef = extData.outputVideoRef; } diff --git a/src/types/nodes.ts b/src/types/nodes.ts index 7a1c9c84..36997757 100644 --- a/src/types/nodes.ts +++ b/src/types/nodes.ts @@ -25,6 +25,7 @@ import type { LLMProvider, LLMModelType, SelectedModel, ProviderType } from "./p export type NodeType = | "imageInput" | "audioInput" + | "videoInput" | "annotation" | "prompt" | "array" @@ -75,6 +76,19 @@ export interface AudioInputNodeData extends BaseNodeData { isOptional?: boolean; } +/** + * Video input node - loads/uploads video files into the workflow + */ +export interface VideoInputNodeData extends BaseNodeData { + video: string | null; // Base64 data URL or blob URL + videoRef?: string; // External video reference for storage optimization + filename: string | null; + duration: number | null; // Duration in seconds + dimensions: { width: number; height: number } | null; + format: string | null; // MIME type (video/mp4, video/webm, etc.) + isOptional?: boolean; +} + /** * Prompt node - text input for AI generation */ @@ -456,6 +470,7 @@ export interface GLBViewerNodeData extends BaseNodeData { export type WorkflowNodeData = | ImageInputNodeData | AudioInputNodeData + | VideoInputNodeData | AnnotationNodeData | PromptNodeData | ArrayNodeData diff --git a/src/utils/mediaStorage.ts b/src/utils/mediaStorage.ts index 9b7a81fa..7c1da3c5 100644 --- a/src/utils/mediaStorage.ts +++ b/src/utils/mediaStorage.ts @@ -158,6 +158,20 @@ async function externalizeNodeMedia( break; } + case "videoInput": { + const d = data as import("@/types").VideoInputNodeData; + // Externalize base64 video data + if (d.videoRef && isDataUrl(d.video)) { + newData = { ...d, video: null }; + } else if (isDataUrl(d.video)) { + const videoRef = await saveVideoAndGetRef(d.video, workflowPath, savedMediaIds); + newData = { ...d, video: null, videoRef: videoRef || undefined }; + } else { + newData = d; + } + break; + } + case "annotation": { const d = data as import("@/types").AnnotationNodeData; let sourceImageRef = d.sourceImageRef; @@ -821,6 +835,20 @@ async function hydrateNodeMedia( break; } + case "videoInput": { + const d = data as import("@/types").VideoInputNodeData; + if (d.videoRef && !d.video) { + const video = await loadMediaById(d.videoRef, workflowPath, loadedMedia, "video"); + newData = { + ...d, + video, + }; + } else { + newData = d; + } + break; + } + case "annotation": { const d = data as import("@/types").AnnotationNodeData; let sourceImage = d.sourceImage;