From 0c29ad198b3801dccd2c2b85bd845e83aeff6001 Mon Sep 17 00:00:00 2001 From: shrimbly Date: Fri, 9 Jan 2026 23:21:57 +1300 Subject: [PATCH] feat(06-01): create GenerateVideoNode component Add GenerateVideoNode for video generation using Replicate and fal.ai: - Add GenerateVideoNodeData interface with video output field - Add generateVideo to NodeType union - Create GenerateVideoNode component with provider/model selection - Register in WorkflowCanvas with purple minimap color (#9333ea) - Add default dimensions and node data creation in store - Update validation.ts for new node type Co-Authored-By: Claude Opus 4.5 --- src/components/WorkflowCanvas.tsx | 7 + src/components/nodes/GenerateVideoNode.tsx | 269 +++++++++++++++++++++ src/components/nodes/index.ts | 1 + src/lib/quickstart/validation.ts | 11 + src/store/workflowStore.ts | 11 + src/types/index.ts | 17 ++ 6 files changed, 316 insertions(+) create mode 100644 src/components/nodes/GenerateVideoNode.tsx diff --git a/src/components/WorkflowCanvas.tsx b/src/components/WorkflowCanvas.tsx index 1ef2ba38..36f5c079 100644 --- a/src/components/WorkflowCanvas.tsx +++ b/src/components/WorkflowCanvas.tsx @@ -22,6 +22,7 @@ import { AnnotationNode, PromptNode, GenerateImageNode, + GenerateVideoNode, LLMGenerateNode, SplitGridNode, OutputNode, @@ -42,6 +43,7 @@ const nodeTypes: NodeTypes = { annotation: AnnotationNode, prompt: PromptNode, nanoBanana: GenerateImageNode, + generateVideo: GenerateVideoNode, llmGenerate: LLMGenerateNode, splitGrid: SplitGridNode, output: OutputNode, @@ -97,6 +99,8 @@ const getNodeHandles = (nodeType: string): { inputs: string[]; outputs: string[] return { inputs: [], outputs: ["text"] }; case "nanoBanana": return { inputs: ["image", "text"], outputs: ["image"] }; + case "generateVideo": + return { inputs: ["image", "text"], outputs: ["image"] }; case "llmGenerate": return { inputs: ["text", "image"], outputs: ["text"] }; case "splitGrid": @@ -725,6 +729,7 @@ export function WorkflowCanvas() { annotation: { width: 300, height: 280 }, prompt: { width: 320, height: 220 }, nanoBanana: { width: 300, height: 300 }, + generateVideo: { width: 300, height: 300 }, llmGenerate: { width: 320, height: 360 }, splitGrid: { width: 300, height: 320 }, output: { width: 320, height: 320 }, @@ -1144,6 +1149,8 @@ export function WorkflowCanvas() { return "#f97316"; case "nanoBanana": return "#22c55e"; + case "generateVideo": + return "#9333ea"; case "llmGenerate": return "#06b6d4"; case "splitGrid": diff --git a/src/components/nodes/GenerateVideoNode.tsx b/src/components/nodes/GenerateVideoNode.tsx new file mode 100644 index 00000000..7875b959 --- /dev/null +++ b/src/components/nodes/GenerateVideoNode.tsx @@ -0,0 +1,269 @@ +"use client"; + +import { useCallback, useState, useEffect, useMemo } from "react"; +import { Handle, Position, NodeProps, Node } from "@xyflow/react"; +import { BaseNode } from "./BaseNode"; +import { useWorkflowStore } from "@/store/workflowStore"; +import { GenerateVideoNodeData, ProviderType, SelectedModel } from "@/types"; +import { ProviderModel, ModelCapability } from "@/lib/providers/types"; + +// Video generation capabilities +const VIDEO_CAPABILITIES: ModelCapability[] = ["text-to-video", "image-to-video"]; + +type GenerateVideoNodeType = Node; + +export function GenerateVideoNode({ id, data, selected }: NodeProps) { + const nodeData = data; + const updateNodeData = useWorkflowStore((state) => state.updateNodeData); + const providerSettings = useWorkflowStore((state) => state.providerSettings); + const [externalModels, setExternalModels] = useState([]); + const [isLoadingModels, setIsLoadingModels] = useState(false); + + // Get the current selected provider (default to fal since Gemini doesn't do video) + const currentProvider: ProviderType = nodeData.selectedModel?.provider || "fal"; + + // Get enabled providers (exclude Gemini since it doesn't do video) + const enabledProviders = useMemo(() => { + const providers: { id: ProviderType; name: string }[] = []; + // fal.ai is always available (works without key but rate limited) + providers.push({ id: "fal", name: "fal.ai" }); + // Add Replicate if configured + if (providerSettings.providers.replicate?.enabled && providerSettings.providers.replicate?.apiKey) { + providers.push({ id: "replicate", name: "Replicate" }); + } + return providers; + }, [providerSettings]); + + // Fetch models from external providers when provider changes + useEffect(() => { + const fetchModels = async () => { + setIsLoadingModels(true); + try { + const capabilities = VIDEO_CAPABILITIES.join(","); + const headers: HeadersInit = {}; + if (providerSettings.providers.replicate?.apiKey) { + headers["X-Replicate-Key"] = providerSettings.providers.replicate.apiKey; + } + if (providerSettings.providers.fal?.apiKey) { + headers["X-Fal-Key"] = providerSettings.providers.fal.apiKey; + } + const response = await fetch(`/api/models?provider=${currentProvider}&capabilities=${capabilities}`, { headers }); + if (response.ok) { + const data = await response.json(); + setExternalModels(data.models || []); + } else { + setExternalModels([]); + } + } catch (error) { + console.error("Failed to fetch video models:", error); + setExternalModels([]); + } finally { + setIsLoadingModels(false); + } + }; + + fetchModels(); + }, [currentProvider, providerSettings]); + + // Handle provider change + const handleProviderChange = useCallback( + (e: React.ChangeEvent) => { + const provider = e.target.value as ProviderType; + // Set placeholder for the provider + const newSelectedModel: SelectedModel = { + provider, + modelId: "", + displayName: "Select model...", + }; + updateNodeData(id, { selectedModel: newSelectedModel }); + }, + [id, updateNodeData] + ); + + // Handle model change + const handleModelChange = useCallback( + (e: React.ChangeEvent) => { + const modelId = e.target.value; + const model = externalModels.find(m => m.id === modelId); + if (model) { + const newSelectedModel: SelectedModel = { + provider: currentProvider, + modelId: model.id, + displayName: model.name, + }; + updateNodeData(id, { selectedModel: newSelectedModel }); + } + }, + [id, currentProvider, externalModels, updateNodeData] + ); + + const handleClearVideo = useCallback(() => { + updateNodeData(id, { outputVideo: null, status: "idle", error: null }); + }, [id, updateNodeData]); + + const regenerateNode = useWorkflowStore((state) => state.regenerateNode); + const isRunning = useWorkflowStore((state) => state.isRunning); + + const handleRegenerate = useCallback(() => { + regenerateNode(id); + }, [id, regenerateNode]); + + return ( + updateNodeData(id, { customTitle: title || undefined })} + onCommentChange={(comment) => updateNodeData(id, { comment: comment || undefined })} + onRun={handleRegenerate} + selected={selected} + isExecuting={isRunning} + hasError={nodeData.status === "error"} + > + {/* Image input - accepts multiple connections */} + + {/* Text input - single connection */} + + {/* Video output - we use "image" handle type for now since the connection system expects image/text */} + + +
+ {/* Preview area */} + {nodeData.outputVideo ? ( +
+
+ ) : ( +
+ {nodeData.status === "loading" ? ( + + + + + ) : nodeData.status === "error" ? ( + + {nodeData.error || "Failed"} + + ) : ( + + Run to generate + + )} +
+ )} + + {/* Provider selector - only show if multiple providers are enabled */} + {enabledProviders.length > 1 && ( + + )} + + {/* Model selector */} + +
+ + ); +} diff --git a/src/components/nodes/index.ts b/src/components/nodes/index.ts index f9c1dad2..acb08ccf 100644 --- a/src/components/nodes/index.ts +++ b/src/components/nodes/index.ts @@ -2,6 +2,7 @@ export { ImageInputNode } from "./ImageInputNode"; export { AnnotationNode } from "./AnnotationNode"; export { PromptNode } from "./PromptNode"; export { GenerateImageNode, NanoBananaNode } from "./GenerateImageNode"; +export { GenerateVideoNode } from "./GenerateVideoNode"; export { LLMGenerateNode } from "./LLMGenerateNode"; export { SplitGridNode } from "./SplitGridNode"; export { OutputNode } from "./OutputNode"; diff --git a/src/lib/quickstart/validation.ts b/src/lib/quickstart/validation.ts index dd4f0701..b3459bea 100644 --- a/src/lib/quickstart/validation.ts +++ b/src/lib/quickstart/validation.ts @@ -16,6 +16,7 @@ const VALID_NODE_TYPES: NodeType[] = [ "annotation", "prompt", "nanoBanana", + "generateVideo", "llmGenerate", "splitGrid", "output", @@ -29,6 +30,7 @@ const DEFAULT_DIMENSIONS: Record = annotation: { width: 300, height: 280 }, prompt: { width: 320, height: 220 }, nanoBanana: { width: 300, height: 300 }, + generateVideo: { width: 300, height: 300 }, llmGenerate: { width: 320, height: 360 }, splitGrid: { width: 300, height: 320 }, output: { width: 320, height: 320 }, @@ -217,6 +219,15 @@ function createDefaultNodeData(type: NodeType): WorkflowNodeData { imageHistory: [], selectedHistoryIndex: 0, }; + case "generateVideo": + return { + inputImages: [], + inputPrompt: null, + outputVideo: null, + selectedModel: undefined, + status: "idle", + error: null, + }; case "llmGenerate": return { inputPrompt: null, diff --git a/src/store/workflowStore.ts b/src/store/workflowStore.ts index e65f3459..cc16661a 100644 --- a/src/store/workflowStore.ts +++ b/src/store/workflowStore.ts @@ -16,6 +16,7 @@ import { AnnotationNodeData, PromptNodeData, NanoBananaNodeData, + GenerateVideoNodeData, LLMGenerateNodeData, SplitGridNodeData, OutputNodeData, @@ -210,6 +211,15 @@ const createDefaultNodeData = (type: NodeType): WorkflowNodeData => { selectedHistoryIndex: 0, } as NanoBananaNodeData; } + case "generateVideo": + return { + inputImages: [], + inputPrompt: null, + outputVideo: null, + selectedModel: undefined, + status: "idle", + error: null, + } as GenerateVideoNodeData; case "llmGenerate": return { inputPrompt: null, @@ -448,6 +458,7 @@ export const useWorkflowStore = create((set, get) => ({ annotation: { width: 300, height: 280 }, prompt: { width: 320, height: 220 }, nanoBanana: { width: 300, height: 300 }, + generateVideo: { width: 300, height: 300 }, llmGenerate: { width: 320, height: 360 }, splitGrid: { width: 300, height: 320 }, output: { width: 320, height: 320 }, diff --git a/src/types/index.ts b/src/types/index.ts index 846e6149..cd730ab8 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -6,6 +6,7 @@ export type NodeType = | "annotation" | "prompt" | "nanoBanana" + | "generateVideo" | "llmGenerate" | "splitGrid" | "output"; @@ -150,6 +151,18 @@ export interface NanoBananaNodeData extends BaseNodeData { selectedHistoryIndex: number; // Currently selected image in carousel } +// Generate Video Node Data (Video Generation) +export interface GenerateVideoNodeData extends BaseNodeData { + inputImages: string[]; + inputImageRefs?: string[]; // External image references for storage optimization + inputPrompt: string | null; + outputVideo: string | null; // Video data URL or URL + outputVideoRef?: string; // External video reference for storage optimization + selectedModel?: SelectedModel; // Required for video generation (no legacy fallback) + status: NodeStatus; + error: string | null; +} + // LLM Generate Node Data (Text Generation) export interface LLMGenerateNodeData extends BaseNodeData { inputPrompt: string | null; @@ -200,6 +213,7 @@ export type WorkflowNodeData = | AnnotationNodeData | PromptNodeData | NanoBananaNodeData + | GenerateVideoNodeData | LLMGenerateNodeData | SplitGridNodeData | OutputNodeData; @@ -233,6 +247,9 @@ export interface GenerateRequest { export interface GenerateResponse { success: boolean; image?: string; + video?: string; + videoUrl?: string; // For large videos, return URL directly + contentType?: "image" | "video"; error?: string; }