Browse Source
Add GenerateVideoNode for video generation using Replicate and fal.ai: - Add GenerateVideoNodeData interface with video output field - Add generateVideo to NodeType union - Create GenerateVideoNode component with provider/model selection - Register in WorkflowCanvas with purple minimap color (#9333ea) - Add default dimensions and node data creation in store - Update validation.ts for new node type Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>handoff-20260429-1057
shrimbly
6 months ago
6 changed files with 316 additions and 0 deletions
@ -0,0 +1,269 @@ |
|||
"use client"; |
|||
|
|||
import { useCallback, useState, useEffect, useMemo } from "react"; |
|||
import { Handle, Position, NodeProps, Node } from "@xyflow/react"; |
|||
import { BaseNode } from "./BaseNode"; |
|||
import { useWorkflowStore } from "@/store/workflowStore"; |
|||
import { GenerateVideoNodeData, ProviderType, SelectedModel } from "@/types"; |
|||
import { ProviderModel, ModelCapability } from "@/lib/providers/types"; |
|||
|
|||
// Video generation capabilities
|
|||
const VIDEO_CAPABILITIES: ModelCapability[] = ["text-to-video", "image-to-video"]; |
|||
|
|||
type GenerateVideoNodeType = Node<GenerateVideoNodeData, "generateVideo">; |
|||
|
|||
export function GenerateVideoNode({ id, data, selected }: NodeProps<GenerateVideoNodeType>) { |
|||
const nodeData = data; |
|||
const updateNodeData = useWorkflowStore((state) => state.updateNodeData); |
|||
const providerSettings = useWorkflowStore((state) => state.providerSettings); |
|||
const [externalModels, setExternalModels] = useState<ProviderModel[]>([]); |
|||
const [isLoadingModels, setIsLoadingModels] = useState(false); |
|||
|
|||
// Get the current selected provider (default to fal since Gemini doesn't do video)
|
|||
const currentProvider: ProviderType = nodeData.selectedModel?.provider || "fal"; |
|||
|
|||
// Get enabled providers (exclude Gemini since it doesn't do video)
|
|||
const enabledProviders = useMemo(() => { |
|||
const providers: { id: ProviderType; name: string }[] = []; |
|||
// fal.ai is always available (works without key but rate limited)
|
|||
providers.push({ id: "fal", name: "fal.ai" }); |
|||
// Add Replicate if configured
|
|||
if (providerSettings.providers.replicate?.enabled && providerSettings.providers.replicate?.apiKey) { |
|||
providers.push({ id: "replicate", name: "Replicate" }); |
|||
} |
|||
return providers; |
|||
}, [providerSettings]); |
|||
|
|||
// Fetch models from external providers when provider changes
|
|||
useEffect(() => { |
|||
const fetchModels = async () => { |
|||
setIsLoadingModels(true); |
|||
try { |
|||
const capabilities = VIDEO_CAPABILITIES.join(","); |
|||
const headers: HeadersInit = {}; |
|||
if (providerSettings.providers.replicate?.apiKey) { |
|||
headers["X-Replicate-Key"] = providerSettings.providers.replicate.apiKey; |
|||
} |
|||
if (providerSettings.providers.fal?.apiKey) { |
|||
headers["X-Fal-Key"] = providerSettings.providers.fal.apiKey; |
|||
} |
|||
const response = await fetch(`/api/models?provider=${currentProvider}&capabilities=${capabilities}`, { headers }); |
|||
if (response.ok) { |
|||
const data = await response.json(); |
|||
setExternalModels(data.models || []); |
|||
} else { |
|||
setExternalModels([]); |
|||
} |
|||
} catch (error) { |
|||
console.error("Failed to fetch video models:", error); |
|||
setExternalModels([]); |
|||
} finally { |
|||
setIsLoadingModels(false); |
|||
} |
|||
}; |
|||
|
|||
fetchModels(); |
|||
}, [currentProvider, providerSettings]); |
|||
|
|||
// Handle provider change
|
|||
const handleProviderChange = useCallback( |
|||
(e: React.ChangeEvent<HTMLSelectElement>) => { |
|||
const provider = e.target.value as ProviderType; |
|||
// Set placeholder for the provider
|
|||
const newSelectedModel: SelectedModel = { |
|||
provider, |
|||
modelId: "", |
|||
displayName: "Select model...", |
|||
}; |
|||
updateNodeData(id, { selectedModel: newSelectedModel }); |
|||
}, |
|||
[id, updateNodeData] |
|||
); |
|||
|
|||
// Handle model change
|
|||
const handleModelChange = useCallback( |
|||
(e: React.ChangeEvent<HTMLSelectElement>) => { |
|||
const modelId = e.target.value; |
|||
const model = externalModels.find(m => m.id === modelId); |
|||
if (model) { |
|||
const newSelectedModel: SelectedModel = { |
|||
provider: currentProvider, |
|||
modelId: model.id, |
|||
displayName: model.name, |
|||
}; |
|||
updateNodeData(id, { selectedModel: newSelectedModel }); |
|||
} |
|||
}, |
|||
[id, currentProvider, externalModels, updateNodeData] |
|||
); |
|||
|
|||
const handleClearVideo = useCallback(() => { |
|||
updateNodeData(id, { outputVideo: null, status: "idle", error: null }); |
|||
}, [id, updateNodeData]); |
|||
|
|||
const regenerateNode = useWorkflowStore((state) => state.regenerateNode); |
|||
const isRunning = useWorkflowStore((state) => state.isRunning); |
|||
|
|||
const handleRegenerate = useCallback(() => { |
|||
regenerateNode(id); |
|||
}, [id, regenerateNode]); |
|||
|
|||
return ( |
|||
<BaseNode |
|||
id={id} |
|||
title="Generate Video" |
|||
customTitle={nodeData.customTitle} |
|||
comment={nodeData.comment} |
|||
onCustomTitleChange={(title) => updateNodeData(id, { customTitle: title || undefined })} |
|||
onCommentChange={(comment) => updateNodeData(id, { comment: comment || undefined })} |
|||
onRun={handleRegenerate} |
|||
selected={selected} |
|||
isExecuting={isRunning} |
|||
hasError={nodeData.status === "error"} |
|||
> |
|||
{/* Image input - accepts multiple connections */} |
|||
<Handle |
|||
type="target" |
|||
position={Position.Left} |
|||
id="image" |
|||
style={{ top: "35%" }} |
|||
data-handletype="image" |
|||
isConnectable={true} |
|||
/> |
|||
{/* Text input - single connection */} |
|||
<Handle |
|||
type="target" |
|||
position={Position.Left} |
|||
id="text" |
|||
style={{ top: "65%" }} |
|||
data-handletype="text" |
|||
/> |
|||
{/* Video output - we use "image" handle type for now since the connection system expects image/text */} |
|||
<Handle |
|||
type="source" |
|||
position={Position.Right} |
|||
id="image" |
|||
data-handletype="image" |
|||
/> |
|||
|
|||
<div className="flex-1 flex flex-col min-h-0 gap-2"> |
|||
{/* Preview area */} |
|||
{nodeData.outputVideo ? ( |
|||
<div className="relative w-full flex-1 min-h-0"> |
|||
<video |
|||
src={nodeData.outputVideo} |
|||
controls |
|||
className="w-full h-full object-contain rounded" |
|||
playsInline |
|||
/> |
|||
{/* Loading overlay for generation */} |
|||
{nodeData.status === "loading" && ( |
|||
<div className="absolute inset-0 bg-neutral-900/70 rounded flex items-center justify-center"> |
|||
<svg |
|||
className="w-6 h-6 animate-spin text-white" |
|||
fill="none" |
|||
viewBox="0 0 24 24" |
|||
> |
|||
<circle |
|||
className="opacity-25" |
|||
cx="12" |
|||
cy="12" |
|||
r="10" |
|||
stroke="currentColor" |
|||
strokeWidth="3" |
|||
/> |
|||
<path |
|||
className="opacity-75" |
|||
fill="currentColor" |
|||
d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4zm2 5.291A7.962 7.962 0 014 12H0c0 3.042 1.135 5.824 3 7.938l3-2.647z" |
|||
/> |
|||
</svg> |
|||
</div> |
|||
)} |
|||
<div className="absolute top-1 right-1"> |
|||
<button |
|||
onClick={handleClearVideo} |
|||
className="w-5 h-5 bg-neutral-900/80 hover:bg-red-600/80 rounded flex items-center justify-center text-neutral-400 hover:text-white transition-colors" |
|||
title="Clear video" |
|||
> |
|||
<svg className="w-3 h-3" fill="none" viewBox="0 0 24 24" stroke="currentColor" strokeWidth={2}> |
|||
<path strokeLinecap="round" strokeLinejoin="round" d="M6 18L18 6M6 6l12 12" /> |
|||
</svg> |
|||
</button> |
|||
</div> |
|||
</div> |
|||
) : ( |
|||
<div className="w-full flex-1 min-h-[112px] border border-dashed border-neutral-600 rounded flex flex-col items-center justify-center"> |
|||
{nodeData.status === "loading" ? ( |
|||
<svg |
|||
className="w-4 h-4 animate-spin text-neutral-400" |
|||
fill="none" |
|||
viewBox="0 0 24 24" |
|||
> |
|||
<circle |
|||
className="opacity-25" |
|||
cx="12" |
|||
cy="12" |
|||
r="10" |
|||
stroke="currentColor" |
|||
strokeWidth="3" |
|||
/> |
|||
<path |
|||
className="opacity-75" |
|||
fill="currentColor" |
|||
d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4zm2 5.291A7.962 7.962 0 014 12H0c0 3.042 1.135 5.824 3 7.938l3-2.647z" |
|||
/> |
|||
</svg> |
|||
) : nodeData.status === "error" ? ( |
|||
<span className="text-[10px] text-red-400 text-center px-2"> |
|||
{nodeData.error || "Failed"} |
|||
</span> |
|||
) : ( |
|||
<span className="text-neutral-500 text-[10px]"> |
|||
Run to generate |
|||
</span> |
|||
)} |
|||
</div> |
|||
)} |
|||
|
|||
{/* Provider selector - only show if multiple providers are enabled */} |
|||
{enabledProviders.length > 1 && ( |
|||
<select |
|||
value={currentProvider} |
|||
onChange={handleProviderChange} |
|||
className="w-full text-[10px] py-1 px-1.5 border border-neutral-700 rounded bg-neutral-900/50 focus:outline-none focus:ring-1 focus:ring-neutral-600 text-neutral-300 shrink-0" |
|||
> |
|||
{enabledProviders.map((p) => ( |
|||
<option key={p.id} value={p.id}> |
|||
{p.name} |
|||
</option> |
|||
))} |
|||
</select> |
|||
)} |
|||
|
|||
{/* Model selector */} |
|||
<select |
|||
value={nodeData.selectedModel?.modelId || ""} |
|||
onChange={handleModelChange} |
|||
disabled={isLoadingModels} |
|||
className="w-full text-[10px] py-1 px-1.5 border border-neutral-700 rounded bg-neutral-900/50 focus:outline-none focus:ring-1 focus:ring-neutral-600 text-neutral-300 shrink-0 disabled:opacity-50" |
|||
> |
|||
{isLoadingModels ? ( |
|||
<option value="">Loading models...</option> |
|||
) : externalModels.length === 0 ? ( |
|||
<option value="">No video models available</option> |
|||
) : ( |
|||
<> |
|||
<option value="">Select model...</option> |
|||
{externalModels.map((m) => ( |
|||
<option key={m.id} value={m.id}> |
|||
{m.name} |
|||
</option> |
|||
))} |
|||
</> |
|||
)} |
|||
</select> |
|||
</div> |
|||
</BaseNode> |
|||
); |
|||
} |
|||
Loading…
Reference in new issue