From 3ebaae82de7c8e76210be303fef0d196b5a32e5c Mon Sep 17 00:00:00 2001 From: shrimbly Date: Sat, 14 Feb 2026 20:02:47 +1300 Subject: [PATCH] =?UTF-8?q?feat:=20improve=20GLB=20viewer=20=E2=80=94=20fi?= =?UTF-8?q?x=20viewport=20sizing,=20add=20spot=20light,=20handle=20labels,?= =?UTF-8?q?=20scroll=20isolation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Fix 3D viewport not filling node by adding resize={{ offsetSize: true }} for correct measurement in CSS-transformed React Flow nodes - Replace grid/axis indicators with spot light for cleaner scene - Add labeled input (3D) and output (Image) handles matching generate node style - Add native wheel event listener to prevent graph zoom/pan while scrolling to zoom 3D model - Remove redundant CSS workarounds (absolute positioning style, extra relative wrapper) - Add edge-to-edge viewport via contentClassName (removes default padding when GLB loaded) - Overlay controls bar on viewport with gradient background Co-Authored-By: Claude Opus 4.6 --- src/app/api/generate/providers/fal.ts | 28 +++- src/app/api/generate/providers/replicate.ts | 16 ++ src/app/api/generate/providers/wavespeed.ts | 16 ++ src/app/api/generate/route.ts | 40 ++++- src/app/api/models/route.ts | 52 ++++++- src/app/globals.css | 6 + src/components/ConnectionDropMenu.tsx | 30 +++- src/components/WorkflowCanvas.tsx | 21 ++- src/components/nodes/BaseNode.tsx | 8 +- src/components/nodes/GLBViewerNode.tsx | 160 ++++++++++++-------- src/components/nodes/GenerateImageNode.tsx | 30 ++-- src/lib/providers/types.ts | 8 +- src/store/execution/executeNode.ts | 86 +++++++++++ src/store/execution/index.ts | 1 + src/store/execution/nanoBananaExecutor.ts | 21 +++ src/store/execution/simpleNodeExecutors.ts | 28 ++++ src/store/utils/connectedInputs.ts | 16 +- src/store/utils/nodeDefaults.ts | 1 + src/store/workflowStore.ts | 31 +++- src/types/api.ts | 5 +- src/types/nodes.ts | 3 +- src/types/providers.ts | 1 + 22 files changed, 504 insertions(+), 104 deletions(-) create mode 100644 src/store/execution/executeNode.ts diff --git a/src/app/api/generate/providers/fal.ts b/src/app/api/generate/providers/fal.ts index aff62f91..9d4f19b7 100644 --- a/src/app/api/generate/providers/fal.ts +++ b/src/app/api/generate/providers/fal.ts @@ -456,10 +456,17 @@ export async function generateWithFalQueue( const result = await resultResponse.json(); - // Extract video URL from result + // Extract media URL from result let mediaUrl: string | null = null; - if (result.video && result.video.url) { + // Check for 3D model output (GLB mesh) — must check before images + if (result.model_mesh?.url) { + mediaUrl = result.model_mesh.url; + } else if (result.mesh?.url) { + mediaUrl = result.mesh.url; + } else if (result.glb?.url) { + mediaUrl = result.glb.url; + } else if (result.video && result.video.url) { mediaUrl = result.video.url; } else if (result.images && Array.isArray(result.images) && result.images.length > 0) { mediaUrl = result.images[0].url; @@ -494,7 +501,24 @@ export async function generateWithFalQueue( }; } + const is3DModel = input.model.capabilities.some(c => c.includes("3d")); const isVideoModel = input.model.capabilities.some(c => c.includes("video")); + + // For 3D models, return URL directly (GLB files are binary — don't base64 encode) + if (is3DModel) { + console.log(`[API:${requestId}] SUCCESS - Returning 3D model URL`); + return { + success: true, + outputs: [ + { + type: "3d", + data: "", + url: mediaUrl, + }, + ], + }; + } + const contentType = mediaResponse.headers.get("content-type") || (isVideoModel ? "video/mp4" : "image/png"); const isVideo = contentType.startsWith("video/"); diff --git a/src/app/api/generate/providers/replicate.ts b/src/app/api/generate/providers/replicate.ts index 78113c26..c35edcc3 100644 --- a/src/app/api/generate/providers/replicate.ts +++ b/src/app/api/generate/providers/replicate.ts @@ -259,6 +259,22 @@ export async function generateWithReplicate( }; } + // Check if this is a 3D model — return URL directly (GLB files are binary) + const is3DModel = input.model.capabilities.some(c => c.includes("3d")); + if (is3DModel) { + console.log(`[API:${requestId}] SUCCESS - Returning 3D model URL`); + return { + success: true, + outputs: [ + { + type: "3d", + data: "", + url: mediaUrl, + }, + ], + }; + } + // Determine MIME type from response const contentType = mediaResponse.headers.get("content-type") || "image/png"; const isVideo = contentType.startsWith("video/"); diff --git a/src/app/api/generate/providers/wavespeed.ts b/src/app/api/generate/providers/wavespeed.ts index 772d73b2..1e38d554 100644 --- a/src/app/api/generate/providers/wavespeed.ts +++ b/src/app/api/generate/providers/wavespeed.ts @@ -88,6 +88,7 @@ export async function generateWithWaveSpeed( console.log(`[API:${requestId}] Dynamic inputs: ${hasDynamicInputs ? Object.keys(input.dynamicInputs!).join(", ") : "none"}`); // Determine output type from model capabilities + const is3DModel = input.model.capabilities.some(c => c.includes("3d")); const isVideoModel = input.model.capabilities.includes("text-to-video") || input.model.capabilities.includes("image-to-video"); @@ -344,6 +345,21 @@ export async function generateWithWaveSpeed( } const outputSizeMB = outputArrayBuffer.byteLength / (1024 * 1024); + // For 3D models, return URL directly (GLB files are binary — don't base64 encode) + if (is3DModel) { + console.log(`[API:${requestId}] SUCCESS - Returning 3D model URL`); + return { + success: true, + outputs: [ + { + type: "3d", + data: "", + url: outputUrl, + }, + ], + }; + } + const rawContentType = outputResponse.headers.get("content-type"); const contentType = (rawContentType && (rawContentType.startsWith("video/") || rawContentType.startsWith("image/"))) diff --git a/src/app/api/generate/route.ts b/src/app/api/generate/route.ts index 4ec3d5dc..9b1a17f7 100644 --- a/src/app/api/generate/route.ts +++ b/src/app/api/generate/route.ts @@ -125,7 +125,7 @@ export async function POST(request: NextRequest) { id: selectedModel!.modelId, name: selectedModel!.displayName, provider: "replicate", - capabilities: mediaType === "video" ? ["text-to-video"] : ["text-to-image"], + capabilities: mediaType === "video" ? ["text-to-video"] : mediaType === "3d" ? ["text-to-3d"] : ["text-to-image"], description: null, }, prompt: prompt || "", @@ -159,6 +159,14 @@ export async function POST(request: NextRequest) { } // Return appropriate fields based on output type + if (output.type === "3d") { + return NextResponse.json({ + success: true, + model3dUrl: output.url, + contentType: "3d", + }); + } + if (output.type === "video") { // Large videos have data="" with url set; normal videos have base64 data const isLargeVideo = !output.data && output.url; @@ -212,7 +220,7 @@ export async function POST(request: NextRequest) { id: selectedModel!.modelId, name: selectedModel!.displayName, provider: "fal", - capabilities: mediaType === "video" ? ["text-to-video"] : ["text-to-image"], + capabilities: mediaType === "video" ? ["text-to-video"] : mediaType === "3d" ? ["text-to-3d"] : ["text-to-image"], description: null, }, prompt: prompt || "", @@ -246,6 +254,14 @@ export async function POST(request: NextRequest) { } // Return appropriate fields based on output type + if (output.type === "3d") { + return NextResponse.json({ + success: true, + model3dUrl: output.url, + contentType: "3d", + }); + } + if (output.type === "video") { // Large videos have data="" with url set; normal videos have base64 data const isLargeVideo = !output.data && output.url; @@ -303,7 +319,7 @@ export async function POST(request: NextRequest) { id: selectedModel!.modelId, name: selectedModel!.displayName, provider: "kie", - capabilities: mediaType === "video" ? ["text-to-video"] : ["text-to-image"], + capabilities: mediaType === "video" ? ["text-to-video"] : mediaType === "3d" ? ["text-to-3d"] : ["text-to-image"], description: null, }, prompt: prompt || "", @@ -337,6 +353,14 @@ export async function POST(request: NextRequest) { } // Return appropriate fields based on output type + if (output.type === "3d") { + return NextResponse.json({ + success: true, + model3dUrl: output.url, + contentType: "3d", + }); + } + if (output.type === "video") { // Large videos have data="" with url set; normal videos have base64 data const isLargeVideo = !output.data && output.url; @@ -394,7 +418,7 @@ export async function POST(request: NextRequest) { id: selectedModel!.modelId, name: selectedModel!.displayName, provider: "wavespeed", - capabilities: mediaType === "video" ? ["text-to-video"] : ["text-to-image"], + capabilities: mediaType === "video" ? ["text-to-video"] : mediaType === "3d" ? ["text-to-3d"] : ["text-to-image"], description: null, }, prompt: prompt || "", @@ -428,6 +452,14 @@ export async function POST(request: NextRequest) { } // Return appropriate fields based on output type + if (output.type === "3d") { + return NextResponse.json({ + success: true, + model3dUrl: output.url, + contentType: "3d", + }); + } + if (output.type === "video") { // Large videos have data="" with url set; normal videos have base64 data const isLargeVideo = !output.data && output.url; diff --git a/src/app/api/models/route.ts b/src/app/api/models/route.ts index b95a87e5..7b1a8b91 100644 --- a/src/app/api/models/route.ts +++ b/src/app/api/models/route.ts @@ -43,12 +43,14 @@ const REPLICATE_API_BASE = "https://api.replicate.com/v1"; const FAL_API_BASE = "https://api.fal.ai/v1"; const WAVESPEED_API_BASE = "https://api.wavespeed.ai/api/v3"; -// Categories we care about for image/video generation (fal.ai) +// Categories we care about for image/video/3D generation (fal.ai) const RELEVANT_CATEGORIES = [ "text-to-image", "image-to-image", "text-to-video", "image-to-video", + "text-to-3d", + "image-to-3d", ]; // Kie.ai models (hardcoded - no discovery API available) @@ -420,7 +422,32 @@ function inferReplicateCapabilities(model: ReplicateModel): ModelCapability[] { const capabilities: ModelCapability[] = []; const searchText = `${model.name} ${model.description ?? ""}`.toLowerCase(); - // Check for video-related keywords first + // Check for 3D-related keywords first + const is3DModel = + searchText.includes("3d") || + searchText.includes("mesh") || + searchText.includes("triposr") || + searchText.includes("tripo") || + searchText.includes("hunyuan3d") || + searchText.includes("instant-mesh") || + searchText.includes("point-e") || + searchText.includes("shap-e"); + + if (is3DModel) { + // 3D model - determine if image-to-3d or text-to-3d + const hasImageInput = + searchText.includes("image") || + searchText.includes("img") || + searchText.includes("photo"); + if (hasImageInput) { + capabilities.push("image-to-3d"); + } else { + capabilities.push("text-to-3d"); + } + return capabilities; + } + + // Check for video-related keywords const isVideoModel = searchText.includes("video") || searchText.includes("animate") || @@ -559,6 +586,27 @@ function inferWaveSpeedCapabilities(model: WaveSpeedModel): ModelCapability[] { const category = (model.category || model.type || "").toLowerCase(); const searchText = `${modelId} ${name} ${description} ${category}`; + // Check for 3D-related keywords first + const is3DModel = + searchText.includes("3d") || + searchText.includes("mesh") || + searchText.includes("tripo") || + searchText.includes("hunyuan3d") || + category.includes("3d"); + + if (is3DModel) { + const hasImageInput = + searchText.includes("image") || + searchText.includes("img") || + searchText.includes("photo"); + if (hasImageInput) { + capabilities.push("image-to-3d"); + } else { + capabilities.push("text-to-3d"); + } + return capabilities; + } + // Check for video-related keywords const isVideoModel = searchText.includes("video") || diff --git a/src/app/globals.css b/src/app/globals.css index 5c2b9936..ad52a5df 100644 --- a/src/app/globals.css +++ b/src/app/globals.css @@ -10,6 +10,7 @@ /* Handle colors (used by labels) */ --handle-color-image: #10b981; --handle-color-text: #3b82f6; + --handle-color-3d: #f97316; } html { @@ -87,6 +88,11 @@ body { background: #3b82f6; } +/* 3D handles - orange */ +.react-flow__handle[data-handletype="3d"] { + background: #f97316; +} + .react-flow__edge-path { stroke: #94a3b8; stroke-width: 3; diff --git a/src/components/ConnectionDropMenu.tsx b/src/components/ConnectionDropMenu.tsx index d439616e..3161bdef 100644 --- a/src/components/ConnectionDropMenu.tsx +++ b/src/components/ConnectionDropMenu.tsx @@ -305,9 +305,35 @@ const AUDIO_SOURCE_OPTIONS: MenuOption[] = [ }, ]; +// 3D target options (nodes that accept 3D input) +const THREE_D_TARGET_OPTIONS: MenuOption[] = [ + { + type: "glbViewer", + label: "3D Viewer", + icon: ( + + + + ), + }, +]; + +// 3D source options (nodes that produce 3D output) +const THREE_D_SOURCE_OPTIONS: MenuOption[] = [ + { + type: "nanoBanana", + label: "Generate Image", + icon: ( + + + + ), + }, +]; + interface ConnectionDropMenuProps { position: { x: number; y: number }; - handleType: "image" | "text" | "video" | "audio" | "easeCurve" | null; + handleType: "image" | "text" | "video" | "audio" | "3d" | "easeCurve" | null; connectionType: "source" | "target"; // source = dragging from output, target = dragging from input onSelect: (selection: { type: NodeType | MenuAction; isAction: boolean }) => void; onClose: () => void; @@ -331,11 +357,13 @@ export function ConnectionDropMenu({ // Dragging from a source handle (output), need nodes with target handles (inputs) if (handleType === "video") return VIDEO_TARGET_OPTIONS; if (handleType === "audio") return AUDIO_TARGET_OPTIONS; + if (handleType === "3d") return THREE_D_TARGET_OPTIONS; return handleType === "image" ? IMAGE_TARGET_OPTIONS : TEXT_TARGET_OPTIONS; } else { // Dragging from a target handle (input), need nodes with source handles (outputs) if (handleType === "video") return VIDEO_SOURCE_OPTIONS; if (handleType === "audio") return AUDIO_SOURCE_OPTIONS; + if (handleType === "3d") return THREE_D_SOURCE_OPTIONS; return handleType === "image" ? IMAGE_SOURCE_OPTIONS : TEXT_SOURCE_OPTIONS; } }, [handleType, connectionType]); diff --git a/src/components/WorkflowCanvas.tsx b/src/components/WorkflowCanvas.tsx index 9b8c4551..773cd8f7 100644 --- a/src/components/WorkflowCanvas.tsx +++ b/src/components/WorkflowCanvas.tsx @@ -84,10 +84,12 @@ const edgeTypes: EdgeTypes = { // - Video handles can only connect to generateVideo or output nodes // Helper to determine handle type from handle ID // For dynamic handles, we use naming convention: image inputs contain "image", text inputs are "prompt" or "negative_prompt" -const getHandleType = (handleId: string | null | undefined): "image" | "text" | "video" | "audio" | "easeCurve" | null => { +const getHandleType = (handleId: string | null | undefined): "image" | "text" | "video" | "audio" | "3d" | "easeCurve" | null => { if (!handleId) return null; // EaseCurve handles (must check before other types) if (handleId === "easeCurve") return "easeCurve"; + // 3D handles + if (handleId === "3d") return "3d"; // Standard handles if (handleId === "video") return "video"; if (handleId === "audio" || handleId.startsWith("audio")) return "audio"; @@ -131,7 +133,7 @@ const getNodeHandles = (nodeType: string): { inputs: string[]; outputs: string[] case "easeCurve": return { inputs: ["video", "easeCurve"], outputs: ["video", "easeCurve"] }; case "glbViewer": - return { inputs: [], outputs: ["image"] }; + return { inputs: ["3d"], outputs: ["image"] }; default: return { inputs: [], outputs: [] }; } @@ -140,7 +142,7 @@ const getNodeHandles = (nodeType: string): { inputs: string[]; outputs: string[] interface ConnectionDropState { position: { x: number; y: number }; flowPosition: { x: number; y: number }; - handleType: "image" | "text" | "video" | "audio" | "easeCurve" | null; + handleType: "image" | "text" | "video" | "audio" | "3d" | "easeCurve" | null; connectionType: "source" | "target"; sourceNodeId: string | null; sourceHandleId: string | null; @@ -331,6 +333,11 @@ export function WorkflowCanvas() { return false; } + // 3D connections: 3d handles can only connect to matching 3d handles + if (sourceType === "3d" || targetType === "3d") { + return sourceType === "3d" && targetType === "3d"; + } + // Audio connections: audio handles can only connect to audio handles if (sourceType === "audio" || targetType === "audio") { return sourceType === "audio" && targetType === "audio"; @@ -455,7 +462,7 @@ export function WorkflowCanvas() { // Helper to find a compatible handle on a node by type const findCompatibleHandle = ( node: Node, - handleType: "image" | "text" | "video" | "audio" | "easeCurve", + handleType: "image" | "text" | "video" | "audio" | "3d" | "easeCurve", needInput: boolean, batchUsed?: Set ): string | null => { @@ -831,6 +838,12 @@ export function WorkflowCanvas() { // VideoStitch accepts audio targetHandleId = "audio"; } + } else if (handleType === "3d") { + if (nodeType === "glbViewer") { + targetHandleId = "3d"; + } else if (nodeType === "nanoBanana") { + sourceHandleIdForNewNode = "3d"; + } } else if (handleType === "easeCurve") { if (nodeType === "easeCurve") { targetHandleId = "easeCurve"; diff --git a/src/components/nodes/BaseNode.tsx b/src/components/nodes/BaseNode.tsx index 467eb8d7..61bfcdf8 100644 --- a/src/components/nodes/BaseNode.tsx +++ b/src/components/nodes/BaseNode.tsx @@ -26,6 +26,7 @@ interface BaseNodeProps { isExecuting?: boolean; hasError?: boolean; className?: string; + contentClassName?: string; minWidth?: number; minHeight?: number; headerAction?: ReactNode; @@ -48,6 +49,7 @@ export function BaseNode({ isExecuting = false, hasError = false, className = "", + contentClassName, minWidth = 180, minHeight = 100, headerAction, @@ -251,14 +253,14 @@ export function BaseNode({ />
-
+
{/* Title Section */}
{titlePrefix} @@ -452,7 +454,7 @@ export function BaseNode({
)}
-
{children}
+
{children}
); diff --git a/src/components/nodes/GLBViewerNode.tsx b/src/components/nodes/GLBViewerNode.tsx index 2a8cb608..4c362b1a 100644 --- a/src/components/nodes/GLBViewerNode.tsx +++ b/src/components/nodes/GLBViewerNode.tsx @@ -110,25 +110,13 @@ function Model({ url, onError }: { url: string; onError?: () => void }) { function SceneEnvironment({ groupRef }: { groupRef: React.RefObject }) { return ( - {/* Ground grid */} - - - {/* Subtle axis indicator at origin */} - - {/* X axis - red */} - - - - - {/* Z axis - blue */} - - - - - ); } @@ -278,6 +266,15 @@ export function GLBViewerNode({ id, data, selected }: NodeProps { + const el = viewportRef.current; + if (!el) return; + const stopWheel = (e: WheelEvent) => e.stopPropagation(); + el.addEventListener("wheel", stopWheel, { passive: false }); + return () => el.removeEventListener("wheel", stopWheel); + }, [nodeData.glbUrl]); + // Shared file processing logic for both click-to-upload and drag-and-drop const processFile = useCallback( (file: File) => { @@ -369,6 +366,7 @@ export function GLBViewerNode({ id, data, selected }: NodeProps updateNodeData(id, { comment: comment || undefined })} selected={selected} commentNavigation={commentNavigation ?? undefined} + contentClassName={nodeData.glbUrl ? "flex-1 min-h-0 overflow-hidden flex flex-col" : undefined} > {nodeData.glbUrl ? ( -
- {/* 3D Viewport — fills available space, responsive to node resize */} +
+ {/* 3D Viewport — fills node edge-to-edge */}
setIsInteracting(true)} onPointerUp={() => setIsInteracting(false)} > { gl.setClearColor(new THREE.Color("#1a1a1a")); gl.toneMapping = THREE.ACESFilmicToneMapping; @@ -423,56 +421,56 @@ export function GLBViewerNode({ id, data, selected }: NodeProps -
- {/* Controls bar */} -
-
- - {nodeData.filename} - - -
+ {/* Controls bar — overlaid on viewport */} +
+
+ + {nodeData.filename} + + +
-
- - +
+ + +
{/* Captured image preview */} {nodeData.capturedImage && ( -
+
@@ -509,13 +507,43 @@ export function GLBViewerNode({ id, data, selected }: NodeProps )} - {/* Output handle - image */} + {/* 3D input handle - accepts generated 3D models */} + +
+ 3D +
+ + {/* Output handle - image (captured viewport) */} +
+ Image +
); } diff --git a/src/components/nodes/GenerateImageNode.tsx b/src/components/nodes/GenerateImageNode.tsx index dbf397b9..5aebe6bc 100644 --- a/src/components/nodes/GenerateImageNode.tsx +++ b/src/components/nodes/GenerateImageNode.tsx @@ -61,7 +61,7 @@ const GEMINI_IMAGE_MODELS: { value: ModelType; label: string }[] = [ ]; // Image generation capabilities -const IMAGE_CAPABILITIES: ModelCapability[] = ["text-to-image", "image-to-image"]; +const IMAGE_CAPABILITIES: ModelCapability[] = ["text-to-image", "image-to-image", "text-to-3d", "image-to-3d"]; type NanoBananaNodeType = Node; @@ -210,9 +210,10 @@ export function GenerateImageNode({ id, data, selected }: NodeProps Prompt
- {/* Image output */} + {/* Output handle — switches to 3d when a 3D model is selected */} c.includes("3d")) ? "3d" : "image"} style={{ top: "50%" }} - data-handletype="image" + data-handletype={nodeData.selectedModel?.capabilities?.some((c: string) => c.includes("3d")) ? "3d" : "image"} /> {/* Output label */}
c.includes("3d")) ? "var(--handle-color-3d)" : "var(--handle-color-image)", }} > - Image + {nodeData.selectedModel?.capabilities?.some((c: string) => c.includes("3d")) ? "3D" : "Image"}
{/* Preview area */} - {nodeData.outputImage ? ( + {nodeData.output3dUrl ? ( +
+ + + + 3D Model Generated + Connect to 3D Viewer +
+ ) : nodeData.outputImage ? ( <>
{ + const regenOpts = options?.useStoredFallback ? { useStoredFallback: true } : undefined; + + switch (ctx.node.type) { + case "imageInput": + case "audioInput": + // Data source nodes — no execution needed + break; + case "annotation": + await executeAnnotation(ctx); + break; + case "prompt": + await executePrompt(ctx); + break; + case "promptConstructor": + await executePromptConstructor(ctx); + break; + case "nanoBanana": + await executeNanoBanana(ctx, regenOpts); + break; + case "generateVideo": + await executeGenerateVideo(ctx, regenOpts); + break; + case "llmGenerate": + await executeLlmGenerate(ctx, regenOpts); + break; + case "splitGrid": + await executeSplitGrid(ctx); + break; + case "output": + await executeOutput(ctx); + break; + case "outputGallery": + await executeOutputGallery(ctx); + break; + case "imageCompare": + await executeImageCompare(ctx); + break; + case "videoStitch": + await executeVideoStitch(ctx); + break; + case "easeCurve": + await executeEaseCurve(ctx); + break; + case "glbViewer": + await executeGlbViewer(ctx); + break; + } +} diff --git a/src/store/execution/index.ts b/src/store/execution/index.ts index f7f6088d..9354dd51 100644 --- a/src/store/execution/index.ts +++ b/src/store/execution/index.ts @@ -15,6 +15,7 @@ export { executeOutput, executeOutputGallery, executeImageCompare, + executeGlbViewer, } from "./simpleNodeExecutors"; export { executeNanoBanana } from "./nanoBananaExecutor"; diff --git a/src/store/execution/nanoBananaExecutor.ts b/src/store/execution/nanoBananaExecutor.ts index b178bc7c..2f3a739a 100644 --- a/src/store/execution/nanoBananaExecutor.ts +++ b/src/store/execution/nanoBananaExecutor.ts @@ -80,6 +80,9 @@ export async function executeNanoBanana( const provider = nodeData.selectedModel?.provider || "gemini"; const headers = buildGenerateHeaders(provider, providerSettings); + // Determine mediaType from model capabilities + const is3DModel = nodeData.selectedModel?.capabilities?.some((c: string) => c.includes("3d")); + const requestPayload = { images, prompt: promptText, @@ -90,6 +93,7 @@ export async function executeNanoBanana( selectedModel: nodeData.selectedModel, parameters: nodeData.parameters, dynamicInputs, + ...(is3DModel ? { mediaType: "3d" as const } : {}), }; try { @@ -119,6 +123,22 @@ export async function executeNanoBanana( const result = await response.json(); + // Handle 3D model response + if (result.success && result.model3dUrl) { + updateNodeData(node.id, { + output3dUrl: result.model3dUrl, + outputImage: null, + status: "complete", + error: null, + }); + + // Track cost if applicable + if (nodeData.selectedModel?.provider === "fal" && nodeData.selectedModel?.pricing) { + addIncurredCost(nodeData.selectedModel.pricing.amount); + } + return; + } + if (result.success && result.image) { const timestamp = Date.now(); const imageId = `${timestamp}`; @@ -144,6 +164,7 @@ export async function executeNanoBanana( updateNodeData(node.id, { outputImage: result.image, + output3dUrl: null, status: "complete", error: null, imageHistory: updatedHistory, diff --git a/src/store/execution/simpleNodeExecutors.ts b/src/store/execution/simpleNodeExecutors.ts index c77c8f93..ccba8281 100644 --- a/src/store/execution/simpleNodeExecutors.ts +++ b/src/store/execution/simpleNodeExecutors.ts @@ -221,3 +221,31 @@ export async function executeImageCompare(ctx: NodeExecutionContext): Promise { + const { node, getConnectedInputs, updateNodeData } = ctx; + const { model3d } = getConnectedInputs(node.id); + if (model3d) { + // Fetch the GLB URL and create a blob URL for the viewer + try { + const response = await fetch(model3d); + if (!response.ok) { + throw new Error(`Failed to fetch 3D model: ${response.status}`); + } + const blob = await response.blob(); + const blobUrl = URL.createObjectURL(blob); + updateNodeData(node.id, { + glbUrl: blobUrl, + filename: "generated.glb", + capturedImage: null, + }); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + console.error(`[Workflow] GLB Viewer node ${node.id} failed:`, message); + updateNodeData(node.id, { error: message }); + } + } +} diff --git a/src/store/utils/connectedInputs.ts b/src/store/utils/connectedInputs.ts index 97f529e4..b6893424 100644 --- a/src/store/utils/connectedInputs.ts +++ b/src/store/utils/connectedInputs.ts @@ -28,6 +28,7 @@ export interface ConnectedInputs { images: string[]; videos: string[]; audio: string[]; + model3d: string | null; text: string | null; dynamicInputs: Record; easeCurve: { bezierHandles: [number, number, number, number]; easingPreset: string | null } | null; @@ -52,7 +53,7 @@ function isTextHandle(handleId: string | null | undefined): boolean { /** * Extract output data and type from a source node */ -function getSourceOutput(sourceNode: WorkflowNode): { type: "image" | "text" | "video" | "audio"; value: string | null } { +function getSourceOutput(sourceNode: WorkflowNode): { type: "image" | "text" | "video" | "audio" | "3d"; value: string | null } { if (sourceNode.type === "imageInput") { return { type: "image", value: (sourceNode.data as ImageInputNodeData).image }; } else if (sourceNode.type === "audioInput") { @@ -60,7 +61,11 @@ function getSourceOutput(sourceNode: WorkflowNode): { type: "image" | "text" | " } else if (sourceNode.type === "annotation") { return { type: "image", value: (sourceNode.data as AnnotationNodeData).outputImage }; } else if (sourceNode.type === "nanoBanana") { - return { type: "image", value: (sourceNode.data as NanoBananaNodeData).outputImage }; + const nbData = sourceNode.data as NanoBananaNodeData; + if (nbData.output3dUrl) { + return { type: "3d", value: nbData.output3dUrl }; + } + return { type: "image", value: nbData.outputImage }; } else if (sourceNode.type === "generateVideo") { return { type: "video", value: (sourceNode.data as GenerateVideoNodeData).outputVideo }; } else if (sourceNode.type === "videoStitch") { @@ -92,6 +97,7 @@ export function getConnectedInputsPure( const images: string[] = []; const videos: string[] = []; const audio: string[] = []; + let model3d: string | null = null; let text: string | null = null; const dynamicInputs: Record = {}; @@ -145,7 +151,9 @@ export function getConnectedInputsPure( } // Route to typed arrays based on source output type - if (type === "video") { + if (type === "3d") { + model3d = value; + } else if (type === "video") { videos.push(value); } else if (type === "audio") { audio.push(value); @@ -172,7 +180,7 @@ export function getConnectedInputsPure( } } - return { images, videos, audio, text, dynamicInputs, easeCurve }; + return { images, videos, audio, model3d, text, dynamicInputs, easeCurve }; } /** diff --git a/src/store/utils/nodeDefaults.ts b/src/store/utils/nodeDefaults.ts index b0a169bd..625baee6 100644 --- a/src/store/utils/nodeDefaults.ts +++ b/src/store/utils/nodeDefaults.ts @@ -121,6 +121,7 @@ export const createDefaultNodeData = (type: NodeType): WorkflowNodeData => { inputImages: [], inputPrompt: null, outputImage: null, + output3dUrl: null, aspectRatio, resolution, model: legacyDefaults.model, // Keep legacy model field for backward compat diff --git a/src/store/workflowStore.ts b/src/store/workflowStore.ts index 0a7edf31..5aaa27f4 100644 --- a/src/store/workflowStore.ts +++ b/src/store/workflowStore.ts @@ -68,6 +68,7 @@ import { executeSplitGrid, executeVideoStitch, executeEaseCurve, + executeGlbViewer, } from "./execution"; import type { NodeExecutionContext } from "./execution"; export type { LevelGroup } from "./utils/executionUtils"; @@ -157,7 +158,7 @@ interface WorkflowStore { // Helpers getNodeById: (id: string) => WorkflowNode | undefined; - getConnectedInputs: (nodeId: string) => { images: string[]; videos: string[]; audio: string[]; text: string | null; dynamicInputs: Record; easeCurve: { bezierHandles: [number, number, number, number]; easingPreset: string | null } | null }; + getConnectedInputs: (nodeId: string) => { images: string[]; videos: string[]; audio: string[]; model3d: string | null; text: string | null; dynamicInputs: Record; easeCurve: { bezierHandles: [number, number, number, number]; easingPreset: string | null } | null }; validateWorkflow: () => { valid: boolean; errors: string[] }; // Global Image History @@ -841,9 +842,11 @@ export const useWorkflowStore = create((set, get) => ({ switch (node.type) { case "imageInput": case "audioInput": - case "glbViewer": // Data source nodes - no execution needed break; + case "glbViewer": + await executeGlbViewer(executionCtx); + break; case "annotation": await executeAnnotation(executionCtx); break; @@ -1050,6 +1053,30 @@ export const useWorkflowStore = create((set, get) => ({ return; } + // After regeneration, execute directly connected downstream consumer nodes + // (e.g. glbViewer needs to fetch+load 3D model from upstream nanoBanana) + const { edges: currentEdges } = get(); + const downstreamEdges = currentEdges.filter(e => e.source === nodeId); + for (const edge of downstreamEdges) { + const targetNode = get().nodes.find(n => n.id === edge.target); + if (!targetNode) continue; + const targetCtx = get()._buildExecutionContext(targetNode); + switch (targetNode.type) { + case "glbViewer": + await executeGlbViewer(targetCtx); + break; + case "output": + await executeOutput(targetCtx); + break; + case "outputGallery": + await executeOutputGallery(targetCtx); + break; + case "imageCompare": + await executeImageCompare(targetCtx); + break; + } + } + logger.info('node.execution', 'Node regeneration completed successfully', { nodeId }); set({ isRunning: false, currentNodeIds: [] }); diff --git a/src/types/api.ts b/src/types/api.ts index b0f9f622..ceb3f652 100644 --- a/src/types/api.ts +++ b/src/types/api.ts @@ -16,7 +16,7 @@ export interface GenerateRequest { resolution?: Resolution; // Only for Nano Banana Pro model?: ModelType; useGoogleSearch?: boolean; // Only for Nano Banana Pro - mediaType?: "image" | "video"; // Indicates expected output type for provider routing + mediaType?: "image" | "video" | "3d"; // Indicates expected output type for provider routing } export interface GenerateResponse { @@ -24,7 +24,8 @@ export interface GenerateResponse { image?: string; video?: string; videoUrl?: string; // For large videos, return URL directly - contentType?: "image" | "video"; + model3dUrl?: string; // For 3D models, return GLB URL directly + contentType?: "image" | "video" | "3d"; error?: string; } diff --git a/src/types/nodes.ts b/src/types/nodes.ts index 18c1683c..a1358837 100644 --- a/src/types/nodes.ts +++ b/src/types/nodes.ts @@ -150,6 +150,7 @@ export interface NanoBananaNodeData extends BaseNodeData { useGoogleSearch: boolean; // Only available for Nano Banana Pro parameters?: Record; // Model-specific parameters for external providers inputSchema?: ModelInputDef[]; // Model's input schema for dynamic handles + output3dUrl?: string | null; // URL to generated 3D GLB model status: NodeStatus; error: string | null; imageHistory: CarouselImageItem[]; // Carousel history (IDs only) @@ -321,7 +322,7 @@ export type WorkflowNode = Node & { /** * Handle types for node connections */ -export type HandleType = "image" | "text" | "audio" | "video" | "easeCurve"; +export type HandleType = "image" | "text" | "audio" | "video" | "3d" | "easeCurve"; /** * Default settings for node types - stored in localStorage diff --git a/src/types/providers.ts b/src/types/providers.ts index 9413fb2d..bfc1d536 100644 --- a/src/types/providers.ts +++ b/src/types/providers.ts @@ -20,6 +20,7 @@ export interface SelectedModel { modelId: string; displayName: string; pricing?: SelectedModelPricing; // Optional pricing info from provider API + capabilities?: string[]; // Model capabilities (e.g., "text-to-image", "image-to-3d") } export interface ProviderConfig {